#!/usr/bin/env python

'''
  pbindex - create an index on field in a protobuf stream
    currently only supports one index type: -i cdb
'''

import sys
import getopt
import lwpb
import lwpb.stream
import lwpb.codec


def shift(L): e = L[0] ; del L[0:1] ; return e

def main():

  key = None
  typename = ""
  pb2file = None
  pb2codec = None
  indextype = None
  indexer = None
  outfile = None
  tempfile = None
  fin = sys.stdin
  infile = '-'
  verbose = 0

  opts, args = getopt.getopt(sys.argv[1:], 'p:k:i:o:t:m:v')

  for o, a in opts:
    if o == '-p':
      pb2file = a
    elif o == '-k':
      key = a
    elif o == '-m':
      typename = a
    elif o == '-o':
      outfile = a
    elif o == '-t':
      tempfile = a
    elif o == '-i':
      indextype = a
    elif o == '-v':
      verbose += 1

  if len(args):
    infile = shift(args)
    fin = file(infile)

  if key == None:
    raise Exception("missing key parameter, specify with -k")

  # create the indexer object

  if indextype == 'cdb':

    import cdb
    outfile = "%s.%s.%s.idx" % (infile, key, indextype)
    tempfile = "%s.%s.%s.tmp" % (infile, key, indextype)
    indexer = cdb.cdbmake( outfile, tempfile )

  elif indextype == None:

    raise Exception("missing index type parameter, specify with -i")

  # create the stream reader

  pb2codec = lwpb.codec.MessageCodec(pb2file=pb2file, typename=typename)
  reader = lwpb.stream.StreamReader(fin, codec=pb2codec)

  # index all the records

  for record in reader:
    indexkey = str(record[key])
    indexval = str(reader.current_offset)
    if verbose: print >> sys.stderr, indexkey
    indexer.add( indexkey, indexval )

  indexer.finish()

  return 0


if __name__ == '__main__':
  sys.exit(main())



@@ -271,16 +271,68 @@
 ain():%0A%0A
+  reader_format = 'pb'%0A  delim = '%5Ct'%0A  fields = %5B%5D%0A
   key = 
@@ -550,16 +550,22 @@
 v%5B1:%5D, '
+R:F:d:
 p:k:i:o:
@@ -604,16 +604,144 @@
  if o ==
+ '-R':%0A      reader_format = a%0A    elif o == '-F':%0A      fields = a.split(',')%0A    elif o == '-d':%0A      delim = a%0A    elif o ==
  '-p':%0A 
@@ -1208,16 +1208,32 @@
  cdb%0A   
+ if not outfile:
  outfile
@@ -1238,20 +1238,20 @@
 le = %22%25s
-.%25s.
+-%25s-
 %25s.idx%22 
@@ -1280,16 +1280,33 @@
 ype)%0A   
+ if not tempfile:
  tempfil
@@ -1317,45 +1317,22 @@
 %22%25s.
-%25s.%25s.
 tmp%22 %25 
-(in
+out
 file
-, key, indextype)
 %0A   
@@ -1540,16 +1540,17 @@
 geCodec(
+ 
 pb2file=
@@ -1575,18 +1575,50 @@
 typename
-)%0A
+ )%0A%0A  if reader_format == 'pb':%0A  
   reader
@@ -1642,15 +1642,16 @@
 ream
-R
+r
 eader(
+ 
 fin,
@@ -1665,16 +1665,191 @@
 pb2codec
+ )%0A  elif reader_format == 'txt':%0A    import percent.stream%0A    reader = percent.stream.PercentCodecReader( fin, '%5Ct', fields )%0A  else:%0A    raise Exception(%22bad reader format%22
 )%0A%0A  # i

enable indexing on txt format files