#!/usr/bin/env python

"""
Generate file spec for 'yaml_files_list' reader.

python make_yaml_file_list.py  > john_smith_files.yaml

streamcorpus_pipeline ../../../configs/john-smith-serif.yaml  -i john_smith_files.yaml 
"""

import argparse
import os
import sys

import yaml


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument('--datadir', default=None, dest='datadir')
    ap.add_argument('arg', nargs='?', default=None)
    ap.add_argument('-o', '--out', default=None)
    args = ap.parse_args()

    if args.out and (args.out != '-'):
        outf = open(args.out, 'wb')
    else:
        outf = sys.stdout

    base_path = args.datadir or args.arg
    if base_path:
        base_path = os.path.abspath(base_path)
    metadata = dict()
    if base_path:
        #metadata['abs_url_base'] = base_path
        metadata['root_path'] = base_path
    metadata['source'] = 'bagga-and-baldwin'
    metadata['annotator_id'] = 'bagga-and-baldwin'

    entries = []
    if not base_path:
        base_path = os.path.abspath(os.getcwd())
    for fname in os.listdir(base_path):
        if fname[0] == '.':
            continue
        fpath = os.path.join(base_path, fname)
        if os.path.isdir(fpath):
            entry = {
                'target_id': fname,
                'doc_path': fname + '/',
                'mentions': ['John', 'Smith'],
                }
            entries.append(entry)
    metadata['entries'] = entries
    yaml.dump(metadata, stream=outf)


if __name__ == '__main__':
    main()

@@ -73,42 +73,127 @@
 er.%0A
-%0Apython make_yaml_file_list.py  %3E 
+To run from streamcorpus-pipeline dir:%0A%0Apython scripts/make_yaml_file_list.py --datadir=data/john-smith/original --out=
 john
@@ -237,17 +237,8 @@
 ine 
-../../../
 conf
@@ -263,17 +263,16 @@
 if.yaml 
- 
 -i john_
@@ -553,24 +553,101 @@
 fault=None)%0A
+    ap.add_argument('--abs-url-is-path', action='store_true', default=False)%0A
     args = a
@@ -932,87 +932,497 @@
     
-#metadata%5B'abs_url_base'%5D = base_path%0A        metadata%5B'root_path'%5D = base_path
+metadata%5B'root_path'%5D = base_path%0A    if base_path and args.abs_url_is_path:%0A        # for debugging, use local filesystem absolute pathe%0A        metadata%5B'abs_url_base'%5D = base_path%0A    else:%0A        # For the purposes of this analysis, the trec-kba project shall be%0A        # considered to be the canonical absolute url source for the %22John%0A        # Smith%22 data.%0A        metadata%5B'abs_url_base'%5D = 'https://raw2.github.com/trec-kba/streamcorpus-pipeline/master/data/john-smith/original'
 %0A   

tweak to set abs_path