'''
step one of https://github.com/mcarruthers/LCNAF-Named-Entity-Reconciliation
'''

import os
from os import path

# you'll need to install the following python modules -- should be as simple as running "pip install tqdm" and "pip install lxml" from the command-line
# lxml is a powerful xml document parser, and tqdm is used to show a command-line progress bar as the script progresses
from lxml import etree as ET
from tqdm import tqdm

# where are the eads?
ead_path = 'path/to/eads'

# tag names - set these to the tag types you want to capture the values of
tag_names = ["persname", "corpname", "geogname", "genreform"]

# initializes the dictionary we'll use to hold the tag names
controlaccess_term_dictionary = {}
for tag in tag_names:
	controlaccess_term_dictionary[tag] = []

# xpath to where in the ead document we'll be looking for the given tag names
controlaccess_xpath = '//controlaccess/*'

# go through the files
for filename in tqdm(os.listdir(ead_path)):
	# only look at xml files
	if filename.endswith(".xml"):
		# create lxml etree version of ead document
		ead_tree = ET.parse(path.join(ead_path, filename))

		# go through the ead and grab all the text appearing in the tag types defined above
		for sub in ead_tree.xpath(controlaccess_xpath):
			for tag in tag_names:
			    # we don't want to grab compound subject terms or empty strings
				if tag in sub.tag and sub.text is not None and "--" not in sub.text:
					controlaccess_term_dictionary[tag].append(sub.text.strip())

# write all the names to a file - one file for each tag type
for tag_type, names in controlaccess_term_dictionary.items():
	with open(tag_type + ".txt", mode="w") as text_file:
		for name in names:
			text_file.write(name.encode("utf-8") + '\n')

@@ -739,17 +739,20 @@
 _names:%0A
-%09
+    
 controla
@@ -972,17 +972,20 @@
 path)):%0A
-%09
+    
 # only l
@@ -1001,17 +1001,20 @@
 l files%0A
-%09
+    
 if filen
@@ -1035,18 +1035,24 @@
 .xml%22):%0A
-%09%09
+        
 # create
@@ -1087,18 +1087,24 @@
 ocument%0A
-%09%09
+        
 ead_tree
@@ -1147,18 +1147,24 @@
 name))%0A%0A
-%09%09
+        
 # go thr
@@ -1239,18 +1239,24 @@
 d above%0A
-%09%09
+        
 for sub 
@@ -1295,19 +1295,28 @@
 xpath):%0A
-%09%09%09
+            
 for tag 
@@ -1329,19 +1329,28 @@
 _names:%0A
-%09%09%09
+            
     # we
@@ -1409,20 +1409,32 @@
 strings%0A
-%09%09%09%09
+                
 if tag i
@@ -1498,13 +1498,28 @@
 xt:%0A
-%09%09%09%09%09
+                    
 cont
@@ -1698,17 +1698,20 @@
 tems():%0A
-%09
+    
 with ope
@@ -1755,18 +1755,24 @@
 t_file:%0A
-%09%09
+        
 for name
@@ -1786,11 +1786,20 @@
 es:%0A
-%09%09%09
+            
 text

convert tabs to spaces