#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Script to convert a MARCXML dump into a series of XML files à la OAI-PMH.
Tested with input MARCXML made by yaz-marcdump from SbnWeb binary MARC export.
Usage: ./BEIC_marcxml2oaipmh.py <input.xml> <outprefix>
"""
#
# (C) Federico Leva e Fondazione BEIC, 2018
#
# Distributed under the terms of the MIT license.
#
__version__ = '0.1.0'
#

from datetime import datetime
from lxml import etree
import sys

def getOaiHeader():
	return """<?xml version="1.0" encoding="UTF-8" ?>
	<OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd">
	<responseDate>""" + datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') + """</responseDate>
	<request></request>
	<ListRecords>
	"""

def getOaiFooter():
	return "</ListRecords>\n</OAI-PMH>"

def makeWrappedRecord(identifier, record):
	return "<record>\n<header>\n<identifier>{}</identifier>\n<datestamp>{}</datestamp>\n</header>\n<metadata>\n{}</metadata>\n</record>".format(
		identifier, datetime.utcnow().strftime('%Y-%m-%d'), record)

def makeOutputFile(recordcount):
	filenumber = str(recordcount / 1000).zfill(4)
	oai = open(sys.argv[2] + '-' + filenumber + '.xml', 'w')
	oai.write(getOaiHeader())
	return oai

if __name__ == '__main__':
	marcxml=open(sys.argv[1], 'r')
	oai = makeOutputFile(0)
	curidentifier = ''
	recordcount = 0

	for event, elem in etree.iterparse(marcxml):
		# The name of the element will be prefixed with the namespace, for instance:
		# end <Element {http://www.loc.gov/MARC21/slim}record at 0x7fd5cfc20fc8>
		if event == "end" and elem.tag.endswith("controlfield") and "001" in elem.attrib.values():
			curidentifier = elem.text
			print "Found identifier {}".format(curidentifier)
		if event == "end" and elem.tag.endswith("record"):
			recordcount += 1
			if recordcount % 1000 == 0:
				# 1000 records done. Close current file and open a new one.
				oai.write(getOaiFooter())
				oai = makeOutputFile(recordcount)

			oai.write(makeWrappedRecord(curidentifier, etree.tostring(elem)))
			elem.clear()
			curidentifier = ''

@@ -58,15 +58,25 @@
  to 
-convert
+wrap records from
  a M
@@ -205,17 +205,158 @@
 C export
-.
+, like:%0A$ yaz-marcdump -f utf8 -t utf8 -i marc -o marcxml in.clean.mrc %3E out.xml%0AControl character are left:%0A$ grep '%5B%5B:cntrl:%5D%5D%5BHI%5D' in.marc%0A
 %0AUsage: 
@@ -563,16 +563,16 @@
 atetime%0A
-
 from lxm
@@ -586,16 +586,26 @@
 t etree%0A
+import re%0A
 import s
@@ -2009,16 +2009,351 @@
 tifier)%0A
+%09%09if event == %22end%22 and elem.tag.endswith(%22subfield%22) and %22a%22 in elem.attrib.values() and elem.text:%0A%09%09%09# The %22asterisk%22 for sorting is given as two prefixes %5B%5B:cntrl:%5D%5D%5BHI%5D in%0A%09%09%09# the original MRC, but control characters are removed in XML conversion%0A%09%09%09elem.text = re.sub(r%22H(%5BA-Z%5Cd%5C%5B%5C%5D%5D.*%5CW)I(%5Cw)%22, r%22%5C1%5C2%22, elem.text, re.UNICODE)%0A
 %09%09if eve

Remove pseudo-asterisk control characters for sorting