import re


def tok_from_norm(data):
	"""
	Takes TT-SGML, extracts norm attribute, and replaces existing tokens with norm values while retaining SGML tags.
	Used to feed parser norms while retaining SGML sentence separators.

	:param data: TTSGML with <norm norm=...> and raw tokens to replace
	:return: TTSGML with tags preserved and tokens replace by norm attribute values
	"""

	outdata = []
	norm = ""
	for line in data.replace("\r","").split("\n"):
		if line.startswith("<"):
			m = re.search(r'norm="([^"]*)"',line)
			if m is not None:
				norm = m.group(1)
			outdata.append(line)
		else:
			if norm != "":
				outdata.append(norm)
				norm=""
	return "\n".join(outdata) + "\n"