"""
    Helper functions for data quality work.
"""

def filled(f,d):
    """
        Check if a field exists in a dictionary, and if that field is not None or the empty string.
    """
    return f in d and d[f] is not None and d[f] != ""

def field_to_flag(f,suffix):
    """
        Convert a field in the form of namespace:termname, along with a flag suffix, to a flag of the form:

            namespace_termname_suffix
    """
    return "{0}_{1}".format(f.replace(":","_"),suffix)

def flag_to_field(f):
    """
        Convert a flag in the form generated by field_to_flag back to the field name it came from.
    """
    return ":".join(f.split("_")[:-1])

def getfield(f, d, t="text"):
    fl = f.lower()
    if fl in d:
        f = fl
    if f in d and d[f] is not None:
        if t == "list":
            return [x.strip() for x in d[f]]
        else:
            if isinstance(d[f], str) or isinstance(d[f], unicode):
                return d[f].strip()
            else:
                return d[f]
    else:
        return None

class WorkflowBase(object):
    """
        A base class to derive all workflows from.
    """
    def __init__(self):
        self.required_fields = []
        self.outputs = []
        self.flags = []

    def process(self,d):
        """
            Create the result format with the flags array.
        """
        r = {"flags":[]}
        return r

class CombinedWorkflow(WorkflowBase):
    """
        A basic class for combining multiple workflows together.

        This simply combines the three list properties from all the classes, and executes them in the specified order when process is called.
    """
    def __init__(self, workflow_list):
        super(CombinedWorkflow,self).__init__()
        self._workflows = []

        for w in workflow_list:
            self.add_workflow(w)

    def add_workflow(self, w):
        """
            Add a new workflow to the combined workflow.
        """
        self.required_fields = list(set(self.required_fields + w.required_fields))
        self.outputs = list(set(self.outputs + w.outputs))
        self.flags = list(set(self.flags + w.flags))
        self._workflows.append(w)

    def process(self, d):
        """
            Process all the workflows together.
        """
        r = super(CombinedWorkflow,self).process(d)

        # Update is shallow, so it won't merge the flag lists
        # This might need custom merge code if you have lots of lists in our data
        flags = set()
        for w in workflows:
            rt = w.process(d)
            flags.union(rt["flags"])
            r.update(rt)
        r["flags"] = list(flags)
@@ -2506,32 +2506,38 @@
        for w in 
+self._
 workflows:%0A     

fix ref