def has_emb_layer_norm_before(model_state):
    """ Determine whether layer norm needs to be applied before the encoder """
    return any(k.startswith("emb_layer_norm_before") for k, param in model_state.items())