def token ( name ) : def wrap ( f ) : tokenizers . append ( ( name , f ) ) return f return wrap
add a token interface python
Marker for a token
def token(name):
    """Marker for a token

    :param str name: Name of tokenizer
    """

    def wrap(f):
        tokenizers.append((name, f))
        return f

    return wrap