def multiprocess_mapping ( func , iterable ) : if os . name == 'nt' : # In Windows there is no fork. return list ( map ( func , iterable ) ) try : p = multiprocessing . Pool ( ) return_data = list ( p . imap ( func , iterable ) ) p . close ( ) p . join ( ) return return_data except OSError : return list ( map ( func , iterable ) )
python multiprocessing how to take only one iterable out of many
Multiprocess mapping the given function on the given iterable .
def multiprocess_mapping(func, iterable):
    """Multiprocess mapping the given function on the given iterable.

    This only works in Linux and Mac systems since Windows has no forking capability. On Windows we fall back on
    single processing. Also, if we reach memory limits we fall back on single cpu processing.

    Args:
        func (func): the function to apply
        iterable (iterable): the iterable with the elements we want to apply the function on
    """
    if os.name == 'nt':  # In Windows there is no fork.
        return list(map(func, iterable))
    try:
        p = multiprocessing.Pool()
        return_data = list(p.imap(func, iterable))
        p.close()
        p.join()
        return return_data
    except OSError:
        return list(map(func, iterable))