def apply ( self , func , workers = 1 , job_size = 10000 ) : if workers == 1 : for lines in self . iter_chunks ( job_size ) : yield func ( lines ) else : with ProcessPoolExecutor ( max_workers = workers ) as executor : for result in executor . map ( func , self . iter_chunks ( job_size ) ) : yield result
python multiprocessing 100k lines file
Apply func to lines of text in parallel or sequential .
def apply(self, func, workers=1, job_size=10000):
    """Apply `func` to lines of text in parallel or sequential.

    Args:
      func : a function that takes a list of lines.
    """
    if workers == 1:
      for lines in self.iter_chunks(job_size):
        yield func(lines)
    else:
      with ProcessPoolExecutor(max_workers=workers) as executor:
        for result in executor.map(func, self.iter_chunks(job_size)):
          yield result