def __run_spark_submit ( lane_yaml , dist_dir , spark_home , spark_args , silent ) : # spark-submit binary cmd = [ 'spark-submit' if spark_home is None else os . path . join ( spark_home , 'bin/spark-submit' ) ] # Supplied spark arguments if spark_args : cmd += spark_args # Packaged App & lane cmd += [ '--py-files' , 'libs.zip,_framework.zip,tasks.zip' , 'main.py' ] cmd += [ '--lane' , lane_yaml ] logging . info ( 'Submitting to Spark' ) logging . debug ( str ( cmd ) ) # Submit devnull = open ( os . devnull , 'w' ) outp = { 'stderr' : STDOUT , 'stdout' : devnull } if silent else { } call ( cmd , cwd = dist_dir , env = MY_ENV , * * outp ) devnull . close ( )
submit python code in pysopark
Submits the packaged application to spark using a spark - submit subprocess
def __run_spark_submit(lane_yaml, dist_dir, spark_home, spark_args, silent):
    """
    Submits the packaged application to spark using a `spark-submit` subprocess

    Parameters
    ----------
    lane_yaml (str): Path to the YAML lane definition file
    dist_dir (str): Path to the directory where the packaged code is located
    spark_args (str): String of any additional spark config args to be passed when submitting
    silent (bool): Flag indicating whether job output should be printed to console
    """
    # spark-submit binary
    cmd = ['spark-submit' if spark_home is None else os.path.join(spark_home, 'bin/spark-submit')]

    # Supplied spark arguments
    if spark_args:
        cmd += spark_args

    # Packaged App & lane
    cmd += ['--py-files', 'libs.zip,_framework.zip,tasks.zip', 'main.py']
    cmd += ['--lane', lane_yaml]

    logging.info('Submitting to Spark')
    logging.debug(str(cmd))

    # Submit
    devnull = open(os.devnull, 'w')
    outp = {'stderr': STDOUT, 'stdout': devnull} if silent else {}
    call(cmd, cwd=dist_dir, env=MY_ENV, **outp)
    devnull.close()