def show_partitioning ( rdd , show = True ) : if show : partitionCount = rdd . getNumPartitions ( ) try : valueCount = rdd . countApprox ( 1000 , confidence = 0.50 ) except : valueCount = - 1 try : name = rdd . name ( ) or None except : pass name = name or "anonymous" logging . info ( "For RDD %s, there are %d partitions with on average %s values" % ( name , partitionCount , int ( valueCount / float ( partitionCount ) ) ) )
how to print rdd size in python
Seems to be significantly more expensive on cluster than locally
def show_partitioning(rdd, show=True):
    """Seems to be significantly more expensive on cluster than locally"""
    if show:
        partitionCount = rdd.getNumPartitions()
        try:
            valueCount = rdd.countApprox(1000, confidence=0.50)
        except:
            valueCount = -1
        try:
            name = rdd.name() or None
        except:
            pass
        name = name or "anonymous"
        logging.info("For RDD %s, there are %d partitions with on average %s values" % 
                     (name, partitionCount, int(valueCount/float(partitionCount))))