def _to_corrected_pandas_type ( dt ) : import numpy as np if type ( dt ) == ByteType : return np . int8 elif type ( dt ) == ShortType : return np . int16 elif type ( dt ) == IntegerType : return np . int32 elif type ( dt ) == FloatType : return np . float32 else : return None
change the data type of datarframe column python
When converting Spark SQL records to Pandas DataFrame the inferred data type may be wrong . This method gets the corrected data type for Pandas if that type may be inferred uncorrectly .
def _to_corrected_pandas_type(dt):
    """
    When converting Spark SQL records to Pandas DataFrame, the inferred data type may be wrong.
    This method gets the corrected data type for Pandas if that type may be inferred uncorrectly.
    """
    import numpy as np
    if type(dt) == ByteType:
        return np.int8
    elif type(dt) == ShortType:
        return np.int16
    elif type(dt) == IntegerType:
        return np.int32
    elif type(dt) == FloatType:
        return np.float32
    else:
        return None