Data Minification
A function that takes a dataframe and converts the data type of each numerical column to the optimum type based on the size of the values. This reduces the size in memory of the dataframe.
def minification(df):
for col in df.columns:
if is_numeric_dtype(df[col]):
if is_integer_dtype(df[col]):
max_value = df[col].max()
min_value = df[col].min()
if min_value >= 0:
if max_value < 255:
df[col] = df[col].astype(np.uint8)
elif max_value < 65535:
df[col] = df[col].astype(np.uint16)
elif max_value < 4294967295:
df[col] = df[col].astype(np.uint32)
else:
df[col] = df[col].astype(np.uint64)
else:
if min_value > np.iinfo(np.int8).min and max_value < np.iinfo(np.int8).max:
df[col] = df[col].astype(np.int8)
elif min_value > np.iinfo(np.int16).min and max_value < np.iinfo(np.int16).max:
df[col] = df[col].astype(np.int16)
elif min_value > np.iinfo(np.int32).min and max_value < np.iinfo(np.int32).max:
df[col] = df[col].astype(np.int32)
elif min_value > np.iinfo(np.int64).min and max_value < np.iinfo(np.int64).max:
df[col] = df[col].astype(np.int64)
else:
df[col] = df[col].astype(np.float32)
return df
minification(df)By detro - Last Updated Nov. 23, 2020, 8:26 p.m.
Very useful, thanks
String - Lstrip - Whitespace - Rstrip
DatasnipsNov. 23, 2020, 10:10 p.m.