python中pandas常用命令详解

pd.read_csv("data.csv")

pd.read_csv("data.csv", usecols=["date", "price"])

pd.read_csv("data.csv", parse_dates=["date"])

pd.read_csv("data.csv", dtype={"house_type": "category"})

pd.read_csv("data.csv", index_col="date")

pd.read_csv("data.csv", nrows=100)

pd.read_csv("data.csv", skiprows=[1, 5])  # skips line 1 and 5
pd.read_csv("data.csv", skiprows=100)  # skips the first 100 lines
pd.read_csv("data.csv", skiprows=lambda x: x > 0 and np.random.rand() > 0.1) # skip 90% of the rows

pd.read_csv("data.csv", na_values=["?"])

pd.read_csv("data.csv", true_values=["yes"], false_values=["no"])

import glob
import os
files = glob.glob("file_*.csv")
result = pd.concat([pd.read_csv(f) for f in files], ignore_index=True)

df = pd.read_clipboard() 

from tabula import read_pdf
# Read pdf into list of DataFrame
df = read_pdf('test.pdf', pages='all')

import pandas_profiling
df = pd.read_csv("data.csv")
profile = df.profile_report(title="Pandas Profiling Report")
profile.to_file(output_file="output.html")

# 选择
df.select_dtypes(include="number")
df.select_dtypes(include=["category", "datetime"])
 
# 排除
df.select_dtypes(exclude="object")

df.infer_objects().dtypes

pd.to_numeric(df.numeric_col, downcast="integer") # smallest signed int dtype
pd.to_numeric(df.numeric_col, downcast="float")  # smallest float dtype

# apply to whole data frame
df = df.apply(pd.to_numeric, errors="coerce")
# apply to specific columns
pd.to_numeric(df.numeric_column, errors="coerce")
# filling NA values with zero
pd.to_numeric(df.numeric_column, errors="coerce").fillna(0)

df = df.astype(
    {
        "date": "datetime64[ns]",
        "price": "int",
        "is_weekend": "bool",
        "status": "category",
    }
)

df = df.rename({"PRICE": "price", "Date (mm/dd/yyyy)": "date"}, axis=1)

df.add_prefix("pre_")
df.add_suffix("_suf")

# create new column of Fahrenheit values from Celcius
df.assign(temp_f=lambda x: x.temp_c * 9 / 5 + 32)

random_col = np.random.randint(10, size=len(df))
df.insert(3, 'random_col', random_col) # inserts at third column

df["logic"] = np.where(df["price"] > 5, "high", "low")

df.drop('col1', axis=1, inplace=True)
df = df.drop(['col1','col2'], axis=1)
s = df.pop('col')
del df['col']
df.drop(df.columns[0], inplace=True)

df.columns = df.columns.str.lower()
df.columns = df.columns.str.replace(' ', '_')

df['name'].str.contains("John")
df['phone_num'].str.contains('...-...-....', regex=True)  # regex
df['email'].str.contains('gmail')

pattern = '([A-Z0-9._%+-]+)@([A-Z0-9.-]+)\\.([A-Z]{2,4})'
df['email'].str.findall(pattern, flags=re.IGNORECASE)

def missing_vals(df):
    """prints out columns with perc of missing values"""
    missing = [
        (df.columns[idx], perc)
        for idx, perc in enumerate(df.isna().mean() * 100)
        if perc > 0
    ]
 
    if len(missing) == 0:
        return "no missing values"
    
    # sort desc by perc
    missing.sort(key=lambda x: x[1], reverse=True)
 
    print(f"There are a total of {len(missing)} variables with missing values\n")
 
    for tup in missing:
        print(str.ljust(f"{tup[0]:<20} => {round(tup[1], 3)}%", 1))
missing_vals(df)

# drop 
df.dropna(axis=0)
df.dropna(axis=1)
# impute
df.fillna(0)
df.fillna(method="ffill")
df.fillna(method='bfill')
# replace
df.replace( -999, np.nan)
df.replace("?", np.nan)
# interpolate
ts.interpolate() # time series
df.interpolate() # fill all consecutive values forward
df.interpolate(limit=1) # fill one consecutive value forward
df.interpolate(limit=1, limit_direction="backward")
df.interpolate(limit_direction="both")

# from today
date.today() + datetime.timedelta(hours=30)
date.today() + datetime.timedelta(days=30)
date.today() + datetime.timedelta(weeks=30)
 
# ago
date.today() - datetime.timedelta(days=365)

df[(df["Date"] > "2015-01-01") & (df["Date"] < "2017-01-01")]

df[(df["Date"] > "2015-01-01") & (df["Date"] < "2017-01-01")]

format_dict = {
    "Date": "{:%d/%m/%y}",
    "Open": "${:.2f}",
    "Close": "${:.2f}",
    "Volume": "{:,}",
}
 
df.style.format(format_dict)

(
    df.style.format(format_dict)
    .hide_index()
    .highlight_min(["Open"], color="red")
    .highlight_max(["Open"], color="green")
    .background_gradient(subset="Close", cmap="Greens")
    .bar('Volume', color='lightblue', align='zero')
    .set_caption('Tesla Stock Prices in 2017')
)

df['col'].idxmin()
df['col'].idxmax()

df.applymap(lambda x: np.log(x))

df.sample(frac=1, random_state=7).reset_index(drop=True)

df['col_name'].pct_change()

df['rank'] = df['column_to_rank'].rank()

df.memory_usage().sum() / (1024**2) #converting to MB

df.explode("col_name").reset_index(drop=True)

subclass = df.MSSubClass
subclass.value_counts()
top_five = subclass.value_counts().nlargest(5).index
mssubclass_new = subclass.where(subclass.isin(top_five), other="Other")
mssubclass_new.value_counts()