labs/Lab-2/Lab-2-Solution/05_Pandas_Log_Cleaning_Solution.ipynb

Module 2 — Lab 5: Pandas — Loading, Inspecting, and Cleaning Logs

Topics: CSV, DataFrame, missing values, duplicates, datetime, filtering

Solution version: This notebook contains the complete solution. Run the cells in order and compare the results with your Lab notebook.

Scenario

The data/auth_log.csv file contains duplicate rows, invalid timestamps, missing critical fields, and inconsistent capitalization. The goal is to produce a clean DataFrame ready for analysis.

In [ ]:

from pathlib import Path
import pandas as pd

DATA_PATH = Path("data/auth_log.csv")
OUTPUT_DIR = Path("output")
OUTPUT_DIR.mkdir(exist_ok=True)

log_df = pd.read_csv(DATA_PATH)
log_df.head()

1. Inspect data quality

In [ ]:

print("Shape:", log_df.shape)
print("Duplicate rows:", log_df.duplicated().sum())
print("Missing values by column:")
print(log_df.isna().sum())
print("\nData types:")
print(log_df.dtypes)

2. Remove duplicates and normalize timestamps and critical fields

In [ ]:

clean_df = log_df.copy()
clean_df = clean_df.drop_duplicates()

clean_df["timestamp"] = pd.to_datetime(
    clean_df["timestamp"], errors="coerce", utc=True
)

clean_df = clean_df.dropna(
    subset=["timestamp", "source_ip", "status"]
)

3. Normalize text and create time-based fields

In [ ]:

clean_df["source_ip"] = clean_df["source_ip"].str.strip()
clean_df["username"] = clean_df["username"].fillna("unknown").str.strip().str.lower()
clean_df["status"] = clean_df["status"].str.strip().str.upper()

clean_df = clean_df[clean_df["status"].isin(["SUCCESS", "FAILED"])]
clean_df["hour"] = clean_df["timestamp"].dt.hour
clean_df = clean_df.sort_values("timestamp").reset_index(drop=True)

clean_df.head()

4. Filter and aggregate

In [ ]:

failed_df = clean_df[clean_df["status"] == "FAILED"].copy()
failed_by_ip = failed_df["source_ip"].value_counts().rename("failed_attempts")
status_counts = clean_df["status"].value_counts()

print("Status counts:")
print(status_counts)
print("\nFailed attempts by IP:")
print(failed_by_ip)

5. Export clean data

In [ ]:

clean_df.to_csv(OUTPUT_DIR / "clean_auth_log.csv", index=False)
clean_df.to_json(
    OUTPUT_DIR / "clean_auth_log.json",
    orient="records",
    indent=2,
    date_format="iso",
)
failed_by_ip.to_csv(
    OUTPUT_DIR / "failed_login_by_ip.csv",
    header=True,
)

print("Exported files to:", OUTPUT_DIR.resolve())

Self-check

In [ ]:

assert log_df.duplicated().sum() >= 1
assert clean_df.duplicated().sum() == 0
assert clean_df["timestamp"].isna().sum() == 0
assert set(clean_df["status"]) <= {"SUCCESS", "FAILED"}
assert clean_df["username"].str.strip().eq(clean_df["username"]).all()
assert failed_by_ip.index[0] == "198.51.100.14"
assert (OUTPUT_DIR / "clean_auth_log.csv").exists()
assert (OUTPUT_DIR / "clean_auth_log.json").exists()
print("✓ Lab 5 completed")