labs/Lab-2/Lab-2-Solution/05_Pandas_Log_Cleaning_Solution.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "97d2f264",
   "metadata": {},
   "source": [
    "# Module 2 — Lab 5: Pandas — Loading, Inspecting, and Cleaning Logs\n",
    "\n",
    "\n",
    "**Topics:** CSV, DataFrame, missing values, duplicates, datetime, filtering\n",
    "\n",
    "\n",
    "> **Solution version:** This notebook contains the complete solution. Run the cells in order and compare the results with your Lab notebook.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7b1ce82e",
   "metadata": {},
   "source": [
    "## Scenario\n",
    "\n",
    "The `data/auth_log.csv` file contains duplicate rows, invalid timestamps, missing critical fields, and inconsistent capitalization. The goal is to produce a clean DataFrame ready for analysis.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9dea9739",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import pandas as pd\n",
    "\n",
    "DATA_PATH = Path(\"data/auth_log.csv\")\n",
    "OUTPUT_DIR = Path(\"output\")\n",
    "OUTPUT_DIR.mkdir(exist_ok=True)\n",
    "\n",
    "log_df = pd.read_csv(DATA_PATH)\n",
    "log_df.head()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6e6da8ed",
   "metadata": {
    "tags": [
     "exercise"
    ]
   },
   "source": [
    "## 1. Inspect data quality\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9ef821c",
   "metadata": {
    "tags": [
     "solution"
    ]
   },
   "outputs": [],
   "source": [
    "print(\"Shape:\", log_df.shape)\n",
    "print(\"Duplicate rows:\", log_df.duplicated().sum())\n",
    "print(\"Missing values by column:\")\n",
    "print(log_df.isna().sum())\n",
    "print(\"\\nData types:\")\n",
    "print(log_df.dtypes)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b759b1c",
   "metadata": {
    "tags": [
     "exercise"
    ]
   },
   "source": [
    "## 2. Remove duplicates and normalize timestamps and critical fields\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d4342ee",
   "metadata": {
    "tags": [
     "solution"
    ]
   },
   "outputs": [],
   "source": [
    "clean_df = log_df.copy()\n",
    "clean_df = clean_df.drop_duplicates()\n",
    "\n",
    "clean_df[\"timestamp\"] = pd.to_datetime(\n",
    "    clean_df[\"timestamp\"], errors=\"coerce\", utc=True\n",
    ")\n",
    "\n",
    "clean_df = clean_df.dropna(\n",
    "    subset=[\"timestamp\", \"source_ip\", \"status\"]\n",
    ")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c5c59040",
   "metadata": {
    "tags": [
     "exercise"
    ]
   },
   "source": [
    "## 3. Normalize text and create time-based fields\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "012ba902",
   "metadata": {
    "tags": [
     "solution"
    ]
   },
   "outputs": [],
   "source": [
    "clean_df[\"source_ip\"] = clean_df[\"source_ip\"].str.strip()\n",
    "clean_df[\"username\"] = clean_df[\"username\"].fillna(\"unknown\").str.strip().str.lower()\n",
    "clean_df[\"status\"] = clean_df[\"status\"].str.strip().str.upper()\n",
    "\n",
    "clean_df = clean_df[clean_df[\"status\"].isin([\"SUCCESS\", \"FAILED\"])]\n",
    "clean_df[\"hour\"] = clean_df[\"timestamp\"].dt.hour\n",
    "clean_df = clean_df.sort_values(\"timestamp\").reset_index(drop=True)\n",
    "\n",
    "clean_df.head()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "354fc8c0",
   "metadata": {
    "tags": [
     "exercise"
    ]
   },
   "source": [
    "## 4. Filter and aggregate\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38f69103",
   "metadata": {
    "tags": [
     "solution"
    ]
   },
   "outputs": [],
   "source": [
    "failed_df = clean_df[clean_df[\"status\"] == \"FAILED\"].copy()\n",
    "failed_by_ip = failed_df[\"source_ip\"].value_counts().rename(\"failed_attempts\")\n",
    "status_counts = clean_df[\"status\"].value_counts()\n",
    "\n",
    "print(\"Status counts:\")\n",
    "print(status_counts)\n",
    "print(\"\\nFailed attempts by IP:\")\n",
    "print(failed_by_ip)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "748f22a9",
   "metadata": {},
   "source": [
    "## 5. Export clean data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fdcb2bf7",
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_df.to_csv(OUTPUT_DIR / \"clean_auth_log.csv\", index=False)\n",
    "clean_df.to_json(\n",
    "    OUTPUT_DIR / \"clean_auth_log.json\",\n",
    "    orient=\"records\",\n",
    "    indent=2,\n",
    "    date_format=\"iso\",\n",
    ")\n",
    "failed_by_ip.to_csv(\n",
    "    OUTPUT_DIR / \"failed_login_by_ip.csv\",\n",
    "    header=True,\n",
    ")\n",
    "\n",
    "print(\"Exported files to:\", OUTPUT_DIR.resolve())\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4b07b67e",
   "metadata": {},
   "source": [
    "## Self-check\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b16f153c",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert log_df.duplicated().sum() >= 1\n",
    "assert clean_df.duplicated().sum() == 0\n",
    "assert clean_df[\"timestamp\"].isna().sum() == 0\n",
    "assert set(clean_df[\"status\"]) <= {\"SUCCESS\", \"FAILED\"}\n",
    "assert clean_df[\"username\"].str.strip().eq(clean_df[\"username\"]).all()\n",
    "assert failed_by_ip.index[0] == \"198.51.100.14\"\n",
    "assert (OUTPUT_DIR / \"clean_auth_log.csv\").exists()\n",
    "assert (OUTPUT_DIR / \"clean_auth_log.json\").exists()\n",
    "print(\"✓ Lab 5 completed\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}