CSAI2026 labs: Lab-2..Lab-6 (Practical + Solution, data), Module-2..Module-6 notebooks and datasets
This commit is contained in:
mikrotik
@@ -0,0 +1,245 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "76db28c7",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Module 3 — Lab 4: Evaluate an Intrusion Classifier\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"**Topics:** confusion matrix, accuracy, precision, recall, F1-score, error analysis\n",
|
||||
"\n",
|
||||
"> **Solution version:** This notebook contains the complete solution. Run the cells in order and compare the results with your Lab notebook.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "778202bf",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Learning objectives\n",
|
||||
"\n",
|
||||
"After completing this lab, you will be able to:\n",
|
||||
"\n",
|
||||
"- calculate a confusion matrix and classification metrics;\n",
|
||||
"- distinguish false positives from false negatives;\n",
|
||||
"- explain why attack recall matters in intrusion detection;\n",
|
||||
"- inspect misclassified network flows.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "91e09f7c",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from pathlib import Path\n",
|
||||
"import numpy as np\n",
|
||||
"import pandas as pd\n",
|
||||
"\n",
|
||||
"DATA_PATH = Path(\"data/cicids2017_binary_sample.csv\")\n",
|
||||
"RANDOM_STATE = 42\n",
|
||||
"\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"from sklearn.pipeline import Pipeline\n",
|
||||
"from sklearn.impute import SimpleImputer\n",
|
||||
"from sklearn.ensemble import RandomForestClassifier\n",
|
||||
"from sklearn.metrics import (\n",
|
||||
" ConfusionMatrixDisplay, accuracy_score, classification_report,\n",
|
||||
" confusion_matrix, precision_score, recall_score, f1_score\n",
|
||||
")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "29e0b207",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 1. Train the baseline pipeline\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "f53d7241",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"df = pd.read_csv(DATA_PATH).drop_duplicates()\n",
|
||||
"df.columns = df.columns.str.strip()\n",
|
||||
"df = df.replace([np.inf, -np.inf], np.nan)\n",
|
||||
"\n",
|
||||
"X = df.drop(columns=\"Label\").select_dtypes(include=\"number\")\n",
|
||||
"y = df[\"Label\"].astype(str).str.strip().str.upper().ne(\"BENIGN\").astype(int)\n",
|
||||
"\n",
|
||||
"X_train, X_test, y_train, y_test = train_test_split(\n",
|
||||
" X, y, test_size=0.2, random_state=RANDOM_STATE, stratify=y\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"pipeline = Pipeline([\n",
|
||||
" (\"imputer\", SimpleImputer(strategy=\"median\")),\n",
|
||||
" (\"model\", RandomForestClassifier(\n",
|
||||
" n_estimators=100, max_depth=12,\n",
|
||||
" random_state=RANDOM_STATE, n_jobs=-1\n",
|
||||
" )),\n",
|
||||
"])\n",
|
||||
"\n",
|
||||
"pipeline.fit(X_train, y_train)\n",
|
||||
"y_pred = pipeline.predict(X_test)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "0781cb26",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 2. Calculate the confusion matrix\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "21075b2f",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"cm = confusion_matrix(y_test, y_pred)\n",
|
||||
"tn, fp, fn, tp = cm.ravel()\n",
|
||||
"\n",
|
||||
"print(\"Confusion matrix:\\n\", cm)\n",
|
||||
"print(f\"TN={tn}, FP={fp}, FN={fn}, TP={tp}\")\n",
|
||||
"\n",
|
||||
"ConfusionMatrixDisplay(\n",
|
||||
" confusion_matrix=cm,\n",
|
||||
" display_labels=[\"BENIGN\", \"ATTACK\"],\n",
|
||||
").plot(cmap=\"Blues\")\n",
|
||||
"plt.title(\"Baseline intrusion classifier\")\n",
|
||||
"plt.show()\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "223681c3",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 3. Calculate classification metrics\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "a5a1597a",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"accuracy = accuracy_score(y_test, y_pred)\n",
|
||||
"attack_precision = precision_score(y_test, y_pred, zero_division=0)\n",
|
||||
"attack_recall = recall_score(y_test, y_pred, zero_division=0)\n",
|
||||
"attack_f1 = f1_score(y_test, y_pred, zero_division=0)\n",
|
||||
"\n",
|
||||
"metrics = pd.Series({\n",
|
||||
" \"accuracy\": accuracy,\n",
|
||||
" \"attack_precision\": attack_precision,\n",
|
||||
" \"attack_recall\": attack_recall,\n",
|
||||
" \"attack_f1\": attack_f1,\n",
|
||||
"})\n",
|
||||
"print(metrics.round(3))\n",
|
||||
"\n",
|
||||
"print(classification_report(\n",
|
||||
" y_test,\n",
|
||||
" y_pred,\n",
|
||||
" target_names=[\"BENIGN\", \"ATTACK\"],\n",
|
||||
" zero_division=0,\n",
|
||||
"))\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "ebbfff71",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 4. Inspect false negatives\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "4adf798b",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"results = X_test.copy()\n",
|
||||
"results[\"actual\"] = y_test\n",
|
||||
"results[\"predicted\"] = y_pred\n",
|
||||
"\n",
|
||||
"false_negatives = results[\n",
|
||||
" (results[\"actual\"] == 1) & (results[\"predicted\"] == 0)\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"print(\"False negatives:\", len(false_negatives))\n",
|
||||
"false_negatives.head()\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "4cba4276",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Self-check\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "0a64652b",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"assert cm.shape == (2, 2)\n",
|
||||
"assert tn + fp + fn + tp == len(y_test)\n",
|
||||
"assert 0 <= accuracy <= 1\n",
|
||||
"assert 0 <= attack_precision <= 1\n",
|
||||
"assert 0 <= attack_recall <= 1\n",
|
||||
"assert 0 <= attack_f1 <= 1\n",
|
||||
"assert len(false_negatives) == fn\n",
|
||||
"print(\"✓ Lab 4 completed\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "8f5b93d9",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Analyst questions\n",
|
||||
"\n",
|
||||
"1. Which confusion-matrix cell represents a missed attack?\n",
|
||||
"2. Which metric should receive more attention if missed attacks are very costly?\n",
|
||||
"3. Why can improving attack recall increase the number of false alerts?\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.6"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
Reference in New Issue
Block a user