269 lines
9.0 KiB
Plaintext
269 lines
9.0 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "cell-001",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Module 6 — Lab 4: One-Class SVM and Isolation Forest\n",
|
|
"\n",
|
|
"\n",
|
|
"**Topics:** normal-only learning, anomaly scores, validation thresholds, PR-AUC, runtime\n",
|
|
"\n",
|
|
"> **Solution version:** Run the cells in order, compare the outputs with your work, and read the suggested answers."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "cell-002",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Learning objectives\n",
|
|
"\n",
|
|
"After completing this lab, you will be able to:\n",
|
|
"\n",
|
|
"- fit two normal-only novelty detectors;\n",
|
|
"- standardize features for One-Class SVM;\n",
|
|
"- derive a common anomaly-score convention;\n",
|
|
"- compare detectors with the same normal-validation alert budget;\n",
|
|
"- explain performance and scalability trade-offs."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "cell-003",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from pathlib import Path\n",
|
|
"import numpy as np\n",
|
|
"import pandas as pd\n",
|
|
"import matplotlib.pyplot as plt\n",
|
|
"\n",
|
|
"TRAIN_PATH = Path(\"data/nsl_kdd_train.csv\")\n",
|
|
"TEST_PATH = Path(\"data/nsl_kdd_test.csv\")\n",
|
|
"\n",
|
|
"missing_files = [str(path) for path in (TRAIN_PATH, TEST_PATH) if not path.exists()]\n",
|
|
"if missing_files:\n",
|
|
" raise FileNotFoundError(\n",
|
|
" \"Missing prepared NSL-KDD files: \" + \", \".join(missing_files) +\n",
|
|
" \". Follow data/README.md and run: python data/prepare_dataset.py\"\n",
|
|
" )\n",
|
|
"\n",
|
|
"TARGET_COLUMNS = [\"attack_label\", \"attack_group\", \"binary_label\", \"difficulty\"]\n",
|
|
"CATEGORICAL_FEATURES = [\"protocol_type\", \"service\", \"flag\"]\n",
|
|
"\n",
|
|
"\n",
|
|
"from sklearn.ensemble import IsolationForest\n",
|
|
"from sklearn.model_selection import train_test_split\n",
|
|
"from sklearn.pipeline import Pipeline\n",
|
|
"from sklearn.preprocessing import StandardScaler\n",
|
|
"from sklearn.svm import OneClassSVM\n",
|
|
"\n",
|
|
"\n",
|
|
"from sklearn.metrics import (\n",
|
|
" accuracy_score, average_precision_score, confusion_matrix,\n",
|
|
" f1_score, precision_score, recall_score,\n",
|
|
")\n",
|
|
"\n",
|
|
"\n",
|
|
"def security_metrics(y_true, y_pred):\n",
|
|
" tn, fp, fn, tp = confusion_matrix(y_true, y_pred, labels=[0, 1]).ravel()\n",
|
|
" return {\n",
|
|
" \"accuracy\": accuracy_score(y_true, y_pred),\n",
|
|
" \"attack_precision\": precision_score(y_true, y_pred, zero_division=0),\n",
|
|
" \"attack_recall\": recall_score(y_true, y_pred, zero_division=0),\n",
|
|
" \"attack_f1\": f1_score(y_true, y_pred, zero_division=0),\n",
|
|
" \"false_positives\": int(fp),\n",
|
|
" \"false_negatives\": int(fn),\n",
|
|
" \"false_positive_rate\": fp / (fp + tn) if fp + tn else 0.0,\n",
|
|
" \"alerts_per_10000\": 10000 * (fp + tp) / len(y_true),\n",
|
|
" }"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "cell-004",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"train_df = pd.read_csv(TRAIN_PATH, low_memory=False)\n",
|
|
"test_df = pd.read_csv(TEST_PATH, low_memory=False)\n",
|
|
"anomaly_features = [\n",
|
|
" \"duration\", \"src_bytes\", \"dst_bytes\", \"wrong_fragment\", \"hot\",\n",
|
|
" \"num_failed_logins\", \"logged_in\", \"count\", \"srv_count\",\n",
|
|
" \"serror_rate\", \"rerror_rate\", \"same_srv_rate\", \"diff_srv_rate\",\n",
|
|
" \"dst_host_count\", \"dst_host_srv_count\", \"dst_host_same_srv_rate\",\n",
|
|
" \"dst_host_diff_srv_rate\", \"dst_host_serror_rate\", \"dst_host_rerror_rate\",\n",
|
|
"]\n",
|
|
"normal = train_df.loc[train_df[\"binary_label\"] == 0, anomaly_features]\n",
|
|
"normal_fit, normal_valid = train_test_split(normal, test_size=0.20, random_state=42)\n",
|
|
"normal_fit = normal_fit.sample(min(5000, len(normal_fit)), random_state=42)\n",
|
|
"X_test = test_df[anomaly_features]\n",
|
|
"y_test = test_df[\"binary_label\"].astype(int)\n",
|
|
"print(\"Normal fit/validation/test:\", len(normal_fit), len(normal_valid), len(X_test))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "cell-005",
|
|
"metadata": {},
|
|
"source": [
|
|
"## 1. Fit the two detectors"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "cell-006",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"detectors = {\n",
|
|
" \"One-Class SVM\": Pipeline([\n",
|
|
" (\"scaler\", StandardScaler()),\n",
|
|
" (\"model\", OneClassSVM(kernel=\"rbf\", gamma=\"scale\", nu=0.02)),\n",
|
|
" ]),\n",
|
|
" \"Isolation Forest\": Pipeline([\n",
|
|
" (\"scaler\", StandardScaler()),\n",
|
|
" (\"model\", IsolationForest(\n",
|
|
" n_estimators=200, max_samples=\"auto\", contamination=\"auto\",\n",
|
|
" random_state=42, n_jobs=-1,\n",
|
|
" )),\n",
|
|
" ]),\n",
|
|
"}\n",
|
|
"\n",
|
|
"for detector in detectors.values():\n",
|
|
" detector.fit(normal_fit)\n",
|
|
"print(\"Fitted detectors:\", list(detectors))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "cell-007",
|
|
"metadata": {},
|
|
"source": [
|
|
"## 2. Apply a common validation-threshold policy"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "cell-008",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"comparison_rows = []\n",
|
|
"detector_outputs = {}\n",
|
|
"\n",
|
|
"for name, detector in detectors.items():\n",
|
|
" validation_scores = -detector.decision_function(normal_valid)\n",
|
|
" test_scores = -detector.decision_function(X_test)\n",
|
|
" threshold = float(np.quantile(validation_scores, 0.99))\n",
|
|
" predictions = (test_scores > threshold).astype(int)\n",
|
|
" metrics = security_metrics(y_test, predictions)\n",
|
|
" metrics[\"pr_auc\"] = average_precision_score(y_test, test_scores)\n",
|
|
" comparison_rows.append({\"detector\": name, \"threshold\": threshold, **metrics})\n",
|
|
" detector_outputs[name] = {\n",
|
|
" \"validation_scores\": validation_scores,\n",
|
|
" \"test_scores\": test_scores,\n",
|
|
" \"predictions\": predictions,\n",
|
|
" }\n",
|
|
"\n",
|
|
"detector_comparison = pd.DataFrame(comparison_rows).set_index(\"detector\")\n",
|
|
"display(detector_comparison.round(4))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "cell-009",
|
|
"metadata": {},
|
|
"source": [
|
|
"## 3. Compare attack-group recall"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "cell-010",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"group_rows = []\n",
|
|
"for name, output in detector_outputs.items():\n",
|
|
" for group in [\"DoS\", \"Probe\", \"R2L\", \"U2R\"]:\n",
|
|
" mask = test_df[\"attack_group\"].eq(group).to_numpy()\n",
|
|
" group_rows.append({\n",
|
|
" \"detector\": name,\n",
|
|
" \"attack_group\": group,\n",
|
|
" \"support\": int(mask.sum()),\n",
|
|
" \"recall\": float(output[\"predictions\"][mask].mean()) if mask.any() else np.nan,\n",
|
|
" })\n",
|
|
"group_recall = pd.DataFrame(group_rows).pivot(\n",
|
|
" index=\"attack_group\", columns=\"detector\", values=\"recall\"\n",
|
|
")\n",
|
|
"display(group_recall.round(4))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "cell-011",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Self-check"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "cell-012",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"assert set(detectors) == {\"One-Class SVM\", \"Isolation Forest\"}\n",
|
|
"assert len(normal_fit) <= 5000\n",
|
|
"assert all(len(output[\"test_scores\"]) == len(test_df) for output in detector_outputs.values())\n",
|
|
"assert detector_comparison.shape[0] == 2\n",
|
|
"assert set(group_recall.index) == {\"DoS\", \"Probe\", \"R2L\", \"U2R\"}\n",
|
|
"assert detector_comparison[\"false_positive_rate\"].between(0, 1).all()\n",
|
|
"print(\"✓ Lab 4 completed\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "cell-013",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Suggested answers\n",
|
|
"\n",
|
|
"1. **Why cap One-Class SVM training?** The RBF implementation scales poorly as the number of training samples grows.\n",
|
|
"2. **Why negate `decision_function`?** Both models then follow one convention: larger values mean more anomalous.\n",
|
|
"3. **Why can rare-group recall differ?** Different attacks occupy different regions of feature space, and some resemble normal traffic in the selected features."
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3 (ipykernel)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.11.6"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|