396 lines
16 KiB
Plaintext
396 lines
16 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "cell-001",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Module 6 — Lab 6: Capstone — Hybrid Network Intrusion Triage\n",
|
|
"\n",
|
|
"\n",
|
|
"**Topics:** classifier, anomaly score, hybrid policy, alert budget, SOC report\n",
|
|
"\n",
|
|
"> **Solution version:** Run the cells in order, compare the outputs with your work, and read the suggested answers."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "cell-002",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Mission\n",
|
|
"\n",
|
|
"Compare three alert policies on the fixed NSL-KDD test set:\n",
|
|
"\n",
|
|
"1. a supervised Decision Tree for known attack patterns;\n",
|
|
"2. a normal-only Isolation Forest for anomaly hunting;\n",
|
|
"3. a hybrid OR policy that alerts when either detector fires.\n",
|
|
"\n",
|
|
"**Required outputs**\n",
|
|
"\n",
|
|
"- `output/hybrid_ids_metrics.csv`\n",
|
|
"- `output/hybrid_ids_group_recall.csv`\n",
|
|
"- `output/hybrid_ids_top_alerts.csv`\n",
|
|
"- `output/hybrid_ids_confusion_matrix.png`\n",
|
|
"- `output/hybrid_ids_score_distribution.png`\n",
|
|
"- `output/hybrid_ids_analyst_summary.md`"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "cell-003",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from pathlib import Path\n",
|
|
"import numpy as np\n",
|
|
"import pandas as pd\n",
|
|
"import matplotlib.pyplot as plt\n",
|
|
"\n",
|
|
"TRAIN_PATH = Path(\"data/nsl_kdd_train.csv\")\n",
|
|
"TEST_PATH = Path(\"data/nsl_kdd_test.csv\")\n",
|
|
"\n",
|
|
"missing_files = [str(path) for path in (TRAIN_PATH, TEST_PATH) if not path.exists()]\n",
|
|
"if missing_files:\n",
|
|
" raise FileNotFoundError(\n",
|
|
" \"Missing prepared NSL-KDD files: \" + \", \".join(missing_files) +\n",
|
|
" \". Follow data/README.md and run: python data/prepare_dataset.py\"\n",
|
|
" )\n",
|
|
"\n",
|
|
"TARGET_COLUMNS = [\"attack_label\", \"attack_group\", \"binary_label\", \"difficulty\"]\n",
|
|
"CATEGORICAL_FEATURES = [\"protocol_type\", \"service\", \"flag\"]\n",
|
|
"\n",
|
|
"\n",
|
|
"from sklearn.compose import ColumnTransformer\n",
|
|
"from sklearn.ensemble import IsolationForest\n",
|
|
"from sklearn.impute import SimpleImputer\n",
|
|
"from sklearn.metrics import ConfusionMatrixDisplay\n",
|
|
"from sklearn.model_selection import train_test_split\n",
|
|
"from sklearn.pipeline import Pipeline\n",
|
|
"from sklearn.preprocessing import OneHotEncoder, StandardScaler\n",
|
|
"from sklearn.tree import DecisionTreeClassifier\n",
|
|
"\n",
|
|
"\n",
|
|
"from sklearn.metrics import (\n",
|
|
" accuracy_score, average_precision_score, confusion_matrix,\n",
|
|
" f1_score, precision_score, recall_score,\n",
|
|
")\n",
|
|
"\n",
|
|
"\n",
|
|
"def security_metrics(y_true, y_pred):\n",
|
|
" tn, fp, fn, tp = confusion_matrix(y_true, y_pred, labels=[0, 1]).ravel()\n",
|
|
" return {\n",
|
|
" \"accuracy\": accuracy_score(y_true, y_pred),\n",
|
|
" \"attack_precision\": precision_score(y_true, y_pred, zero_division=0),\n",
|
|
" \"attack_recall\": recall_score(y_true, y_pred, zero_division=0),\n",
|
|
" \"attack_f1\": f1_score(y_true, y_pred, zero_division=0),\n",
|
|
" \"false_positives\": int(fp),\n",
|
|
" \"false_negatives\": int(fn),\n",
|
|
" \"false_positive_rate\": fp / (fp + tn) if fp + tn else 0.0,\n",
|
|
" \"alerts_per_10000\": 10000 * (fp + tp) / len(y_true),\n",
|
|
" }\n",
|
|
"\n",
|
|
"\n",
|
|
"OUTPUT_DIR = Path(\"output\")\n",
|
|
"OUTPUT_DIR.mkdir(exist_ok=True)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "cell-004",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"train_df = pd.read_csv(TRAIN_PATH, low_memory=False)\n",
|
|
"test_df = pd.read_csv(TEST_PATH, low_memory=False)\n",
|
|
"feature_cols = [column for column in train_df.columns if column not in TARGET_COLUMNS]\n",
|
|
"numeric_features = [column for column in feature_cols if column not in CATEGORICAL_FEATURES]\n",
|
|
"X_train, y_train = train_df[feature_cols], train_df[\"binary_label\"].astype(int)\n",
|
|
"X_test, y_test = test_df[feature_cols], test_df[\"binary_label\"].astype(int)\n",
|
|
"\n",
|
|
"supervised_preprocessor = ColumnTransformer([\n",
|
|
" (\"categorical\", OneHotEncoder(handle_unknown=\"ignore\"), CATEGORICAL_FEATURES),\n",
|
|
" (\"numeric\", SimpleImputer(strategy=\"median\"), numeric_features),\n",
|
|
"])\n",
|
|
"tree_model = Pipeline([\n",
|
|
" (\"preprocessor\", supervised_preprocessor),\n",
|
|
" (\"tree\", DecisionTreeClassifier(\n",
|
|
" max_depth=8, min_samples_leaf=5, class_weight=\"balanced\", random_state=42\n",
|
|
" )),\n",
|
|
"])\n",
|
|
"tree_model.fit(X_train, y_train)\n",
|
|
"tree_predictions = tree_model.predict(X_test)\n",
|
|
"tree_probabilities = tree_model.predict_proba(X_test)[:, 1]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "cell-005",
|
|
"metadata": {},
|
|
"source": [
|
|
"## 1. Fit a normal-only Isolation Forest and set its alert budget"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "cell-006",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"anomaly_features = [\n",
|
|
" \"duration\", \"src_bytes\", \"dst_bytes\", \"wrong_fragment\", \"hot\",\n",
|
|
" \"num_failed_logins\", \"logged_in\", \"count\", \"srv_count\",\n",
|
|
" \"serror_rate\", \"rerror_rate\", \"same_srv_rate\", \"diff_srv_rate\",\n",
|
|
" \"dst_host_count\", \"dst_host_srv_count\", \"dst_host_same_srv_rate\",\n",
|
|
" \"dst_host_diff_srv_rate\", \"dst_host_serror_rate\", \"dst_host_rerror_rate\",\n",
|
|
"]\n",
|
|
"normal = train_df.loc[train_df[\"binary_label\"] == 0, anomaly_features]\n",
|
|
"normal_fit, normal_valid = train_test_split(normal, test_size=0.20, random_state=42)\n",
|
|
"\n",
|
|
"anomaly_model = Pipeline([\n",
|
|
" (\"scaler\", StandardScaler()),\n",
|
|
" (\"model\", IsolationForest(\n",
|
|
" n_estimators=250, contamination=\"auto\", random_state=42, n_jobs=-1\n",
|
|
" )),\n",
|
|
"])\n",
|
|
"anomaly_model.fit(normal_fit)\n",
|
|
"validation_scores = -anomaly_model.decision_function(normal_valid)\n",
|
|
"anomaly_scores = -anomaly_model.decision_function(test_df[anomaly_features])\n",
|
|
"anomaly_threshold = float(np.quantile(validation_scores, 0.99))\n",
|
|
"anomaly_predictions = (anomaly_scores > anomaly_threshold).astype(int)\n",
|
|
"print(\"Anomaly threshold:\", anomaly_threshold)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "cell-007",
|
|
"metadata": {},
|
|
"source": [
|
|
"## 2. Compare classifier, anomaly, and hybrid policies"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "cell-008",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"policy_predictions = {\n",
|
|
" \"Classifier\": tree_predictions,\n",
|
|
" \"Anomaly\": anomaly_predictions,\n",
|
|
" \"Hybrid_OR\": np.logical_or(tree_predictions == 1, anomaly_predictions == 1).astype(int),\n",
|
|
"}\n",
|
|
"policy_scores = {\n",
|
|
" \"Classifier\": tree_probabilities,\n",
|
|
" \"Anomaly\": anomaly_scores,\n",
|
|
" \"Hybrid_OR\": np.maximum(\n",
|
|
" tree_probabilities,\n",
|
|
" np.clip((anomaly_scores - anomaly_threshold) + 0.5, 0, 1),\n",
|
|
" ),\n",
|
|
"}\n",
|
|
"\n",
|
|
"metric_rows = []\n",
|
|
"for name, predictions in policy_predictions.items():\n",
|
|
" metrics = security_metrics(y_test, predictions)\n",
|
|
" metrics[\"pr_auc\"] = average_precision_score(y_test, policy_scores[name])\n",
|
|
" metric_rows.append({\"policy\": name, **metrics})\n",
|
|
"metrics_table = pd.DataFrame(metric_rows).set_index(\"policy\")\n",
|
|
"display(metrics_table.round(4))\n",
|
|
"\n",
|
|
"acceptable = metrics_table[metrics_table[\"false_positive_rate\"] <= 0.10]\n",
|
|
"selection_pool = acceptable if not acceptable.empty else metrics_table\n",
|
|
"selected_policy = selection_pool.sort_values(\n",
|
|
" [\"attack_recall\", \"attack_f1\", \"false_positive_rate\"],\n",
|
|
" ascending=[False, False, True],\n",
|
|
").index[0]\n",
|
|
"selected_predictions = policy_predictions[selected_policy]\n",
|
|
"print(\"Selected policy:\", selected_policy)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "cell-009",
|
|
"metadata": {},
|
|
"source": [
|
|
"## 3. Audit attack groups and create the alert evidence table"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "cell-010",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"group_rows = []\n",
|
|
"for policy, predictions in policy_predictions.items():\n",
|
|
" for group in [\"DoS\", \"Probe\", \"R2L\", \"U2R\"]:\n",
|
|
" mask = test_df[\"attack_group\"].eq(group).to_numpy()\n",
|
|
" group_rows.append({\n",
|
|
" \"policy\": policy,\n",
|
|
" \"attack_group\": group,\n",
|
|
" \"support\": int(mask.sum()),\n",
|
|
" \"recall\": float(predictions[mask].mean()) if mask.any() else np.nan,\n",
|
|
" })\n",
|
|
"group_recall = pd.DataFrame(group_rows)\n",
|
|
"display(group_recall.pivot(index=\"attack_group\", columns=\"policy\", values=\"recall\").round(4))\n",
|
|
"\n",
|
|
"alert_table = test_df[[\n",
|
|
" \"protocol_type\", \"service\", \"flag\", \"src_bytes\", \"dst_bytes\",\n",
|
|
" \"count\", \"srv_count\", \"attack_label\", \"attack_group\", \"binary_label\",\n",
|
|
"]].copy()\n",
|
|
"alert_table[\"tree_probability\"] = tree_probabilities\n",
|
|
"alert_table[\"anomaly_score\"] = anomaly_scores\n",
|
|
"alert_table[\"tree_alert\"] = tree_predictions\n",
|
|
"alert_table[\"anomaly_alert\"] = anomaly_predictions\n",
|
|
"alert_table[\"selected_alert\"] = selected_predictions\n",
|
|
"alert_table[\"alert_reason\"] = np.select(\n",
|
|
" [\n",
|
|
" (tree_predictions == 1) & (anomaly_predictions == 1),\n",
|
|
" tree_predictions == 1,\n",
|
|
" anomaly_predictions == 1,\n",
|
|
" ],\n",
|
|
" [\"classifier_and_anomaly\", \"classifier_only\", \"anomaly_only\"],\n",
|
|
" default=\"no_alert\",\n",
|
|
")\n",
|
|
"top_alerts = alert_table.loc[alert_table[\"selected_alert\"] == 1].sort_values(\n",
|
|
" [\"anomaly_score\", \"tree_probability\"], ascending=False\n",
|
|
").head(200)\n",
|
|
"display(top_alerts.head(10))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "cell-011",
|
|
"metadata": {},
|
|
"source": [
|
|
"## 4. Export evidence and write the SOC recommendation"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "cell-012",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"fig, ax = plt.subplots(figsize=(6, 5))\n",
|
|
"ConfusionMatrixDisplay.from_predictions(\n",
|
|
" y_test, selected_predictions, labels=[0, 1],\n",
|
|
" display_labels=[\"normal\", \"attack\"], cmap=\"Blues\", colorbar=False, ax=ax,\n",
|
|
")\n",
|
|
"ax.set_title(f\"{selected_policy} policy — fixed test set\")\n",
|
|
"fig.tight_layout()\n",
|
|
"fig.savefig(OUTPUT_DIR / \"hybrid_ids_confusion_matrix.png\", dpi=160)\n",
|
|
"plt.show()\n",
|
|
"\n",
|
|
"clipped_scores = np.clip(anomaly_scores, None, np.quantile(anomaly_scores, 0.995))\n",
|
|
"fig, ax = plt.subplots(figsize=(9, 5))\n",
|
|
"ax.hist(clipped_scores[y_test.to_numpy() == 0], bins=60, alpha=0.6, density=True, label=\"Normal\")\n",
|
|
"ax.hist(clipped_scores[y_test.to_numpy() == 1], bins=60, alpha=0.5, density=True, label=\"Attack\")\n",
|
|
"ax.axvline(anomaly_threshold, color=\"black\", linestyle=\"--\", label=\"Threshold\")\n",
|
|
"ax.set_title(\"Isolation Forest anomaly-score distribution\")\n",
|
|
"ax.set_xlabel(\"Anomaly score\")\n",
|
|
"ax.set_ylabel(\"Density\")\n",
|
|
"ax.legend()\n",
|
|
"fig.tight_layout()\n",
|
|
"fig.savefig(OUTPUT_DIR / \"hybrid_ids_score_distribution.png\", dpi=160)\n",
|
|
"plt.show()\n",
|
|
"\n",
|
|
"selected_metrics = metrics_table.loc[selected_policy]\n",
|
|
"analyst_summary = \"\\n\".join([\n",
|
|
" \"# Hybrid IDS Capstone — SOC Analyst Summary\", \"\",\n",
|
|
" \"## Recommendation\", \"\",\n",
|
|
" f\"Selected policy: **{selected_policy}**.\", \"\",\n",
|
|
" f\"- Attack recall: {selected_metrics['attack_recall']:.3f}\",\n",
|
|
" f\"- Attack precision: {selected_metrics['attack_precision']:.3f}\",\n",
|
|
" f\"- Attack F1: {selected_metrics['attack_f1']:.3f}\",\n",
|
|
" f\"- False-positive rate: {selected_metrics['false_positive_rate']:.3f}\",\n",
|
|
" f\"- Alerts per 10,000 flows: {selected_metrics['alerts_per_10000']:.1f}\",\n",
|
|
" f\"- False negatives: {int(selected_metrics['false_negatives'])}\", \"\",\n",
|
|
" \"## Operating model\", \"\",\n",
|
|
" \"The classifier handles patterns represented by labeled training attacks. The anomaly detector adds coverage for flows that deviate from the learned normal baseline. The hybrid policy can improve recall, but every added anomaly alert consumes analyst capacity.\", \"\",\n",
|
|
" \"## Limitations\", \"\",\n",
|
|
" \"NSL-KDD is a legacy benchmark with synthetic historical traffic and unrealistic prevalence. The results do not demonstrate readiness for encrypted enterprise traffic, current botnets, concept drift, or adversarial evasion. Production use would require current telemetry, time-aware validation, calibration, monitoring, and layered signature or sandbox evidence.\", \"\",\n",
|
|
"])\n",
|
|
"\n",
|
|
"metrics_table.to_csv(OUTPUT_DIR / \"hybrid_ids_metrics.csv\")\n",
|
|
"group_recall.to_csv(OUTPUT_DIR / \"hybrid_ids_group_recall.csv\", index=False)\n",
|
|
"top_alerts.to_csv(OUTPUT_DIR / \"hybrid_ids_top_alerts.csv\", index=False)\n",
|
|
"(OUTPUT_DIR / \"hybrid_ids_analyst_summary.md\").write_text(analyst_summary, encoding=\"utf-8\")\n",
|
|
"print(analyst_summary)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "cell-013",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Self-check"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "cell-014",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"expected_files = {\n",
|
|
" \"hybrid_ids_metrics.csv\", \"hybrid_ids_group_recall.csv\",\n",
|
|
" \"hybrid_ids_top_alerts.csv\", \"hybrid_ids_confusion_matrix.png\",\n",
|
|
" \"hybrid_ids_score_distribution.png\", \"hybrid_ids_analyst_summary.md\",\n",
|
|
"}\n",
|
|
"assert set(policy_predictions) == {\"Classifier\", \"Anomaly\", \"Hybrid_OR\"}\n",
|
|
"assert selected_policy in policy_predictions\n",
|
|
"assert len(alert_table) == len(test_df)\n",
|
|
"assert set(group_recall[\"attack_group\"]) == {\"DoS\", \"Probe\", \"R2L\", \"U2R\"}\n",
|
|
"assert expected_files.issubset({path.name for path in OUTPUT_DIR.iterdir()})\n",
|
|
"assert \"NSL-KDD\" in analyst_summary\n",
|
|
"print(\"✓ Lab 6 completed\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "cell-015",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Suggested review checklist\n",
|
|
"\n",
|
|
"- The anomaly model learned from normal training records only.\n",
|
|
"- The anomaly threshold came from normal validation scores.\n",
|
|
"- The fixed test set influenced evaluation, not training or threshold selection.\n",
|
|
"- The recommendation balances attack recall with false-positive rate and alert volume.\n",
|
|
"- Per-group recall accompanies the aggregate binary score.\n",
|
|
"- The report separates known-pattern classification from anomaly hunting.\n",
|
|
"- The report states that NSL-KDD performance is not a production guarantee."
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3 (ipykernel)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.11.6"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|