labs/Lab-6/Lab-6-Solution/06_Capstone_Hybrid_IDS_Solution.ipynb

Module 6 — Lab 6: Capstone — Hybrid Network Intrusion Triage

Topics: classifier, anomaly score, hybrid policy, alert budget, SOC report

Solution version: Run the cells in order, compare the outputs with your work, and read the suggested answers.

Mission

Compare three alert policies on the fixed NSL-KDD test set:

1. a supervised Decision Tree for known attack patterns;
2. a normal-only Isolation Forest for anomaly hunting;
3. a hybrid OR policy that alerts when either detector fires.

Required outputs

• output/hybrid_ids_metrics.csv
• output/hybrid_ids_group_recall.csv
• output/hybrid_ids_top_alerts.csv
• output/hybrid_ids_confusion_matrix.png
• output/hybrid_ids_score_distribution.png
• output/hybrid_ids_analyst_summary.md

In [ ]:

from pathlib import Path
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

TRAIN_PATH = Path("data/nsl_kdd_train.csv")
TEST_PATH = Path("data/nsl_kdd_test.csv")

missing_files = [str(path) for path in (TRAIN_PATH, TEST_PATH) if not path.exists()]
if missing_files:
    raise FileNotFoundError(
        "Missing prepared NSL-KDD files: " + ", ".join(missing_files) +
        ". Follow data/README.md and run: python data/prepare_dataset.py"
    )

TARGET_COLUMNS = ["attack_label", "attack_group", "binary_label", "difficulty"]
CATEGORICAL_FEATURES = ["protocol_type", "service", "flag"]


from sklearn.compose import ColumnTransformer
from sklearn.ensemble import IsolationForest
from sklearn.impute import SimpleImputer
from sklearn.metrics import ConfusionMatrixDisplay
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import OneHotEncoder, StandardScaler
from sklearn.tree import DecisionTreeClassifier


from sklearn.metrics import (
    accuracy_score, average_precision_score, confusion_matrix,
    f1_score, precision_score, recall_score,
)


def security_metrics(y_true, y_pred):
    tn, fp, fn, tp = confusion_matrix(y_true, y_pred, labels=[0, 1]).ravel()
    return {
        "accuracy": accuracy_score(y_true, y_pred),
        "attack_precision": precision_score(y_true, y_pred, zero_division=0),
        "attack_recall": recall_score(y_true, y_pred, zero_division=0),
        "attack_f1": f1_score(y_true, y_pred, zero_division=0),
        "false_positives": int(fp),
        "false_negatives": int(fn),
        "false_positive_rate": fp / (fp + tn) if fp + tn else 0.0,
        "alerts_per_10000": 10000 * (fp + tp) / len(y_true),
    }


OUTPUT_DIR = Path("output")
OUTPUT_DIR.mkdir(exist_ok=True)

In [ ]:

train_df = pd.read_csv(TRAIN_PATH, low_memory=False)
test_df = pd.read_csv(TEST_PATH, low_memory=False)
feature_cols = [column for column in train_df.columns if column not in TARGET_COLUMNS]
numeric_features = [column for column in feature_cols if column not in CATEGORICAL_FEATURES]
X_train, y_train = train_df[feature_cols], train_df["binary_label"].astype(int)
X_test, y_test = test_df[feature_cols], test_df["binary_label"].astype(int)

supervised_preprocessor = ColumnTransformer([
    ("categorical", OneHotEncoder(handle_unknown="ignore"), CATEGORICAL_FEATURES),
    ("numeric", SimpleImputer(strategy="median"), numeric_features),
])
tree_model = Pipeline([
    ("preprocessor", supervised_preprocessor),
    ("tree", DecisionTreeClassifier(
        max_depth=8, min_samples_leaf=5, class_weight="balanced", random_state=42
    )),
])
tree_model.fit(X_train, y_train)
tree_predictions = tree_model.predict(X_test)
tree_probabilities = tree_model.predict_proba(X_test)[:, 1]

1. Fit a normal-only Isolation Forest and set its alert budget

In [ ]:

anomaly_features = [
    "duration", "src_bytes", "dst_bytes", "wrong_fragment", "hot",
    "num_failed_logins", "logged_in", "count", "srv_count",
    "serror_rate", "rerror_rate", "same_srv_rate", "diff_srv_rate",
    "dst_host_count", "dst_host_srv_count", "dst_host_same_srv_rate",
    "dst_host_diff_srv_rate", "dst_host_serror_rate", "dst_host_rerror_rate",
]
normal = train_df.loc[train_df["binary_label"] == 0, anomaly_features]
normal_fit, normal_valid = train_test_split(normal, test_size=0.20, random_state=42)

anomaly_model = Pipeline([
    ("scaler", StandardScaler()),
    ("model", IsolationForest(
        n_estimators=250, contamination="auto", random_state=42, n_jobs=-1
    )),
])
anomaly_model.fit(normal_fit)
validation_scores = -anomaly_model.decision_function(normal_valid)
anomaly_scores = -anomaly_model.decision_function(test_df[anomaly_features])
anomaly_threshold = float(np.quantile(validation_scores, 0.99))
anomaly_predictions = (anomaly_scores > anomaly_threshold).astype(int)
print("Anomaly threshold:", anomaly_threshold)

2. Compare classifier, anomaly, and hybrid policies

In [ ]:

policy_predictions = {
    "Classifier": tree_predictions,
    "Anomaly": anomaly_predictions,
    "Hybrid_OR": np.logical_or(tree_predictions == 1, anomaly_predictions == 1).astype(int),
}
policy_scores = {
    "Classifier": tree_probabilities,
    "Anomaly": anomaly_scores,
    "Hybrid_OR": np.maximum(
        tree_probabilities,
        np.clip((anomaly_scores - anomaly_threshold) + 0.5, 0, 1),
    ),
}

metric_rows = []
for name, predictions in policy_predictions.items():
    metrics = security_metrics(y_test, predictions)
    metrics["pr_auc"] = average_precision_score(y_test, policy_scores[name])
    metric_rows.append({"policy": name, **metrics})
metrics_table = pd.DataFrame(metric_rows).set_index("policy")
display(metrics_table.round(4))

acceptable = metrics_table[metrics_table["false_positive_rate"] <= 0.10]
selection_pool = acceptable if not acceptable.empty else metrics_table
selected_policy = selection_pool.sort_values(
    ["attack_recall", "attack_f1", "false_positive_rate"],
    ascending=[False, False, True],
).index[0]
selected_predictions = policy_predictions[selected_policy]
print("Selected policy:", selected_policy)

3. Audit attack groups and create the alert evidence table

In [ ]:

group_rows = []
for policy, predictions in policy_predictions.items():
    for group in ["DoS", "Probe", "R2L", "U2R"]:
        mask = test_df["attack_group"].eq(group).to_numpy()
        group_rows.append({
            "policy": policy,
            "attack_group": group,
            "support": int(mask.sum()),
            "recall": float(predictions[mask].mean()) if mask.any() else np.nan,
        })
group_recall = pd.DataFrame(group_rows)
display(group_recall.pivot(index="attack_group", columns="policy", values="recall").round(4))

alert_table = test_df[[
    "protocol_type", "service", "flag", "src_bytes", "dst_bytes",
    "count", "srv_count", "attack_label", "attack_group", "binary_label",
]].copy()
alert_table["tree_probability"] = tree_probabilities
alert_table["anomaly_score"] = anomaly_scores
alert_table["tree_alert"] = tree_predictions
alert_table["anomaly_alert"] = anomaly_predictions
alert_table["selected_alert"] = selected_predictions
alert_table["alert_reason"] = np.select(
    [
        (tree_predictions == 1) & (anomaly_predictions == 1),
        tree_predictions == 1,
        anomaly_predictions == 1,
    ],
    ["classifier_and_anomaly", "classifier_only", "anomaly_only"],
    default="no_alert",
)
top_alerts = alert_table.loc[alert_table["selected_alert"] == 1].sort_values(
    ["anomaly_score", "tree_probability"], ascending=False
).head(200)
display(top_alerts.head(10))

4. Export evidence and write the SOC recommendation

In [ ]:

fig, ax = plt.subplots(figsize=(6, 5))
ConfusionMatrixDisplay.from_predictions(
    y_test, selected_predictions, labels=[0, 1],
    display_labels=["normal", "attack"], cmap="Blues", colorbar=False, ax=ax,
)
ax.set_title(f"{selected_policy} policy — fixed test set")
fig.tight_layout()
fig.savefig(OUTPUT_DIR / "hybrid_ids_confusion_matrix.png", dpi=160)
plt.show()

clipped_scores = np.clip(anomaly_scores, None, np.quantile(anomaly_scores, 0.995))
fig, ax = plt.subplots(figsize=(9, 5))
ax.hist(clipped_scores[y_test.to_numpy() == 0], bins=60, alpha=0.6, density=True, label="Normal")
ax.hist(clipped_scores[y_test.to_numpy() == 1], bins=60, alpha=0.5, density=True, label="Attack")
ax.axvline(anomaly_threshold, color="black", linestyle="--", label="Threshold")
ax.set_title("Isolation Forest anomaly-score distribution")
ax.set_xlabel("Anomaly score")
ax.set_ylabel("Density")
ax.legend()
fig.tight_layout()
fig.savefig(OUTPUT_DIR / "hybrid_ids_score_distribution.png", dpi=160)
plt.show()

selected_metrics = metrics_table.loc[selected_policy]
analyst_summary = "\n".join([
    "# Hybrid IDS Capstone — SOC Analyst Summary", "",
    "## Recommendation", "",
    f"Selected policy: **{selected_policy}**.", "",
    f"- Attack recall: {selected_metrics['attack_recall']:.3f}",
    f"- Attack precision: {selected_metrics['attack_precision']:.3f}",
    f"- Attack F1: {selected_metrics['attack_f1']:.3f}",
    f"- False-positive rate: {selected_metrics['false_positive_rate']:.3f}",
    f"- Alerts per 10,000 flows: {selected_metrics['alerts_per_10000']:.1f}",
    f"- False negatives: {int(selected_metrics['false_negatives'])}", "",
    "## Operating model", "",
    "The classifier handles patterns represented by labeled training attacks. The anomaly detector adds coverage for flows that deviate from the learned normal baseline. The hybrid policy can improve recall, but every added anomaly alert consumes analyst capacity.", "",
    "## Limitations", "",
    "NSL-KDD is a legacy benchmark with synthetic historical traffic and unrealistic prevalence. The results do not demonstrate readiness for encrypted enterprise traffic, current botnets, concept drift, or adversarial evasion. Production use would require current telemetry, time-aware validation, calibration, monitoring, and layered signature or sandbox evidence.", "",
])

metrics_table.to_csv(OUTPUT_DIR / "hybrid_ids_metrics.csv")
group_recall.to_csv(OUTPUT_DIR / "hybrid_ids_group_recall.csv", index=False)
top_alerts.to_csv(OUTPUT_DIR / "hybrid_ids_top_alerts.csv", index=False)
(OUTPUT_DIR / "hybrid_ids_analyst_summary.md").write_text(analyst_summary, encoding="utf-8")
print(analyst_summary)

Self-check

In [ ]:

expected_files = {
    "hybrid_ids_metrics.csv", "hybrid_ids_group_recall.csv",
    "hybrid_ids_top_alerts.csv", "hybrid_ids_confusion_matrix.png",
    "hybrid_ids_score_distribution.png", "hybrid_ids_analyst_summary.md",
}
assert set(policy_predictions) == {"Classifier", "Anomaly", "Hybrid_OR"}
assert selected_policy in policy_predictions
assert len(alert_table) == len(test_df)
assert set(group_recall["attack_group"]) == {"DoS", "Probe", "R2L", "U2R"}
assert expected_files.issubset({path.name for path in OUTPUT_DIR.iterdir()})
assert "NSL-KDD" in analyst_summary
print("✓ Lab 6 completed")

Suggested review checklist

• The anomaly model learned from normal training records only.
• The anomaly threshold came from normal validation scores.
• The fixed test set influenced evaluation, not training or threshold selection.
• The recommendation balances attack recall with false-positive rate and alert volume.
• Per-group recall accompanies the aggregate binary score.
• The report separates known-pattern classification from anomaly hunting.
• The report states that NSL-KDD performance is not a production guarantee.