labs/Lab-5/Lab-5-Solution/04_Security_Metrics_Error_Costs_Solution.ipynb

Module 5 — Lab 4: Security Metrics and Error Costs

Topics: false negatives, class weights, validation threshold, test discipline

Solution version: Run the cells in order, compare the outputs with your work, and read the suggested answers.

Learning objectives

After completing this lab, you will be able to:

• quantify false-negative and false-positive costs;
• compare class-weight policies on validation data;
• select a probability threshold without inspecting the test set;
• evaluate the chosen policy once on held-out test data;
• describe the operational trade-off between missed malware and analyst workload.

In [ ]:

from pathlib import Path
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

DATA_PATH = Path("data/malware_features.csv")

if not DATA_PATH.exists():
    raise FileNotFoundError(
        "Missing data/malware_features.csv. Follow data/README.md, place the three "
        "official UCI CSV files under data/raw/, and run: python data/prepare_dataset.py"
    )


from sklearn.dummy import DummyClassifier
from sklearn.impute import SimpleImputer
from sklearn.metrics import confusion_matrix, precision_score, recall_score, f1_score, accuracy_score
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.tree import DecisionTreeClassifier


def security_metrics(y_true, y_pred):
    tn, fp, fn, tp = confusion_matrix(y_true, y_pred, labels=[0, 1]).ravel()
    return {
        "accuracy": accuracy_score(y_true, y_pred),
        "malware_precision": precision_score(y_true, y_pred, zero_division=0),
        "malware_recall": recall_score(y_true, y_pred, zero_division=0),
        "malware_f1": f1_score(y_true, y_pred, zero_division=0),
        "false_positives": int(fp),
        "false_negatives": int(fn),
        "false_positive_rate": fp / (fp + tn) if fp + tn else 0.0,
    }

In [ ]:

df = pd.read_csv(DATA_PATH, low_memory=False)
feature_cols = [column for column in df.columns if column not in {"source", "label"}]
X = df[feature_cols].apply(pd.to_numeric, errors="coerce")
y = df["label"].astype(int)

X_train_full, X_test, y_train_full, y_test = train_test_split(
    X, y, test_size=0.20, stratify=y, random_state=42
)
X_train, X_valid, y_train, y_valid = train_test_split(
    X_train_full, y_train_full, test_size=0.25, stratify=y_train_full, random_state=42
)
print("Train/validation/test:", len(X_train), len(X_valid), len(X_test))

1. Compare class-weight policies on validation data

Weighting malware errors more heavily may reduce false negatives, but it can increase false positives. The correct choice depends on the cost of missed malware and available analyst capacity.

In [ ]:

weight_policies = {
    "unweighted": None,
    "malware_x2": {0: 1, 1: 2},
    "malware_x4": {0: 1, 1: 4},
}
validation_rows = []
fitted_candidates = {}

for name, weights in weight_policies.items():
    model = Pipeline([
        ("imputer", SimpleImputer(strategy="median", add_indicator=True)),
        ("tree", DecisionTreeClassifier(
            max_depth=4, min_samples_leaf=10, class_weight=weights, random_state=42
        )),
    ])
    model.fit(X_train, y_train)
    predictions = model.predict(X_valid)
    validation_rows.append({"policy": name, **security_metrics(y_valid, predictions)})
    fitted_candidates[name] = model

validation_results = pd.DataFrame(validation_rows).set_index("policy")
display(validation_results.round(4))
selected_policy = validation_results.sort_values(
    ["malware_recall", "false_positive_rate", "malware_f1"],
    ascending=[False, True, False],
).index[0]
selected_model = fitted_candidates[selected_policy]
print("Selected class-weight policy:", selected_policy)

2. Select a validation threshold

Search thresholds on validation probabilities only. Prefer thresholds that achieve at least 98% validation malware recall; among them, choose the one with the lowest false-positive rate.

In [ ]:

validation_probabilities = selected_model.predict_proba(X_valid)[:, 1]
threshold_rows = []
for threshold in np.arange(0.20, 0.81, 0.05):
    predictions = (validation_probabilities >= threshold).astype(int)
    threshold_rows.append({"threshold": round(float(threshold), 2), **security_metrics(y_valid, predictions)})

threshold_results = pd.DataFrame(threshold_rows).set_index("threshold")
eligible = threshold_results[threshold_results["malware_recall"] >= 0.98]
pool = eligible if not eligible.empty else threshold_results.loc[[threshold_results["malware_recall"].idxmax()]]
selected_threshold = float(pool.sort_values(
    ["false_positive_rate", "malware_f1"], ascending=[True, False]
).index[0])
display(threshold_results.round(4))
print("Selected threshold:", selected_threshold)

3. Refit on all training data and evaluate once on the test set

The test set is used only after all policy decisions are fixed.

In [ ]:

final_model = Pipeline([
    ("imputer", SimpleImputer(strategy="median", add_indicator=True)),
    ("tree", DecisionTreeClassifier(
        max_depth=4,
        min_samples_leaf=10,
        class_weight=weight_policies[selected_policy],
        random_state=42,
    )),
])
final_model.fit(X_train_full, y_train_full)
test_probabilities = final_model.predict_proba(X_test)[:, 1]
default_predictions = (test_probabilities >= 0.50).astype(int)
tuned_predictions = (test_probabilities >= selected_threshold).astype(int)

test_results = pd.DataFrame([
    security_metrics(y_test, default_predictions),
    security_metrics(y_test, tuned_predictions),
], index=["threshold_0.50", f"threshold_{selected_threshold:.2f}"])
display(test_results.round(4))

Self-check

In [ ]:

assert set(validation_results.index) == set(weight_policies)
assert 0.20 <= selected_threshold <= 0.80
assert len(test_probabilities) == len(y_test)
assert set(tuned_predictions).issubset({0, 1})
assert test_results.shape[0] == 2
assert final_model.named_steps["tree"].get_depth() <= 4
print("✓ Lab 4 completed")

Suggested answers

1. Why tune on validation data? Repeatedly choosing settings from test results turns the test set into training information.
2. Why prioritize malware recall? A false negative allows a malicious file to pass undetected. However, maximizing recall without an FPR constraint can overwhelm analysts.
3. Is the chosen threshold universal? No. It depends on prevalence, feature drift, collection methods, and operational costs.