CSAI2026 labs: Lab-2..Lab-6 (Practical + Solution, data), Module-2..Module-6 notebooks and datasets
This commit is contained in:
mikrotik
@@ -0,0 +1,373 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "cell-001",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Module 5 — Lab 6: Capstone — An Explainable Malware Detection Workflow\n",
|
||||
"\n",
|
||||
" \n",
|
||||
"**Topics:** end-to-end workflow, model comparison, error evidence, analyst recommendation\n",
|
||||
"\n",
|
||||
"> **Solution version:** Run the cells in order, compare the outputs with your work, and read the suggested answers."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "cell-002",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Mission\n",
|
||||
"\n",
|
||||
"Build and report a malware detector using the prepared UCI feature table. Compare an explainable bounded tree with a deeper tree and a Random Forest, then recommend a model based on detection quality, false positives, interpretability, and dataset limitations.\n",
|
||||
"\n",
|
||||
"**Required outputs**\n",
|
||||
"\n",
|
||||
"- `output/capstone_metrics.csv`\n",
|
||||
"- `output/capstone_errors.csv`\n",
|
||||
"- `output/capstone_tree_rules.txt`\n",
|
||||
"- `output/capstone_confusion_matrix.png`\n",
|
||||
"- `output/capstone_analyst_summary.md`"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "cell-003",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from pathlib import Path\n",
|
||||
"import numpy as np\n",
|
||||
"import pandas as pd\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"\n",
|
||||
"DATA_PATH = Path(\"data/malware_features.csv\")\n",
|
||||
"\n",
|
||||
"if not DATA_PATH.exists():\n",
|
||||
" raise FileNotFoundError(\n",
|
||||
" \"Missing data/malware_features.csv. Follow data/README.md, place the three \"\n",
|
||||
" \"official UCI CSV files under data/raw/, and run: python data/prepare_dataset.py\"\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"from sklearn.dummy import DummyClassifier\n",
|
||||
"from sklearn.impute import SimpleImputer\n",
|
||||
"from sklearn.metrics import confusion_matrix, precision_score, recall_score, f1_score, accuracy_score\n",
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"from sklearn.pipeline import Pipeline\n",
|
||||
"from sklearn.tree import DecisionTreeClassifier\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"from sklearn.ensemble import RandomForestClassifier\n",
|
||||
"from sklearn.metrics import ConfusionMatrixDisplay\n",
|
||||
"from sklearn.tree import export_text\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def security_metrics(y_true, y_pred):\n",
|
||||
" tn, fp, fn, tp = confusion_matrix(y_true, y_pred, labels=[0, 1]).ravel()\n",
|
||||
" return {\n",
|
||||
" \"accuracy\": accuracy_score(y_true, y_pred),\n",
|
||||
" \"malware_precision\": precision_score(y_true, y_pred, zero_division=0),\n",
|
||||
" \"malware_recall\": recall_score(y_true, y_pred, zero_division=0),\n",
|
||||
" \"malware_f1\": f1_score(y_true, y_pred, zero_division=0),\n",
|
||||
" \"false_positives\": int(fp),\n",
|
||||
" \"false_negatives\": int(fn),\n",
|
||||
" \"false_positive_rate\": fp / (fp + tn) if fp + tn else 0.0,\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"OUTPUT_DIR = Path(\"output\")\n",
|
||||
"OUTPUT_DIR.mkdir(exist_ok=True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "cell-004",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"df = pd.read_csv(DATA_PATH, low_memory=False)\n",
|
||||
"feature_cols = [column for column in df.columns if column not in {\"source\", \"label\"}]\n",
|
||||
"X = df[feature_cols].apply(pd.to_numeric, errors=\"coerce\")\n",
|
||||
"y = df[\"label\"].astype(int)\n",
|
||||
"X_train, X_test, y_train, y_test, source_train, source_test = train_test_split(\n",
|
||||
" X, y, df[\"source\"], test_size=0.20, stratify=y, random_state=42\n",
|
||||
")\n",
|
||||
"print(\"Train/test rows:\", len(X_train), len(X_test))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "cell-005",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 1. Build and compare candidate models"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "cell-006",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"models = {\n",
|
||||
" \"Dummy\": DummyClassifier(strategy=\"most_frequent\"),\n",
|
||||
" \"ExplainableTree\": Pipeline([\n",
|
||||
" (\"imputer\", SimpleImputer(strategy=\"median\", add_indicator=True)),\n",
|
||||
" (\"tree\", DecisionTreeClassifier(max_depth=4, min_samples_leaf=10, random_state=42)),\n",
|
||||
" ]),\n",
|
||||
" \"DeepTree\": Pipeline([\n",
|
||||
" (\"imputer\", SimpleImputer(strategy=\"median\", add_indicator=True)),\n",
|
||||
" (\"tree\", DecisionTreeClassifier(random_state=42)),\n",
|
||||
" ]),\n",
|
||||
" \"RandomForest\": Pipeline([\n",
|
||||
" (\"imputer\", SimpleImputer(strategy=\"median\", add_indicator=True)),\n",
|
||||
" (\"forest\", RandomForestClassifier(\n",
|
||||
" n_estimators=200,\n",
|
||||
" min_samples_leaf=2,\n",
|
||||
" random_state=42,\n",
|
||||
" n_jobs=-1,\n",
|
||||
" )),\n",
|
||||
" ]),\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"metric_rows = []\n",
|
||||
"predictions = {}\n",
|
||||
"probabilities = {}\n",
|
||||
"for name, model in models.items():\n",
|
||||
" model.fit(X_train, y_train)\n",
|
||||
" pred = model.predict(X_test)\n",
|
||||
" predictions[name] = pred\n",
|
||||
" if hasattr(model, \"predict_proba\"):\n",
|
||||
" probabilities[name] = model.predict_proba(X_test)[:, 1]\n",
|
||||
" metric_rows.append({\"model\": name, **security_metrics(y_test, pred)})\n",
|
||||
"\n",
|
||||
"metrics_table = pd.DataFrame(metric_rows).set_index(\"model\").sort_values(\n",
|
||||
" [\"malware_recall\", \"malware_f1\"], ascending=False\n",
|
||||
")\n",
|
||||
"display(metrics_table.round(4))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "cell-007",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 2. Select a model with an explicit decision rule\n",
|
||||
"\n",
|
||||
"Prefer the explainable tree when its malware recall is within 0.03 of the best non-dummy recall and its false-positive rate is no more than 0.05 above the best non-dummy FPR. Otherwise choose the non-dummy model with the highest malware F1."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "cell-008",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"eligible_metrics = metrics_table.drop(index=\"Dummy\")\n",
|
||||
"best_recall = eligible_metrics[\"malware_recall\"].max()\n",
|
||||
"best_fpr = eligible_metrics[\"false_positive_rate\"].min()\n",
|
||||
"explainable_ok = (\n",
|
||||
" eligible_metrics.loc[\"ExplainableTree\", \"malware_recall\"] >= best_recall - 0.03\n",
|
||||
" and eligible_metrics.loc[\"ExplainableTree\", \"false_positive_rate\"] <= best_fpr + 0.05\n",
|
||||
")\n",
|
||||
"selected_name = (\n",
|
||||
" \"ExplainableTree\"\n",
|
||||
" if explainable_ok\n",
|
||||
" else eligible_metrics[\"malware_f1\"].idxmax()\n",
|
||||
")\n",
|
||||
"selected_model = models[selected_name]\n",
|
||||
"selected_predictions = predictions[selected_name]\n",
|
||||
"print(\"Selected model:\", selected_name)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "cell-009",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 3. Create error evidence and a confusion matrix"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "cell-010",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"selected_probability = probabilities.get(selected_name, selected_predictions.astype(float))\n",
|
||||
"error_table = pd.DataFrame({\n",
|
||||
" \"source\": source_test.to_numpy(),\n",
|
||||
" \"true_label\": y_test.to_numpy(),\n",
|
||||
" \"predicted_label\": selected_predictions,\n",
|
||||
" \"malware_probability\": selected_probability,\n",
|
||||
"}, index=X_test.index)\n",
|
||||
"error_table[\"error_type\"] = np.select(\n",
|
||||
" [\n",
|
||||
" (error_table[\"true_label\"] == 1) & (error_table[\"predicted_label\"] == 0),\n",
|
||||
" (error_table[\"true_label\"] == 0) & (error_table[\"predicted_label\"] == 1),\n",
|
||||
" ],\n",
|
||||
" [\"false_negative\", \"false_positive\"],\n",
|
||||
" default=\"correct\",\n",
|
||||
")\n",
|
||||
"errors_only = error_table[error_table[\"error_type\"] != \"correct\"].copy()\n",
|
||||
"display(errors_only[\"error_type\"].value_counts())\n",
|
||||
"\n",
|
||||
"fig, ax = plt.subplots(figsize=(6, 5))\n",
|
||||
"ConfusionMatrixDisplay.from_predictions(\n",
|
||||
" y_test,\n",
|
||||
" selected_predictions,\n",
|
||||
" labels=[0, 1],\n",
|
||||
" display_labels=[\"benign\", \"malware\"],\n",
|
||||
" cmap=\"Blues\",\n",
|
||||
" ax=ax,\n",
|
||||
" colorbar=False,\n",
|
||||
")\n",
|
||||
"ax.set_title(f\"{selected_name} — held-out test set\")\n",
|
||||
"fig.tight_layout()\n",
|
||||
"fig.savefig(OUTPUT_DIR / \"capstone_confusion_matrix.png\", dpi=160)\n",
|
||||
"plt.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "cell-011",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 4. Export rules and write the analyst recommendation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "cell-012",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"explainable_model = models[\"ExplainableTree\"]\n",
|
||||
"explainable_imputer = explainable_model.named_steps[\"imputer\"]\n",
|
||||
"explainable_estimator = explainable_model.named_steps[\"tree\"]\n",
|
||||
"explainable_names = explainable_imputer.get_feature_names_out(feature_cols)\n",
|
||||
"tree_rules = export_text(\n",
|
||||
" explainable_estimator,\n",
|
||||
" feature_names=list(explainable_names),\n",
|
||||
" max_depth=4,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"selected_metrics = metrics_table.loc[selected_name]\n",
|
||||
"analyst_summary = \"\\n\".join([\n",
|
||||
" \"# Malware Detection Capstone — Analyst Summary\",\n",
|
||||
" \"\",\n",
|
||||
" \"## Recommendation\",\n",
|
||||
" \"\",\n",
|
||||
" f\"Selected model: **{selected_name}**.\",\n",
|
||||
" \"\",\n",
|
||||
" f\"- Malware recall: {selected_metrics['malware_recall']:.3f}\",\n",
|
||||
" f\"- Malware precision: {selected_metrics['malware_precision']:.3f}\",\n",
|
||||
" f\"- Malware F1: {selected_metrics['malware_f1']:.3f}\",\n",
|
||||
" f\"- False positives: {int(selected_metrics['false_positives'])}\",\n",
|
||||
" f\"- False negatives: {int(selected_metrics['false_negatives'])}\",\n",
|
||||
" \"\",\n",
|
||||
" \"The selection followed the documented recall/FPR/interpretability rule. \"\n",
|
||||
" \"The bounded tree is preferred when its detection quality is close to the \"\n",
|
||||
" \"strongest candidate; otherwise the best non-dummy F1 model is selected.\",\n",
|
||||
" \"\",\n",
|
||||
" \"## Operational interpretation\",\n",
|
||||
" \"\",\n",
|
||||
" \"False negatives represent malware that would pass the detector. False positives \"\n",
|
||||
" \"represent benign files that would create review workload. A deployment threshold \"\n",
|
||||
" \"must therefore be matched to analyst capacity and the cost of a missed infection.\",\n",
|
||||
" \"\",\n",
|
||||
" \"## Limitations\",\n",
|
||||
" \"\",\n",
|
||||
" \"This historical dataset is imbalanced, and benign/malware records come from \"\n",
|
||||
" \"different source collections. Random-split performance may include collection \"\n",
|
||||
" \"artifacts and does not demonstrate robustness to future malware, concept drift, \"\n",
|
||||
" \"packing, or adversarial evasion. The model should be used as one triage signal \"\n",
|
||||
" \"alongside signatures, sandboxing, and analyst review.\",\n",
|
||||
" \"\",\n",
|
||||
"])\n",
|
||||
"\n",
|
||||
"metrics_table.to_csv(OUTPUT_DIR / \"capstone_metrics.csv\")\n",
|
||||
"errors_only.to_csv(OUTPUT_DIR / \"capstone_errors.csv\", index=True)\n",
|
||||
"(OUTPUT_DIR / \"capstone_tree_rules.txt\").write_text(tree_rules, encoding=\"utf-8\")\n",
|
||||
"(OUTPUT_DIR / \"capstone_analyst_summary.md\").write_text(analyst_summary, encoding=\"utf-8\")\n",
|
||||
"print(analyst_summary)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "cell-013",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Self-check"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "cell-014",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"expected_files = {\n",
|
||||
" \"capstone_metrics.csv\",\n",
|
||||
" \"capstone_errors.csv\",\n",
|
||||
" \"capstone_tree_rules.txt\",\n",
|
||||
" \"capstone_confusion_matrix.png\",\n",
|
||||
" \"capstone_analyst_summary.md\",\n",
|
||||
"}\n",
|
||||
"assert set(models) == {\"Dummy\", \"ExplainableTree\", \"DeepTree\", \"RandomForest\"}\n",
|
||||
"assert selected_name in set(models) - {\"Dummy\"}\n",
|
||||
"assert len(error_table) == len(y_test)\n",
|
||||
"assert set(errors_only[\"error_type\"].unique()).issubset({\"false_negative\", \"false_positive\"})\n",
|
||||
"assert isinstance(tree_rules, str) and \"class:\" in tree_rules\n",
|
||||
"assert expected_files.issubset({path.name for path in OUTPUT_DIR.iterdir()})\n",
|
||||
"print(\"✓ Lab 6 completed\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "cell-015",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Suggested review checklist\n",
|
||||
"\n",
|
||||
"- The recommendation follows a rule stated before inspecting individual errors.\n",
|
||||
"- Accuracy is not used as the sole decision metric.\n",
|
||||
"- False negatives and false positives are translated into operational consequences.\n",
|
||||
"- `source` is used only for auditing, never as a model feature.\n",
|
||||
"- Tree rules are treated as model explanations, not universal malware signatures.\n",
|
||||
"- Dataset age, imbalance, source confounding, drift, and evasion are acknowledged.\n",
|
||||
"- The detector is positioned as one layer in a defense workflow, not an autonomous verdict."
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.6"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
Reference in New Issue
Block a user