407 lines
12 KiB
Plaintext
407 lines
12 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {
|
||
"id": "5Qdt9Zcpwy56"
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"import numpy as np\n",
|
||
"import pandas as pd\n",
|
||
"import matplotlib.pyplot as plt\n",
|
||
"\n",
|
||
"%matplotlib inline"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {
|
||
"id": "eXE4wozRwskK"
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"dataset= pd.read_csv('datasets2/network-logs.csv')"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/",
|
||
"height": 281
|
||
},
|
||
"id": "haqtAIADxFDq",
|
||
"outputId": "6f182e24-81da-4043-f106-bebd63f8fa33"
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"hist_dist = dataset[['LATENCY','THROUGHPUT']].hist(grid=False, figsize=(10,4))"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/",
|
||
"height": 295
|
||
},
|
||
"id": "jwS1qbeWxgj8",
|
||
"outputId": "af12a8b1-8153-4d15-9db5-aea591fc613f"
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"data = dataset[['LATENCY','THROUGHPUT']].values\n",
|
||
"plt.scatter(data[:, 0], data[:, 1], alpha=0.6)\n",
|
||
"plt.xlabel('LATENCY')\n",
|
||
"plt.ylabel('THROUGHPUT')\n",
|
||
"plt.title('Data Flow')\n",
|
||
"plt.show()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {
|
||
"id": "0ir5L5ZIzrtd"
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"\"\"\"Anomaly Detection Module\"\"\"\n",
|
||
"\n",
|
||
"import math\n",
|
||
"import numpy as np\n",
|
||
"\n",
|
||
"\n",
|
||
"class GaussianAnomalyDetection:\n",
|
||
" \"\"\"GaussianAnomalyDetection Class\"\"\"\n",
|
||
"\n",
|
||
" def __init__(self, data):\n",
|
||
" \"\"\"GaussianAnomalyDetection constructor\"\"\"\n",
|
||
"\n",
|
||
" # Estimate Gaussian distribution.\n",
|
||
" (self.mu_param, self.sigma_squared) = GaussianAnomalyDetection.estimate_gaussian(data)\n",
|
||
"\n",
|
||
" # Save training data.\n",
|
||
" self.data = data\n",
|
||
"\n",
|
||
" def multivariate_gaussian(self, data):\n",
|
||
" \"\"\"Computes the probability density function of the multivariate gaussian distribution\"\"\"\n",
|
||
"\n",
|
||
" mu_param = self.mu_param\n",
|
||
" sigma_squared = self.sigma_squared\n",
|
||
"\n",
|
||
" # Get number of training sets and features.\n",
|
||
" (num_examples, num_features) = data.shape\n",
|
||
"\n",
|
||
" # nit probabilities matrix.\n",
|
||
" probabilities = np.ones((num_examples, 1))\n",
|
||
"\n",
|
||
" # Go through all training examples and through all features.\n",
|
||
" for example_index in range(num_examples):\n",
|
||
" for feature_index in range(num_features):\n",
|
||
" # Calculate the power of e.\n",
|
||
" power_dividend = (data[example_index, feature_index] - mu_param[feature_index]) ** 2\n",
|
||
" power_divider = 2 * sigma_squared[feature_index]\n",
|
||
" e_power = -1 * power_dividend / power_divider\n",
|
||
"\n",
|
||
" # Calculate the prefix multiplier.\n",
|
||
" probability_prefix = 1 / math.sqrt(2 * math.pi * sigma_squared[feature_index])\n",
|
||
"\n",
|
||
" # Calculate the probability for the current feature of current example.\n",
|
||
" probability = probability_prefix * (math.e ** e_power)\n",
|
||
" probabilities[example_index] *= probability\n",
|
||
"\n",
|
||
" # Return probabilities for all training examples.\n",
|
||
" return probabilities\n",
|
||
"\n",
|
||
" @staticmethod\n",
|
||
" def estimate_gaussian(data):\n",
|
||
" \"\"\"This function estimates the parameters of a Gaussian distribution using the data in X.\"\"\"\n",
|
||
"\n",
|
||
" # Get number of features and number of examples.\n",
|
||
" num_examples = data.shape[0]\n",
|
||
"\n",
|
||
" # Estimate Gaussian parameters mu and sigma_squared for every feature.\n",
|
||
" mu_param = (1 / num_examples) * np.sum(data, axis=0)\n",
|
||
" sigma_squared = (1 / num_examples) * np.sum((data - mu_param) ** 2, axis=0)\n",
|
||
"\n",
|
||
" # Return Gaussian parameters.\n",
|
||
" return mu_param, sigma_squared\n",
|
||
"\n",
|
||
" @staticmethod\n",
|
||
" def select_threshold(labels, probabilities):\n",
|
||
" # pylint: disable=R0914\n",
|
||
" \"\"\"Finds the best threshold (epsilon) to use for selecting outliers\"\"\"\n",
|
||
"\n",
|
||
" best_epsilon = 0\n",
|
||
" best_f1 = 0\n",
|
||
"\n",
|
||
" # History data to build the plots.\n",
|
||
" precision_history = []\n",
|
||
" recall_history = []\n",
|
||
" f1_history = []\n",
|
||
"\n",
|
||
" # Calculate the epsilon steps.\n",
|
||
" min_probability = np.min(probabilities)\n",
|
||
" max_probability = np.max(probabilities)\n",
|
||
" step_size = (max_probability - min_probability) / 1000\n",
|
||
"\n",
|
||
" # Go through all possible epsilons and pick the one with the highest f1 score.\n",
|
||
" for epsilon in np.arange(min_probability, max_probability, step_size):\n",
|
||
" predictions = probabilities < epsilon\n",
|
||
"\n",
|
||
" # The number of false positives: the ground truth label says it’s not\n",
|
||
" # an anomaly, but our algorithm incorrectly classified it as an anomaly.\n",
|
||
" false_positives = np.sum((predictions == 1) & (labels == 0))\n",
|
||
"\n",
|
||
" # The number of false negatives: the ground truth label says it’s an anomaly,\n",
|
||
" # but our algorithm incorrectly classified it as not being anomalous.\n",
|
||
" false_negatives = np.sum((predictions == 0) & (labels == 1))\n",
|
||
"\n",
|
||
" # The number of true positives: the ground truth label says it’s an\n",
|
||
" # anomaly and our algorithm correctly classified it as an anomaly.\n",
|
||
" true_positives = np.sum((predictions == 1) & (labels == 1))\n",
|
||
"\n",
|
||
" # Prevent division by zero.\n",
|
||
" if (true_positives + false_positives) == 0 or (true_positives + false_negatives) == 0:\n",
|
||
" continue\n",
|
||
"\n",
|
||
" # Precision.\n",
|
||
" precision = true_positives / (true_positives + false_positives)\n",
|
||
"\n",
|
||
" # Recall.\n",
|
||
" recall = true_positives / (true_positives + false_negatives)\n",
|
||
"\n",
|
||
" # F1.\n",
|
||
" f1_score = 2 * precision * recall / (precision + recall)\n",
|
||
"\n",
|
||
" # Save history data.\n",
|
||
" precision_history.append(precision)\n",
|
||
" recall_history.append(recall)\n",
|
||
" f1_history.append(f1_score)\n",
|
||
"\n",
|
||
" if f1_score > best_f1:\n",
|
||
" best_epsilon = epsilon\n",
|
||
" best_f1 = f1_score\n",
|
||
"\n",
|
||
" return best_epsilon, best_f1, precision_history, recall_history, f1_history"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/"
|
||
},
|
||
"id": "xvPaIOLPyJ4y",
|
||
"outputId": "3a45d922-29cb-4819-971a-e33121bb2900"
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"#Thanks to Oleksii Trekhleb: https://github.com/trekhleb/homemade-machine-learning/blob/master/homemade/anomaly_detection/gaussian_anomaly_detection.py\n",
|
||
"gaussian_anomaly_detection = GaussianAnomalyDetection(data)\n",
|
||
"\n",
|
||
"print('mu param estimation: ')\n",
|
||
"print(gaussian_anomaly_detection.mu_param)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/"
|
||
},
|
||
"id": "JSdpJIlX1BI7",
|
||
"outputId": "68b88a84-c0eb-435e-9f33-62ad5d61896e"
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"# print sigma squared estimation\n",
|
||
"print(gaussian_anomaly_detection.sigma_squared)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/"
|
||
},
|
||
"id": "Xku4v-771Nmi",
|
||
"outputId": "dd8b213a-83e6-4477-959b-ed96af3ec24f"
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"targets = dataset['ANOMALY'].values.reshape((data.shape[0], 1))\n",
|
||
"probs = gaussian_anomaly_detection.multivariate_gaussian(data)\n",
|
||
"(threshold, F1, precision_, recall_, f1_) = gaussian_anomaly_detection.select_threshold(targets, probs)\n",
|
||
"print('\\n')\n",
|
||
"print('threshold estimation: ')\n",
|
||
"print(threshold)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/",
|
||
"height": 295
|
||
},
|
||
"id": "tzl0vJA215XU",
|
||
"outputId": "25cc06b4-bd7c-4af1-fbdb-0b84dc45ffe3"
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"outliers = np.where(probs < threshold)[0]\n",
|
||
"plt.scatter(data[:, 0], data[:, 1], alpha=0.6, label='Dataset')\n",
|
||
"plt.xlabel('LATENCY')\n",
|
||
"plt.ylabel('THROUGHPUT')\n",
|
||
"plt.title('Data Flow')\n",
|
||
"plt.scatter(data[outliers, 0], data[outliers, 1], alpha=0.6, c = 'red', label='Outliers')\n",
|
||
"plt.legend()\n",
|
||
"plt.show()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {
|
||
"id": "FfHXl2wI3SyB"
|
||
},
|
||
"source": [
|
||
"**False alarm management in anomaly detection**\n",
|
||
"\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/"
|
||
},
|
||
"id": "a5xbB9oH3xyl",
|
||
"outputId": "3563dffd-3472-4821-abf5-41cb9fac2078"
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Sensitivity or True Positive rate (TPR) = True positive / (True Positive + False Negative)\n",
|
||
"# F1 = 2* precision * sensitivity /(Precision * sensitivy)\n",
|
||
"print('F1 score: ')\n",
|
||
"print(F1)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {
|
||
"id": "lspjMm4_4src"
|
||
},
|
||
"source": [
|
||
"**Reveiver operating characteristic analysis**"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {
|
||
"id": "vMeNGF1f4xHp"
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"from sklearn.metrics import roc_curve\n",
|
||
"FPR,TPR,OPC = roc_curve(targets, probs)\n",
|
||
"#TPR is true positive rate\n",
|
||
"#FPR is false positive rate\n",
|
||
"# Operating characteristic OPC"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/",
|
||
"height": 281
|
||
},
|
||
"id": "T2HXei9P5gJz",
|
||
"outputId": "9a2e2dfc-9c16-43b9-c6ac-f68ff5bfda3d"
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Plotting sensitivity\n",
|
||
"plt.plot(OPC, TPR)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/",
|
||
"height": 281
|
||
},
|
||
"id": "ZcKxWc_65qrN",
|
||
"outputId": "d81b9678-76f0-4615-ac5c-d5e5b46f596d"
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Plotting ROC cuver\n",
|
||
"plt.plot(FPR, TPR)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
}
|
||
],
|
||
"metadata": {
|
||
"colab": {
|
||
"name": "Udemy Gaussian Amomaly detection.ipynb",
|
||
"provenance": []
|
||
},
|
||
"kernelspec": {
|
||
"display_name": "Python 3",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 3
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython3",
|
||
"version": "3.8.8"
|
||
},
|
||
"widgets": {
|
||
"application/vnd.jupyter.widget-state+json": {
|
||
"state": {},
|
||
"version_major": 2,
|
||
"version_minor": 0
|
||
}
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 4
|
||
}
|