{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": { "id": "5Qdt9Zcpwy56" }, "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "\n", "%matplotlib inline" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "eXE4wozRwskK" }, "outputs": [], "source": [ "dataset= pd.read_csv('datasets2/network-logs.csv')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 281 }, "id": "haqtAIADxFDq", "outputId": "6f182e24-81da-4043-f106-bebd63f8fa33" }, "outputs": [], "source": [ "hist_dist = dataset[['LATENCY','THROUGHPUT']].hist(grid=False, figsize=(10,4))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 295 }, "id": "jwS1qbeWxgj8", "outputId": "af12a8b1-8153-4d15-9db5-aea591fc613f" }, "outputs": [], "source": [ "data = dataset[['LATENCY','THROUGHPUT']].values\n", "plt.scatter(data[:, 0], data[:, 1], alpha=0.6)\n", "plt.xlabel('LATENCY')\n", "plt.ylabel('THROUGHPUT')\n", "plt.title('Data Flow')\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "0ir5L5ZIzrtd" }, "outputs": [], "source": [ "\"\"\"Anomaly Detection Module\"\"\"\n", "\n", "import math\n", "import numpy as np\n", "\n", "\n", "class GaussianAnomalyDetection:\n", " \"\"\"GaussianAnomalyDetection Class\"\"\"\n", "\n", " def __init__(self, data):\n", " \"\"\"GaussianAnomalyDetection constructor\"\"\"\n", "\n", " # Estimate Gaussian distribution.\n", " (self.mu_param, self.sigma_squared) = GaussianAnomalyDetection.estimate_gaussian(data)\n", "\n", " # Save training data.\n", " self.data = data\n", "\n", " def multivariate_gaussian(self, data):\n", " \"\"\"Computes the probability density function of the multivariate gaussian distribution\"\"\"\n", "\n", " mu_param = self.mu_param\n", " sigma_squared = self.sigma_squared\n", "\n", " # Get number of training sets and features.\n", " (num_examples, num_features) = data.shape\n", "\n", " # nit probabilities matrix.\n", " probabilities = np.ones((num_examples, 1))\n", "\n", " # Go through all training examples and through all features.\n", " for example_index in range(num_examples):\n", " for feature_index in range(num_features):\n", " # Calculate the power of e.\n", " power_dividend = (data[example_index, feature_index] - mu_param[feature_index]) ** 2\n", " power_divider = 2 * sigma_squared[feature_index]\n", " e_power = -1 * power_dividend / power_divider\n", "\n", " # Calculate the prefix multiplier.\n", " probability_prefix = 1 / math.sqrt(2 * math.pi * sigma_squared[feature_index])\n", "\n", " # Calculate the probability for the current feature of current example.\n", " probability = probability_prefix * (math.e ** e_power)\n", " probabilities[example_index] *= probability\n", "\n", " # Return probabilities for all training examples.\n", " return probabilities\n", "\n", " @staticmethod\n", " def estimate_gaussian(data):\n", " \"\"\"This function estimates the parameters of a Gaussian distribution using the data in X.\"\"\"\n", "\n", " # Get number of features and number of examples.\n", " num_examples = data.shape[0]\n", "\n", " # Estimate Gaussian parameters mu and sigma_squared for every feature.\n", " mu_param = (1 / num_examples) * np.sum(data, axis=0)\n", " sigma_squared = (1 / num_examples) * np.sum((data - mu_param) ** 2, axis=0)\n", "\n", " # Return Gaussian parameters.\n", " return mu_param, sigma_squared\n", "\n", " @staticmethod\n", " def select_threshold(labels, probabilities):\n", " # pylint: disable=R0914\n", " \"\"\"Finds the best threshold (epsilon) to use for selecting outliers\"\"\"\n", "\n", " best_epsilon = 0\n", " best_f1 = 0\n", "\n", " # History data to build the plots.\n", " precision_history = []\n", " recall_history = []\n", " f1_history = []\n", "\n", " # Calculate the epsilon steps.\n", " min_probability = np.min(probabilities)\n", " max_probability = np.max(probabilities)\n", " step_size = (max_probability - min_probability) / 1000\n", "\n", " # Go through all possible epsilons and pick the one with the highest f1 score.\n", " for epsilon in np.arange(min_probability, max_probability, step_size):\n", " predictions = probabilities < epsilon\n", "\n", " # The number of false positives: the ground truth label says it’s not\n", " # an anomaly, but our algorithm incorrectly classified it as an anomaly.\n", " false_positives = np.sum((predictions == 1) & (labels == 0))\n", "\n", " # The number of false negatives: the ground truth label says it’s an anomaly,\n", " # but our algorithm incorrectly classified it as not being anomalous.\n", " false_negatives = np.sum((predictions == 0) & (labels == 1))\n", "\n", " # The number of true positives: the ground truth label says it’s an\n", " # anomaly and our algorithm correctly classified it as an anomaly.\n", " true_positives = np.sum((predictions == 1) & (labels == 1))\n", "\n", " # Prevent division by zero.\n", " if (true_positives + false_positives) == 0 or (true_positives + false_negatives) == 0:\n", " continue\n", "\n", " # Precision.\n", " precision = true_positives / (true_positives + false_positives)\n", "\n", " # Recall.\n", " recall = true_positives / (true_positives + false_negatives)\n", "\n", " # F1.\n", " f1_score = 2 * precision * recall / (precision + recall)\n", "\n", " # Save history data.\n", " precision_history.append(precision)\n", " recall_history.append(recall)\n", " f1_history.append(f1_score)\n", "\n", " if f1_score > best_f1:\n", " best_epsilon = epsilon\n", " best_f1 = f1_score\n", "\n", " return best_epsilon, best_f1, precision_history, recall_history, f1_history" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "xvPaIOLPyJ4y", "outputId": "3a45d922-29cb-4819-971a-e33121bb2900" }, "outputs": [], "source": [ "#Thanks to Oleksii Trekhleb: https://github.com/trekhleb/homemade-machine-learning/blob/master/homemade/anomaly_detection/gaussian_anomaly_detection.py\n", "gaussian_anomaly_detection = GaussianAnomalyDetection(data)\n", "\n", "print('mu param estimation: ')\n", "print(gaussian_anomaly_detection.mu_param)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "JSdpJIlX1BI7", "outputId": "68b88a84-c0eb-435e-9f33-62ad5d61896e" }, "outputs": [], "source": [ "# print sigma squared estimation\n", "print(gaussian_anomaly_detection.sigma_squared)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "Xku4v-771Nmi", "outputId": "dd8b213a-83e6-4477-959b-ed96af3ec24f" }, "outputs": [], "source": [ "targets = dataset['ANOMALY'].values.reshape((data.shape[0], 1))\n", "probs = gaussian_anomaly_detection.multivariate_gaussian(data)\n", "(threshold, F1, precision_, recall_, f1_) = gaussian_anomaly_detection.select_threshold(targets, probs)\n", "print('\\n')\n", "print('threshold estimation: ')\n", "print(threshold)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 295 }, "id": "tzl0vJA215XU", "outputId": "25cc06b4-bd7c-4af1-fbdb-0b84dc45ffe3" }, "outputs": [], "source": [ "outliers = np.where(probs < threshold)[0]\n", "plt.scatter(data[:, 0], data[:, 1], alpha=0.6, label='Dataset')\n", "plt.xlabel('LATENCY')\n", "plt.ylabel('THROUGHPUT')\n", "plt.title('Data Flow')\n", "plt.scatter(data[outliers, 0], data[outliers, 1], alpha=0.6, c = 'red', label='Outliers')\n", "plt.legend()\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "colab": { "name": "Udemy Gaussian Amomaly detection.ipynb", "provenance": [] }, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.8" }, "widgets": { "application/vnd.jupyter.widget-state+json": { "state": {}, "version_major": 2, "version_minor": 0 } } }, "nbformat": 4, "nbformat_minor": 4 }