Detecting Anomalies with ML

!pip install tensorflow

import os; os.environ['TF_CPP_MIN_LOG_LEVEL']='2'
import warnings
warnings.filterwarnings('ignore')
 
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import StandardScaler
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Input, Dense
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping
 
# Generate a time series with a trend and seasonality
np.random.seed(42)
n_points = 24 * 7  # One week of hourly data
time_index = pd.date_range('2024-01-01', periods=n_points, freq='H')
 
# Base trend and seasonality
requests = np.linspace(500, 1500, n_points)  # Upward trend
daily_seasonality = 200 * np.sin(np.arange(n_points) * 2 * np.pi / 24)
noise = np.random.normal(0, 50, n_points)
requests = requests + daily_seasonality + noise
 
# Introduce anomalies
requests[50] += 5000
requests[100] += 3000
requests[130:135] = requests[130:135] / 10
 
# Create a DataFrame
server_data = pd.DataFrame({
    'timestamp': time_index,
    'requests': requests
}).set_index('timestamp')
 
# Visualize the data
plt.figure(figsize=(15, 6))
plt.plot(server_data.index, server_data['requests'], color='skyblue')
plt.title('Simulated Server Request Rates with Anomalies')
plt.xlabel('Time')
plt.ylabel('Requests per Hour')
plt.grid(True)
plt.show()

 
# Scale the data
scaler = StandardScaler()
scaled_data = scaler.fit_transform(server_data[['requests']])
 
# Reshape data for the Autoencoder
X = scaled_data
 
 
input_dim = X.shape[1]
encoding_dim = 1  # A compressed representation of the data
 
input_layer = Input(shape=(input_dim,))
 
# Encoder (compresses the data)
encoder = Dense(2, activation="relu")(input_layer)
encoder = Dense(encoding_dim, activation="relu")(encoder)
 
# Decoder (reconstructs the data)
decoder = Dense(2, activation="relu")(encoder)
decoder = Dense(input_dim, activation="sigmoid")(decoder)
 
# Autoencoder model
autoencoder = Model(inputs=input_layer, outputs=decoder)
autoencoder.compile(optimizer=Adam(learning_rate=0.001), loss='mse')
 
# Train the Autoencoder
# We use a small batch size for this simple example and early stopping to prevent overfitting
history = autoencoder.fit(
  X, X,
  epochs=50,
  batch_size=32,
  validation_split=0.1,
  callbacks=[EarlyStopping(monitor='val_loss', patience=5)],
verbose=0)

# Get the reconstruction of the original data
reconstructions = autoencoder.predict(X)
 
# Calculate the reconstruction error (Mean Squared Error) for each data point
mse_loss = np.mean(np.power(X - reconstructions, 2), axis=1)
 
# Find a reasonable threshold for anomalies
# We'll use the 95th percentile of the reconstruction loss as our threshold
threshold = np.percentile(mse_loss, 95)
# print(f"\nAnomaly Threshold: {threshold:.4f}")
 
# Predict anomalies by comparing reconstruction loss to the threshold
anomalies = server_data.iloc[mse_loss > threshold]
print(f"\nNumber of anomalies detected: {len(anomalies)}")
 
# Visualize the results
plt.figure(figsize=(15, 6))
plt.plot(server_data.index, server_data['requests'], label='Requests', color='skyblue')
plt.scatter(anomalies.index, anomalies['requests'], color='red', marker='o', s=50, label='Anomalies')
plt.title('Server Request Rates with Detected Anomalies (Autoencoder)')
plt.xlabel('Time')
plt.ylabel('Requests per Hour')
plt.legend()
plt.grid(True)
plt.show()

from sklearn.ensemble import IsolationForest
 
# Create an Isolation Forest model
# contamination: The amount of contamination of the data set, i.e.,
# the proportion of outliers in the data set. Used when fitting to define the threshold.
model = IsolationForest(n_estimators=100, contamination=0.02, random_state=42)
 
# Train the model (it's unsupervised, so we just pass the data)
model.fit(server_data[['requests']])
 
# Predict the anomalies. -1 means anomaly, 1 means normal.
server_data['anomaly'] = model.predict(server_data[['requests']])
 
# Extract the anomalies
anomalies = server_data[server_data['anomaly'] == -1]
 
print(f"Number of anomalies detected: {len(anomalies)}")
print("\nDetected Anomalies:")
print(anomalies)
 
# Plot the original data and the detected anomalies
plt.figure(figsize=(15, 6))
plt.plot(server_data.index, server_data['requests'], label='Requests', color='skyblue')
plt.scatter(anomalies.index, anomalies['requests'], color='red', marker='o', s=50, label='Anomalies')
plt.title('Server Request Rates with Detected Anomalies')
plt.xlabel('Time')
plt.ylabel('Requests per Hour')
plt.legend()
plt.grid(True)
plt.show()