Python Data Science: Analytics and Machine Learning

# This simplicity is why Python wins for data science
import pandas as pd
import numpy as np

# Load and explore data in just a few lines
data = pd.read_csv('sales_data.csv')
monthly_revenue = data.groupby('month')['revenue'].sum()
growth_rate = monthly_revenue.pct_change().mean()

print(f"Average monthly growth: {growth_rate:.2%}")

# The stack in action - each library plays its role
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.linear_model import LinearRegression

# NumPy: efficient numerical operations
prices = np.array([100, 105, 98, 110, 115])
returns = np.diff(prices) / prices[:-1]

# Pandas: structured data manipulation  
df = pd.DataFrame({
    'date': pd.date_range('2024-01-01', periods=len(returns)),
    'returns': returns
})

# Matplotlib: visualization
plt.plot(df['date'], df['returns'])
plt.title('Daily Returns')

# Scikit-learn: machine learning
model = LinearRegression()
X = np.arange(len(returns)).reshape(-1, 1)
model.fit(X, returns)

# Create a clean environment for data science work
conda create -n datasci python=3.9
conda activate datasci

# Install the core stack
conda install numpy pandas matplotlib seaborn
conda install scikit-learn jupyter notebook
conda install -c conda-forge plotly

# For specific projects, add requirements.txt
pip install -r requirements.txt

# Notebook cell: great for exploration
data = pd.read_csv('customer_data.csv')
data.head()  # Immediately see the results
data.describe()  # Quick statistical summary
data.isnull().sum()  # Check for missing values

# .py file with proper tooling setup
import pandas as pd
import numpy as np

def analyze_sales_trends(data_path: str) -> pd.DataFrame:
    """Analyze sales trends from CSV data.
    
    Args:
        data_path: Path to CSV file with sales data
        
    Returns:
        DataFrame with monthly trend analysis
    """
    data = pd.read_csv(data_path)
    
    # Convert date column and set as index
    data['date'] = pd.to_datetime(data['date'])
    data.set_index('date', inplace=True)
    
    # Calculate monthly aggregates
    monthly = data.resample('M').agg({
        'revenue': 'sum',
        'orders': 'count',
        'customers': 'nunique'
    })
    
    return monthly

import numpy as np
import time

# Compare Python list vs NumPy array performance
def python_sum_squares(data):
    return sum(x**2 for x in data)

def numpy_sum_squares(data):
    return np.sum(data**2)

# Create test data
size = 1000000
python_list = list(range(size))
numpy_array = np.array(python_list)

# Time both approaches
start = time.time()
result1 = python_sum_squares(python_list)
python_time = time.time() - start

start = time.time()
result2 = numpy_sum_squares(numpy_array)
numpy_time = time.time() - start

print(f"Python: {python_time:.4f}s, NumPy: {numpy_time:.4f}s")
print(f"NumPy is {python_time/numpy_time:.1f}x faster")

# NumPy automatically chooses appropriate dtypes
int_array = np.array([1, 2, 3])          # dtype: int64
float_array = np.array([1.0, 2.0, 3.0])  # dtype: float64
mixed_array = np.array([1, 2.0, 3])      # dtype: float64 (upcasted)

print(f"Integer array: {int_array.dtype}")
print(f"Float array: {float_array.dtype}")
print(f"Mixed array: {mixed_array.dtype}")

# Explicit dtype specification for memory efficiency
large_ints = np.array([100, 200, 300], dtype=np.int8)
print(f"Memory usage: {large_ints.nbytes} bytes")

# Structured array creation
zeros = np.zeros((3, 4))           # 3x4 array of zeros
ones = np.ones((2, 3))             # 2x3 array of ones  
identity = np.eye(4)               # 4x4 identity matrix

# Sequences and ranges
sequence = np.arange(0, 10, 2)     # [0, 2, 4, 6, 8]
linspace = np.linspace(0, 1, 5)    # 5 evenly spaced values

print("Sequence:", sequence)
print("Linear space:", linspace)

# Boolean indexing for data filtering
temperatures = np.array([20, 25, 30, 15, 35, 28, 22])
hot_days = temperatures > 25

print("Hot day mask:", hot_days)
print("Hot temperatures:", temperatures[hot_days])  # [30 35 28]

# Complex conditions
moderate_days = (temperatures >= 20) & (temperatures <= 30)
print("Moderate temperatures:", temperatures[moderate_days])

# Vectorized operations
arr = np.array([1, 2, 3, 4, 5])

# Element-wise operations on entire array
squared = arr ** 2                    # [1, 4, 9, 16, 25]
normalized = (arr - arr.mean()) / arr.std()  # Z-score normalization

print("Original:", arr)
print("Squared:", squared)
print("Normalized:", normalized)

# Broadcasting in action
matrix = np.array([[1, 2, 3],
                   [4, 5, 6]])
row_vector = np.array([10, 20, 30])

# Broadcasting adds the vector to each row
result = matrix + row_vector
print("Matrix + row vector:")
print(result)
# Output: [[11 22 33]
#          [14 25 36]]

# Element-wise mathematical functions
data = np.array([1, 4, 9, 16, 25])

sqrt_data = np.sqrt(data)           # [1, 2, 3, 4, 5]
log_data = np.log(data)             # Natural logarithm

# Statistical analysis
random_data = np.random.normal(0, 1, 1000)  # 1000 random normal values

print(f"Mean: {np.mean(random_data):.3f}")
print(f"Standard deviation: {np.std(random_data):.3f}")
print(f"Min: {np.min(random_data):.3f}")
print(f"Max: {np.max(random_data):.3f}")

# Sample sales data: products × quarters
sales_data = np.array([[100, 120, 110],  # Product A
                       [80, 90, 95],     # Product B  
                       [150, 140, 160]]) # Product C

# Aggregations along different axes
total_by_product = np.sum(sales_data, axis=1)    # Sum across quarters
total_by_quarter = np.sum(sales_data, axis=0)    # Sum across products

print("Sales by product:", total_by_product)     # [330, 265, 450]
print("Sales by quarter:", total_by_quarter)     # [330, 350, 365]

import pandas as pd
import numpy as np

# Create a DataFrame from a dictionary
data = {
    'name': ['Alice', 'Bob', 'Charlie', 'Diana'],
    'age': [25, 30, 35, 28],
    'salary': [50000, 60000, 70000, 55000],
    'department': ['Engineering', 'Sales', 'Engineering', 'Marketing']
}

df = pd.DataFrame(data)
print(df)
print(f"\nData types:\n{df.dtypes}")

# For demonstration, create sample data
np.random.seed(42)
sample_data = pd.DataFrame({
    'date': pd.date_range('2024-01-01', periods=100),
    'sales': np.random.normal(1000, 200, 100),
    'region': np.random.choice(['North', 'South', 'East', 'West'], 100),
    'product': np.random.choice(['A', 'B', 'C'], 100)
})

# Essential exploration methods
print("Dataset shape:", sample_data.shape)
print("\nFirst few rows:")
print(sample_data.head())
print("\nData types and memory usage:")
print(sample_data.info())

# Column selection
sales_column = sample_data['sales']           # Single column (Series)
subset = sample_data[['sales', 'region']]     # Multiple columns (DataFrame)

# Row selection by condition
high_sales = sample_data[sample_data['sales'] > 1200]
north_region = sample_data[sample_data['region'] == 'North']

# Combined conditions
north_high_sales = sample_data[
    (sample_data['region'] == 'North') & 
    (sample_data['sales'] > 1200)
]

print(f"High sales records: {len(high_sales)}")
print(f"North region records: {len(north_region)}")

# Introduce some missing values for demonstration
sample_data_dirty = sample_data.copy()
sample_data_dirty.loc[5:10, 'sales'] = np.nan

# Check for missing values
print("Missing values per column:")
print(sample_data_dirty.isnull().sum())

# Handle missing values
clean_data = sample_data_dirty.dropna()
filled_data = sample_data_dirty.fillna({
    'sales': sample_data_dirty['sales'].mean()
})

print(f"Original: {len(sample_data_dirty)} rows")
print(f"After dropna: {len(clean_data)} rows")
print(f"After fillna: {len(filled_data)} rows")

# Data type conversions and new columns
sample_data['product'] = sample_data['product'].astype('category')
sample_data['date'] = pd.to_datetime(sample_data['date'])
sample_data['month'] = sample_data['date'].dt.month
sample_data['sales_category'] = pd.cut(sample_data['sales'], 
                                      bins=[0, 800, 1200, float('inf')],
                                      labels=['Low', 'Medium', 'High'])

print("New columns:")
print(sample_data[['date', 'month', 'sales', 'sales_category']].head())

# Basic grouping operations
region_stats = sample_data.groupby('region')['sales'].agg([
    'count', 'mean', 'std', 'min', 'max'
])

print("Sales statistics by region:")
print(region_stats.round(2))

# Multiple grouping variables
monthly_region_sales = sample_data.groupby(['month', 'region'])['sales'].sum()
print("\nMonthly sales by region:")
print(monthly_region_sales.head(10))

# Custom aggregation functions
def sales_range(series):
    return series.max() - series.min()

custom_stats = sample_data.groupby('region')['sales'].agg([
    ('mean', 'mean'),
    ('range', sales_range)
])

print("Custom statistics by region:")
print(custom_stats.round(2))

# Create sample datasets to merge
customers = pd.DataFrame({
    'customer_id': [1, 2, 3, 4],
    'name': ['Alice', 'Bob', 'Charlie', 'Diana'],
    'city': ['New York', 'London', 'Tokyo', 'Paris']
})

orders = pd.DataFrame({
    'order_id': [101, 102, 103, 104],
    'customer_id': [1, 2, 2, 3],
    'amount': [100, 150, 200, 75]
})

# Inner join (only matching records)
merged = pd.merge(customers, orders, on='customer_id', how='inner')
print("Merged data:")
print(merged)

# Time series operations
sample_data.set_index('date', inplace=True)

# Resampling to different frequencies
weekly_sales = sample_data['sales'].resample('W').mean()
monthly_sales = sample_data['sales'].resample('M').sum()

print("Monthly sales:")
print(monthly_sales)

# Rolling window calculations
sample_data['sales_7day_avg'] = sample_data['sales'].rolling(window=7).mean()

print("\nSales with moving average:")
print(sample_data[['sales', 'sales_7day_avg']].head(10))

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

# Create sample data
np.random.seed(42)
x = np.linspace(0, 10, 100)
y = np.sin(x) + np.random.normal(0, 0.1, 100)

# Basic plotting
plt.figure(figsize=(10, 6))
plt.plot(x, y, 'b-', alpha=0.7, label='Noisy sine wave')
plt.xlabel('X values')
plt.ylabel('Y values')
plt.title('Basic Line Plot')
plt.legend()
plt.grid(True, alpha=0.3)
plt.show()

import seaborn as sns

# Create sample dataset
data = pd.DataFrame({
    'group': np.random.choice(['A', 'B', 'C'], 300),
    'value': np.random.normal(0, 1, 300),
    'category': np.random.choice(['X', 'Y'], 300)
})

# Statistical plots with seaborn
fig, axes = plt.subplots(2, 2, figsize=(12, 10))

# Box plot
sns.boxplot(data=data, x='group', y='value', ax=axes[0,0])
axes[0,0].set_title('Distribution by Group')

# Violin plot
sns.violinplot(data=data, x='group', y='value', hue='category', ax=axes[0,1])
axes[0,1].set_title('Distribution by Group and Category')

# Scatter plot with regression
sns.scatterplot(data=data, x='value', y=np.random.normal(0, 1, 300), 
                hue='group', ax=axes[1,0])
axes[1,0].set_title('Scatter Plot with Groups')

# Histogram
sns.histplot(data=data, x='value', hue='group', ax=axes[1,1])
axes[1,1].set_title('Histogram by Group')

plt.tight_layout()
plt.show()

# Load sample sales data
sales_data = pd.DataFrame({
    'date': pd.date_range('2023-01-01', periods=365),
    'sales': np.random.normal(1000, 200, 365) + 
             50 * np.sin(np.arange(365) * 2 * np.pi / 365),  # Seasonal pattern
    'region': np.random.choice(['North', 'South', 'East', 'West'], 365),
    'product': np.random.choice(['A', 'B', 'C'], 365)
})

# Time series plot
plt.figure(figsize=(15, 10))

plt.subplot(2, 3, 1)
plt.plot(sales_data['date'], sales_data['sales'])
plt.title('Sales Over Time')
plt.xticks(rotation=45)

# Distribution plot
plt.subplot(2, 3, 2)
sns.histplot(sales_data['sales'], bins=30)
plt.title('Sales Distribution')

# Box plot by category
plt.subplot(2, 3, 3)
sns.boxplot(data=sales_data, x='region', y='sales')
plt.title('Sales by Region')

# Correlation heatmap (for numerical data)
plt.subplot(2, 3, 4)
sales_data['month'] = sales_data['date'].dt.month
sales_data['day_of_year'] = sales_data['date'].dt.dayofyear
corr_data = sales_data[['sales', 'month', 'day_of_year']].corr()
sns.heatmap(corr_data, annot=True, cmap='coolwarm', center=0)
plt.title('Correlation Matrix')

# Scatter plot with trend
plt.subplot(2, 3, 5)
sns.scatterplot(data=sales_data, x='day_of_year', y='sales', hue='region', alpha=0.6)
plt.title('Sales vs Day of Year')

# Bar plot of averages
plt.subplot(2, 3, 6)
region_avg = sales_data.groupby('region')['sales'].mean()
region_avg.plot(kind='bar')
plt.title('Average Sales by Region')
plt.xticks(rotation=45)

plt.tight_layout()
plt.show()

# Create complex sample data
np.random.seed(42)
complex_data = pd.DataFrame({
    'x': np.random.normal(0, 1, 1000),
    'y': np.random.normal(0, 1, 1000),
    'size': np.random.uniform(10, 100, 1000),
    'category': np.random.choice(['Type1', 'Type2', 'Type3', 'Type4'], 1000)
})

# Multi-dimensional scatter plot
plt.figure(figsize=(12, 8))

# Use size and color to show 4 dimensions
scatter = plt.scatter(complex_data['x'], complex_data['y'], 
                     s=complex_data['size'], 
                     c=complex_data['category'].astype('category').cat.codes,
                     alpha=0.6, cmap='viridis')

plt.xlabel('X Dimension')
plt.ylabel('Y Dimension')
plt.title('Multi-dimensional Scatter Plot')
plt.colorbar(scatter, label='Category')

# Add size legend
sizes = [20, 50, 100]
labels = ['Small', 'Medium', 'Large']
legend_elements = [plt.scatter([], [], s=s, c='gray', alpha=0.6) for s in sizes]
plt.legend(legend_elements, labels, title='Size', loc='upper right')

plt.show()

# Set style for professional appearance
plt.style.use('seaborn-v0_8')
sns.set_palette("husl")

# Create a publication-ready plot
fig, ax = plt.subplots(figsize=(10, 6))

# Plot with custom styling
for region in sales_data['region'].unique():
    region_data = sales_data[sales_data['region'] == region]
    monthly_sales = region_data.groupby(region_data['date'].dt.month)['sales'].mean()
    
    ax.plot(monthly_sales.index, monthly_sales.values, 
            marker='o', linewidth=2, markersize=6, label=region)

ax.set_xlabel('Month', fontsize=12, fontweight='bold')
ax.set_ylabel('Average Sales', fontsize=12, fontweight='bold')
ax.set_title('Seasonal Sales Patterns by Region', fontsize=14, fontweight='bold')
ax.legend(title='Region', title_fontsize=12, fontsize=10)
ax.grid(True, alpha=0.3)

# Customize tick labels
ax.set_xticks(range(1, 13))
ax.set_xticklabels(['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
                   'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'])

plt.tight_layout()
plt.show()

import plotly.express as px
import plotly.graph_objects as go

# Interactive scatter plot
fig = px.scatter(sales_data, x='day_of_year', y='sales', 
                color='region', size='sales',
                hover_data=['date', 'product'],
                title='Interactive Sales Analysis')

fig.update_layout(
    xaxis_title='Day of Year',
    yaxis_title='Sales Amount',
    font=dict(size=12)
)

# This would show an interactive plot in Jupyter
# fig.show()

import numpy as np
import pandas as pd
import scipy.stats as stats
import matplotlib.pyplot as plt
import seaborn as sns

# Generate sample data with different characteristics
np.random.seed(42)
normal_data = np.random.normal(100, 15, 1000)
skewed_data = np.random.exponential(2, 1000)
bimodal_data = np.concatenate([np.random.normal(80, 10, 500), 
                              np.random.normal(120, 10, 500)])

# Comprehensive descriptive statistics
def analyze_distribution(data, name):
    stats_dict = {
        'mean': np.mean(data),
        'median': np.median(data),
        'std': np.std(data),
        'skewness': stats.skew(data),
        'kurtosis': stats.kurtosis(data),
        'min': np.min(data),
        'max': np.max(data),
        'q25': np.percentile(data, 25),
        'q75': np.percentile(data, 75)
    }
    
    print(f"\n{name} Distribution:")
    for key, value in stats_dict.items():
        print(f"{key}: {value:.3f}")
    
    return stats_dict

# Analyze different distributions
normal_stats = analyze_distribution(normal_data, "Normal")
skewed_stats = analyze_distribution(skewed_data, "Skewed")
bimodal_stats = analyze_distribution(bimodal_data, "Bimodal")

# Calculate confidence intervals for different scenarios
def confidence_interval(data, confidence=0.95):
    """Calculate confidence interval for the mean."""
    n = len(data)
    mean = np.mean(data)
    std_err = stats.sem(data)  # Standard error of the mean
    
    # t-distribution for small samples, normal for large samples
    if n < 30:
        t_val = stats.t.ppf((1 + confidence) / 2, n - 1)
        margin_error = t_val * std_err
    else:
        z_val = stats.norm.ppf((1 + confidence) / 2)
        margin_error = z_val * std_err
    
    return mean - margin_error, mean + margin_error

# Compare confidence intervals for different sample sizes
sample_sizes = [10, 30, 100, 1000]
population = np.random.normal(100, 15, 10000)

print("Confidence Intervals by Sample Size:")
for n in sample_sizes:
    sample = np.random.choice(population, n, replace=False)
    ci_lower, ci_upper = confidence_interval(sample)
    width = ci_upper - ci_lower
    
    print(f"n={n:4d}: [{ci_lower:.2f}, {ci_upper:.2f}] (width: {width:.2f})")

# A/B test example: comparing two marketing campaigns
def ab_test_analysis(control_data, treatment_data, alpha=0.05):
    """Perform independent t-test for A/B testing."""
    
    # Descriptive statistics
    control_mean = np.mean(control_data)
    treatment_mean = np.mean(treatment_data)
    
    # Statistical test
    t_stat, p_value = stats.ttest_ind(control_data, treatment_data)
    
    # Effect size (Cohen's d)
    pooled_std = np.sqrt(((len(control_data) - 1) * np.var(control_data, ddof=1) + 
                         (len(treatment_data) - 1) * np.var(treatment_data, ddof=1)) / 
                        (len(control_data) + len(treatment_data) - 2))
    cohens_d = (treatment_mean - control_mean) / pooled_std
    
    # Results interpretation
    is_significant = p_value < alpha
    improvement = ((treatment_mean - control_mean) / control_mean) * 100
    
    results = {
        'control_mean': control_mean,
        'treatment_mean': treatment_mean,
        'improvement_pct': improvement,
        't_statistic': t_stat,
        'p_value': p_value,
        'is_significant': is_significant,
        'effect_size': cohens_d,
        'sample_sizes': (len(control_data), len(treatment_data))
    }
    
    return results

# Simulate A/B test data
np.random.seed(42)
control_conversions = np.random.normal(0.12, 0.05, 1000)  # 12% baseline conversion
treatment_conversions = np.random.normal(0.14, 0.05, 1000)  # 14% treatment conversion

# Analyze results
ab_results = ab_test_analysis(control_conversions, treatment_conversions)

print("A/B Test Results:")
print(f"Control mean: {ab_results['control_mean']:.4f}")
print(f"Treatment mean: {ab_results['treatment_mean']:.4f}")
print(f"Improvement: {ab_results['improvement_pct']:.2f}%")
print(f"P-value: {ab_results['p_value']:.6f}")
print(f"Statistically significant: {ab_results['is_significant']}")
print(f"Effect size (Cohen's d): {ab_results['effect_size']:.3f}")

# Demonstrate correlation analysis and interpretation
def correlation_analysis(df, var1, var2):
    """Comprehensive correlation analysis between two variables."""
    
    # Calculate different correlation coefficients
    pearson_r, pearson_p = stats.pearsonr(df[var1], df[var2])
    spearman_r, spearman_p = stats.spearmanr(df[var1], df[var2])
    
    # Linear regression for trend line
    slope, intercept, r_value, p_value, std_err = stats.linregress(df[var1], df[var2])
    
    print(f"Correlation Analysis: {var1} vs {var2}")
    print(f"Pearson correlation: {pearson_r:.3f} (p={pearson_p:.6f})")
    print(f"Spearman correlation: {spearman_r:.3f} (p={spearman_p:.6f})")
    print(f"R-squared: {r_value**2:.3f}")
    
    # Visualization
    plt.figure(figsize=(10, 6))
    plt.scatter(df[var1], df[var2], alpha=0.6)
    
    # Add trend line
    x_trend = np.linspace(df[var1].min(), df[var1].max(), 100)
    y_trend = slope * x_trend + intercept
    plt.plot(x_trend, y_trend, 'r-', linewidth=2)
    
    plt.xlabel(var1)
    plt.ylabel(var2)
    plt.title(f'Correlation: {var1} vs {var2} (r={pearson_r:.3f})')
    plt.show()
    
    return pearson_r, spearman_r

# Create sample data with different correlation patterns
np.random.seed(42)
sample_df = pd.DataFrame({
    'advertising_spend': np.random.uniform(1000, 10000, 200),
    'sales': np.random.normal(50000, 10000, 200),
    'temperature': np.random.normal(20, 10, 200),
    'ice_cream_sales': np.random.normal(1000, 300, 200)
})

# Add some actual correlation
sample_df['sales'] += sample_df['advertising_spend'] * 2 + np.random.normal(0, 5000, 200)
sample_df['ice_cream_sales'] += sample_df['temperature'] * 50 + np.random.normal(0, 200, 200)

# Analyze correlations
correlation_analysis(sample_df, 'advertising_spend', 'sales')
correlation_analysis(sample_df, 'temperature', 'ice_cream_sales')

# Power analysis for experiment design
def power_analysis(effect_size, alpha=0.05, power=0.8):
    """Calculate required sample size for given effect size and power."""
    from scipy.stats import norm
    
    # Z-scores for alpha and power
    z_alpha = norm.ppf(1 - alpha/2)  # Two-tailed test
    z_beta = norm.ppf(power)
    
    # Sample size calculation (per group)
    n_per_group = 2 * ((z_alpha + z_beta) / effect_size) ** 2
    
    return int(np.ceil(n_per_group))

# Calculate sample sizes for different scenarios
effect_sizes = [0.1, 0.2, 0.5, 0.8]  # Small, small-medium, medium, large
powers = [0.8, 0.9, 0.95]

print("Sample Size Requirements (per group):")
print("Effect Size | Power=0.8 | Power=0.9 | Power=0.95")
print("-" * 50)

for es in effect_sizes:
    row = f"{es:10.1f} |"
    for power in powers:
        n = power_analysis(es, power=power)
        row += f"{n:9d} |"
    print(row)

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_squared_error, r2_score
import matplotlib.pyplot as plt

# Create sample dataset for demonstration
np.random.seed(42)
n_samples = 1000

# Generate features with different relationships
X = pd.DataFrame({
    'feature1': np.random.normal(0, 1, n_samples),
    'feature2': np.random.normal(0, 1, n_samples),
    'feature3': np.random.uniform(0, 10, n_samples),
    'feature4': np.random.exponential(2, n_samples)
})

# Create target with known relationships
y = (2 * X['feature1'] + 
     -1.5 * X['feature2'] + 
     0.5 * X['feature3']**2 + 
     np.log(X['feature4'] + 1) + 
     np.random.normal(0, 0.5, n_samples))

print("Dataset shape:", X.shape)
print("Target statistics:")
print(f"Mean: {y.mean():.3f}, Std: {y.std():.3f}")

# Split data before any preprocessing to avoid data leakage
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# Feature scaling - important for many algorithms
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# Feature engineering - create polynomial features
X_train_engineered = X_train.copy()
X_test_engineered = X_test.copy()

# Add polynomial terms
X_train_engineered['feature3_squared'] = X_train['feature3'] ** 2
X_test_engineered['feature3_squared'] = X_test['feature3'] ** 2

# Add logarithmic transformation
X_train_engineered['feature4_log'] = np.log(X_train['feature4'] + 1)
X_test_engineered['feature4_log'] = np.log(X_test['feature4'] + 1)

# Add interaction terms
X_train_engineered['feature1_x_feature2'] = X_train['feature1'] * X_train['feature2']
X_test_engineered['feature1_x_feature2'] = X_test['feature1'] * X_test['feature2']

print("Original features:", X_train.shape[1])
print("Engineered features:", X_train_engineered.shape[1])

# Define models to compare
models = {
    'Linear Regression': LinearRegression(),
    'Random Forest': RandomForestRegressor(n_estimators=100, random_state=42)
}

# Train and evaluate models
results = {}

for name, model in models.items():
    # Train on engineered features
    model.fit(X_train_engineered, y_train)
    
    # Make predictions
    y_pred_train = model.predict(X_train_engineered)
    y_pred_test = model.predict(X_test_engineered)
    
    # Calculate metrics
    train_mse = mean_squared_error(y_train, y_pred_train)
    test_mse = mean_squared_error(y_test, y_pred_test)
    train_r2 = r2_score(y_train, y_pred_train)
    test_r2 = r2_score(y_test, y_pred_test)
    
    results[name] = {
        'train_mse': train_mse,
        'test_mse': test_mse,
        'train_r2': train_r2,
        'test_r2': test_r2
    }
    
    print(f"\n{name} Results:")
    print(f"Train R²: {train_r2:.3f}, Test R²: {test_r2:.3f}")
    print(f"Train MSE: {train_mse:.3f}, Test MSE: {test_mse:.3f}")

from sklearn.model_selection import cross_val_score, KFold

# Set up cross-validation
cv = KFold(n_splits=5, shuffle=True, random_state=42)

# Evaluate models with cross-validation
cv_results = {}

for name, model in models.items():
    # Cross-validation scores
    cv_scores = cross_val_score(model, X_train_engineered, y_train, 
                               cv=cv, scoring='r2')
    
    cv_results[name] = {
        'mean_score': cv_scores.mean(),
        'std_score': cv_scores.std(),
        'scores': cv_scores
    }
    
    print(f"\n{name} Cross-Validation:")
    print(f"Mean R²: {cv_scores.mean():.3f} ± {cv_scores.std():.3f}")
    print(f"Individual scores: {cv_scores}")

from sklearn.model_selection import GridSearchCV

# Define parameter grids for tuning
param_grids = {
    'Random Forest': {
        'n_estimators': [50, 100, 200],
        'max_depth': [None, 10, 20],
        'min_samples_split': [2, 5, 10]
    }
}

# Perform grid search
tuned_models = {}

for name, model in models.items():
    if name in param_grids:
        print(f"\nTuning {name}...")
        
        grid_search = GridSearchCV(
            model, param_grids[name], 
            cv=3, scoring='r2', n_jobs=-1
        )
        
        grid_search.fit(X_train_engineered, y_train)
        
        tuned_models[name] = grid_search.best_estimator_
        
        print(f"Best parameters: {grid_search.best_params_}")
        print(f"Best CV score: {grid_search.best_score_:.3f}")
    else:
        tuned_models[name] = model

# Feature importance for tree-based models
rf_model = tuned_models['Random Forest']
feature_names = X_train_engineered.columns

# Get feature importances
importances = rf_model.feature_importances_
feature_importance_df = pd.DataFrame({
    'feature': feature_names,
    'importance': importances
}).sort_values('importance', ascending=False)

print("Feature Importance (Random Forest):")
print(feature_importance_df)

# Visualize feature importance
plt.figure(figsize=(10, 6))
plt.barh(feature_importance_df['feature'], feature_importance_df['importance'])
plt.xlabel('Importance')
plt.title('Feature Importance - Random Forest')
plt.gca().invert_yaxis()
plt.tight_layout()
plt.show()

# Generate predictions for diagnostic plots
best_model = tuned_models['Random Forest']
y_pred = best_model.predict(X_test_engineered)

# Diagnostic plots
fig, axes = plt.subplots(2, 2, figsize=(12, 10))

# Predicted vs Actual
axes[0,0].scatter(y_test, y_pred, alpha=0.6)
axes[0,0].plot([y_test.min(), y_test.max()], [y_test.min(), y_test.max()], 'r--')
axes[0,0].set_xlabel('Actual')
axes[0,0].set_ylabel('Predicted')
axes[0,0].set_title('Predicted vs Actual')

# Residuals vs Predicted
residuals = y_test - y_pred
axes[0,1].scatter(y_pred, residuals, alpha=0.6)
axes[0,1].axhline(y=0, color='r', linestyle='--')
axes[0,1].set_xlabel('Predicted')
axes[0,1].set_ylabel('Residuals')
axes[0,1].set_title('Residuals vs Predicted')

# Residuals histogram
axes[1,0].hist(residuals, bins=30, alpha=0.7)
axes[1,0].set_xlabel('Residuals')
axes[1,0].set_ylabel('Frequency')
axes[1,0].set_title('Residual Distribution')

# Q-Q plot for normality
from scipy import stats
stats.probplot(residuals, dist="norm", plot=axes[1,1])
axes[1,1].set_title('Q-Q Plot')

plt.tight_layout()
plt.show()

# Calculate final metrics
final_mse = mean_squared_error(y_test, y_pred)
final_r2 = r2_score(y_test, y_pred)

print(f"\nFinal Model Performance:")
print(f"Test R²: {final_r2:.3f}")
print(f"Test RMSE: {np.sqrt(final_mse):.3f}")

import numpy as np
import pandas as pd
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, VotingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.model_selection import cross_val_score
from sklearn.datasets import make_classification
from sklearn.preprocessing import StandardScaler

# Create sample classification dataset
X, y = make_classification(n_samples=1000, n_features=20, n_informative=15, 
                          n_redundant=5, n_classes=2, random_state=42)

# Individual models
models = {
    'Logistic Regression': LogisticRegression(random_state=42),
    'Random Forest': RandomForestClassifier(n_estimators=100, random_state=42),
    'Gradient Boosting': GradientBoostingClassifier(random_state=42),
    'SVM': SVC(probability=True, random_state=42)
}

# Evaluate individual models
individual_scores = {}
for name, model in models.items():
    scores = cross_val_score(model, X, y, cv=5, scoring='accuracy')
    individual_scores[name] = scores.mean()
    print(f"{name}: {scores.mean():.3f} ± {scores.std():.3f}")

# Create ensemble
ensemble = VotingClassifier([
    ('lr', LogisticRegression(random_state=42)),
    ('rf', RandomForestClassifier(n_estimators=100, random_state=42)),
    ('gb', GradientBoostingClassifier(random_state=42))
], voting='soft')

# Evaluate ensemble
ensemble_scores = cross_val_score(ensemble, X, y, cv=5, scoring='accuracy')
print(f"\nEnsemble: {ensemble_scores.mean():.3f} ± {ensemble_scores.std():.3f}")

from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
import matplotlib.pyplot as plt

# Apply PCA for dimensionality reduction
pca = PCA()
X_pca = pca.fit_transform(X)

# Analyze explained variance
cumsum_variance = np.cumsum(pca.explained_variance_ratio_)
n_components_95 = np.argmax(cumsum_variance >= 0.95) + 1

print(f"Components needed for 95% variance: {n_components_95}")

# Visualize explained variance
plt.figure(figsize=(12, 5))

plt.subplot(1, 2, 1)
plt.plot(range(1, len(pca.explained_variance_ratio_) + 1), 
         pca.explained_variance_ratio_, 'bo-')
plt.xlabel('Component')
plt.ylabel('Explained Variance Ratio')
plt.title('Individual Component Variance')

plt.subplot(1, 2, 2)
plt.plot(range(1, len(cumsum_variance) + 1), cumsum_variance, 'ro-')
plt.axhline(y=0.95, color='k', linestyle='--', alpha=0.7)
plt.xlabel('Number of Components')
plt.ylabel('Cumulative Explained Variance')
plt.title('Cumulative Explained Variance')

plt.tight_layout()
plt.show()

# Use reduced dimensions for modeling
X_reduced = PCA(n_components=n_components_95).fit_transform(X)
print(f"Original dimensions: {X.shape[1]}")
print(f"Reduced dimensions: {X_reduced.shape[1]}")

from sklearn.datasets import make_classification
from sklearn.metrics import classification_report, confusion_matrix
from imblearn.over_sampling import SMOTE
from imblearn.under_sampling import RandomUnderSampler
from collections import Counter

# Create imbalanced dataset
X_imb, y_imb = make_classification(n_samples=1000, n_classes=2, weights=[0.9, 0.1],
                                  n_features=20, n_informative=15, random_state=42)

print("Original class distribution:")
print(Counter(y_imb))

# Resampling techniques
smote = SMOTE(random_state=42)
X_smote, y_smote = smote.fit_resample(X_imb, y_imb)

undersampler = RandomUnderSampler(random_state=42)
X_under, y_under = undersampler.fit_resample(X_imb, y_imb)

print("\nAfter SMOTE:")
print(Counter(y_smote))
print("\nAfter undersampling:")
print(Counter(y_under))

# Compare model performance on different datasets
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score, roc_auc_score

datasets = {
    'Original': (X_imb, y_imb),
    'SMOTE': (X_smote, y_smote),
    'Undersampled': (X_under, y_under)
}

for name, (X_data, y_data) in datasets.items():
    X_train, X_test, y_train, y_test = train_test_split(
        X_data, y_data, test_size=0.2, random_state=42, stratify=y_data
    )
    
    model = RandomForestClassifier(random_state=42)
    model.fit(X_train, y_train)
    y_pred = model.predict(X_test)
    
    f1 = f1_score(y_test, y_pred)
    auc = roc_auc_score(y_test, model.predict_proba(X_test)[:, 1])
    
    print(f"\n{name} Dataset:")
    print(f"F1 Score: {f1:.3f}")
    print(f"AUC-ROC: {auc:.3f}")

from sklearn.feature_selection import SelectKBest, f_classif, RFE
from sklearn.linear_model import LogisticRegression

# Univariate feature selection
selector_univariate = SelectKBest(score_func=f_classif, k=10)
X_univariate = selector_univariate.fit_transform(X, y)

# Recursive feature elimination
estimator = LogisticRegression(random_state=42)
selector_rfe = RFE(estimator, n_features_to_select=10)
X_rfe = selector_rfe.fit_transform(X, y)

# Compare feature selection methods
methods = {
    'All Features': X,
    'Univariate Selection': X_univariate,
    'RFE Selection': X_rfe
}

for name, X_selected in methods.items():
    scores = cross_val_score(LogisticRegression(random_state=42), 
                           X_selected, y, cv=5, scoring='accuracy')
    print(f"{name}: {scores.mean():.3f} ± {scores.std():.3f}")

# Note: This requires 'pip install shap'
try:
    import shap
    
    # Train a model for interpretation
    model = RandomForestClassifier(n_estimators=100, random_state=42)
    model.fit(X, y)
    
    # Create SHAP explainer
    explainer = shap.TreeExplainer(model)
    shap_values = explainer.shap_values(X[:100])  # Explain first 100 samples
    
    print("SHAP analysis completed - would show feature importance plots")
    print("Mean absolute SHAP values (feature importance):")
    
    # Calculate mean absolute SHAP values
    mean_shap = np.mean(np.abs(shap_values[1]), axis=0)  # For positive class
    feature_importance = pd.DataFrame({
        'feature': [f'feature_{i}' for i in range(len(mean_shap))],
        'importance': mean_shap
    }).sort_values('importance', ascending=False)
    
    print(feature_importance.head())
    
except ImportError:
    print("SHAP not installed - skipping interpretability analysis")

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from statsmodels.tsa.seasonal import seasonal_decompose
from statsmodels.tsa.stattools import adfuller
import warnings
warnings.filterwarnings('ignore')

# Create synthetic time series with known components
np.random.seed(42)
dates = pd.date_range('2020-01-01', periods=365*3, freq='D')

# Trend component
trend = np.linspace(100, 200, len(dates))

# Seasonal component (yearly and weekly patterns)
yearly_season = 20 * np.sin(2 * np.pi * np.arange(len(dates)) / 365.25)
weekly_season = 5 * np.sin(2 * np.pi * np.arange(len(dates)) / 7)

# Random noise
noise = np.random.normal(0, 10, len(dates))

# Combine components
ts_data = trend + yearly_season + weekly_season + noise

# Create time series DataFrame
ts_df = pd.DataFrame({
    'date': dates,
    'value': ts_data
})
ts_df.set_index('date', inplace=True)

# Decompose time series
decomposition = seasonal_decompose(ts_df['value'], model='additive', period=365)

# Plot decomposition
fig, axes = plt.subplots(4, 1, figsize=(12, 10))

ts_df['value'].plot(ax=axes[0], title='Original Time Series')
decomposition.trend.plot(ax=axes[1], title='Trend')
decomposition.seasonal.plot(ax=axes[2], title='Seasonal')
decomposition.resid.plot(ax=axes[3], title='Residual')

plt.tight_layout()
plt.show()

print("Time series components identified:")
print(f"Trend range: {decomposition.trend.min():.1f} to {decomposition.trend.max():.1f}")
print(f"Seasonal range: {decomposition.seasonal.min():.1f} to {decomposition.seasonal.max():.1f}")

def check_stationarity(timeseries, title):
    """Perform Augmented Dickey-Fuller test for stationarity."""
    
    # Perform ADF test
    result = adfuller(timeseries.dropna())
    
    print(f'\n{title}:')
    print(f'ADF Statistic: {result[0]:.6f}')
    print(f'p-value: {result[1]:.6f}')
    print(f'Critical Values:')
    for key, value in result[4].items():
        print(f'\t{key}: {value:.3f}')
    
    if result[1] <= 0.05:
        print("Series is stationary (reject null hypothesis)")
    else:
        print("Series is non-stationary (fail to reject null hypothesis)")
    
    return result[1] <= 0.05

# Test original series
is_stationary = check_stationarity(ts_df['value'], "Original Series")

# Apply differencing if non-stationary
if not is_stationary:
    ts_df['diff1'] = ts_df['value'].diff()
    ts_df['diff2'] = ts_df['diff1'].diff()
    
    # Test differenced series
    check_stationarity(ts_df['diff1'], "First Difference")
    check_stationarity(ts_df['diff2'], "Second Difference")

# Plot original vs differenced series
fig, axes = plt.subplots(3, 1, figsize=(12, 8))

ts_df['value'].plot(ax=axes[0], title='Original Series')
ts_df['diff1'].plot(ax=axes[1], title='First Difference')
ts_df['diff2'].plot(ax=axes[2], title='Second Difference')

plt.tight_layout()
plt.show()

from statsmodels.tsa.arima.model import ARIMA
from statsmodels.tsa.stattools import acf, pacf
from statsmodels.graphics.tsaplots import plot_acf, plot_pacf

# Determine ARIMA parameters using ACF and PACF plots
fig, axes = plt.subplots(2, 1, figsize=(12, 6))

# Use stationary series for parameter selection
stationary_series = ts_df['diff1'].dropna()

plot_acf(stationary_series, ax=axes[0], lags=40)
plot_pacf(stationary_series, ax=axes[1], lags=40)

plt.tight_layout()
plt.show()

# Fit ARIMA model
# Using (1,1,1) as starting point - adjust based on ACF/PACF plots
model = ARIMA(ts_df['value'], order=(1,1,1))
fitted_model = model.fit()

print("ARIMA Model Summary:")
print(fitted_model.summary())

# Generate forecasts
forecast_steps = 30
forecast = fitted_model.forecast(steps=forecast_steps)
forecast_ci = fitted_model.get_forecast(steps=forecast_steps).conf_int()

# Create forecast dates
last_date = ts_df.index[-1]
forecast_dates = pd.date_range(start=last_date + pd.Timedelta(days=1), 
                              periods=forecast_steps, freq='D')

# Plot results
plt.figure(figsize=(12, 6))

# Plot last 100 days of actual data
ts_df['value'][-100:].plot(label='Actual', color='blue')

# Plot forecast
plt.plot(forecast_dates, forecast, label='Forecast', color='red')
plt.fill_between(forecast_dates, 
                forecast_ci.iloc[:, 0], 
                forecast_ci.iloc[:, 1], 
                color='red', alpha=0.3, label='Confidence Interval')

plt.legend()
plt.title('ARIMA Forecast')
plt.show()

from statsmodels.tsa.statespace.sarimax import SARIMAX

# Fit SARIMA model with seasonal component
# (p,d,q) x (P,D,Q,s) where s is seasonal period
sarima_model = SARIMAX(ts_df['value'], 
                      order=(1,1,1), 
                      seasonal_order=(1,1,1,365))

fitted_sarima = sarima_model.fit(disp=False)

print("SARIMA Model Summary:")
print(fitted_sarima.summary())

# Generate SARIMA forecasts
sarima_forecast = fitted_sarima.forecast(steps=forecast_steps)
sarima_ci = fitted_sarima.get_forecast(steps=forecast_steps).conf_int()

# Compare ARIMA vs SARIMA forecasts
plt.figure(figsize=(12, 6))

ts_df['value'][-100:].plot(label='Actual', color='blue')
plt.plot(forecast_dates, forecast, label='ARIMA Forecast', color='red')
plt.plot(forecast_dates, sarima_forecast, label='SARIMA Forecast', color='green')

plt.legend()
plt.title('ARIMA vs SARIMA Forecasts')
plt.show()

from sklearn.metrics import mean_absolute_error, mean_squared_error

def time_series_cv_score(data, model_func, n_splits=5, test_size=30):
    """Time series cross-validation with expanding window."""
    
    scores = []
    total_size = len(data)
    
    for i in range(n_splits):
        # Expanding window: use more data for each iteration
        train_size = total_size - (n_splits - i) * test_size
        
        if train_size < 100:  # Minimum training size
            continue
            
        train_data = data[:train_size]
        test_data = data[train_size:train_size + test_size]
        
        if len(test_data) < test_size:
            test_data = data[train_size:]
        
        # Fit model and forecast
        model = model_func(train_data)
        forecast = model.forecast(steps=len(test_data))
        
        # Calculate error metrics
        mae = mean_absolute_error(test_data, forecast)
        rmse = np.sqrt(mean_squared_error(test_data, forecast))
        
        scores.append({'mae': mae, 'rmse': rmse})
    
    return scores

# Define model fitting functions
def fit_arima(data):
    return ARIMA(data, order=(1,1,1)).fit(disp=False)

def fit_sarima(data):
    return SARIMAX(data, order=(1,1,1), seasonal_order=(1,1,1,365)).fit(disp=False)

# Evaluate models
arima_scores = time_series_cv_score(ts_df['value'], fit_arima)
sarima_scores = time_series_cv_score(ts_df['value'], fit_sarima)

# Compare results
print("Cross-Validation Results:")
print("\nARIMA Model:")
arima_mae = np.mean([s['mae'] for s in arima_scores])
arima_rmse = np.mean([s['rmse'] for s in arima_scores])
print(f"Average MAE: {arima_mae:.3f}")
print(f"Average RMSE: {arima_rmse:.3f}")

print("\nSARIMA Model:")
sarima_mae = np.mean([s['mae'] for s in sarima_scores])
sarima_rmse = np.mean([s['rmse'] for s in sarima_scores])
print(f"Average MAE: {sarima_mae:.3f}")
print(f"Average RMSE: {sarima_rmse:.3f}")

# Prophet for automatic seasonality detection
try:
    from prophet import Prophet
    
    # Prepare data for Prophet
    prophet_df = ts_df.reset_index()
    prophet_df.columns = ['ds', 'y']
    
    # Fit Prophet model
    prophet_model = Prophet(yearly_seasonality=True, 
                           weekly_seasonality=True,
                           daily_seasonality=False)
    prophet_model.fit(prophet_df)
    
    # Generate future dates and forecast
    future = prophet_model.make_future_dataframe(periods=forecast_steps)
    prophet_forecast = prophet_model.predict(future)
    
    # Plot Prophet results
    fig = prophet_model.plot(prophet_forecast)
    plt.title('Prophet Forecast')
    plt.show()
    
    # Plot components
    fig = prophet_model.plot_components(prophet_forecast)
    plt.show()
    
    print("Prophet model fitted successfully")
    
except ImportError:
    print("Prophet not installed - skipping Prophet analysis")

import requests
import pandas as pd
import json
import time
from typing import Dict, List, Optional

class APIClient:
    """Generic API client with rate limiting and error handling."""
    
    def __init__(self, base_url: str, api_key: Optional[str] = None, 
                 rate_limit: float = 1.0):
        self.base_url = base_url.rstrip('/')
        self.api_key = api_key
        self.rate_limit = rate_limit
        self.last_request_time = 0
        
    def _wait_for_rate_limit(self):
        """Ensure we don't exceed rate limits."""
        elapsed = time.time() - self.last_request_time
        if elapsed < self.rate_limit:
            time.sleep(self.rate_limit - elapsed)
        self.last_request_time = time.time()
    
    def make_request(self, endpoint: str, params: Dict = None) -> Dict:
        """Make API request with error handling."""
        self._wait_for_rate_limit()
        
        url = f"{self.base_url}/{endpoint.lstrip('/')}"
        headers = {}
        
        if self.api_key:
            headers['Authorization'] = f'Bearer {self.api_key}'
        
        try:
            response = requests.get(url, params=params, headers=headers)
            response.raise_for_status()
            return response.json()
            
        except requests.exceptions.RequestException as e:
            print(f"API request failed: {e}")
            return {}

# Example: Weather data API (using a hypothetical service)
def fetch_weather_data(cities: List[str]) -> pd.DataFrame:
    """Fetch weather data for multiple cities."""
    
    # This is a mock example - replace with actual API
    weather_data = []
    
    for city in cities:
        # Simulate API response
        mock_data = {
            'city': city,
            'temperature': 20 + hash(city) % 20,  # Mock temperature
            'humidity': 40 + hash(city) % 40,     # Mock humidity
            'timestamp': pd.Timestamp.now()
        }
        weather_data.append(mock_data)
        
        # Simulate rate limiting
        time.sleep(0.1)
    
    return pd.DataFrame(weather_data)

# Fetch data for multiple cities
cities = ['New York', 'London', 'Tokyo', 'Sydney']
weather_df = fetch_weather_data(cities)
print("Weather Data:")
print(weather_df)

import requests
from bs4 import BeautifulSoup
import pandas as pd
import time
from urllib.parse import urljoin, urlparse

class WebScraper:
    """Web scraper with polite crawling practices."""
    
    def __init__(self, delay: float = 1.0):
        self.delay = delay
        self.session = requests.Session()
        # Set a user agent to identify your scraper
        self.session.headers.update({
            'User-Agent': 'Mozilla/5.0 (compatible; DataScienceScraper/1.0)'
        })
    
    def get_page(self, url: str) -> Optional[BeautifulSoup]:
        """Fetch and parse a web page."""
        try:
            time.sleep(self.delay)  # Be polite
            response = self.session.get(url)
            response.raise_for_status()
            
            return BeautifulSoup(response.content, 'html.parser')
            
        except requests.exceptions.RequestException as e:
            print(f"Failed to fetch {url}: {e}")
            return None
    
    def extract_table_data(self, soup: BeautifulSoup, 
                          table_selector: str) -> pd.DataFrame:
        """Extract data from HTML tables."""
        table = soup.select_one(table_selector)
        if not table:
            return pd.DataFrame()
        
        # Extract headers
        headers = []
        header_row = table.select_one('thead tr, tr:first-child')
        if header_row:
            headers = [th.get_text(strip=True) for th in 
                      header_row.select('th, td')]
        
        # Extract data rows
        rows = []
        data_rows = table.select('tbody tr, tr')[1:] if headers else table.select('tr')
        
        for row in data_rows:
            cells = [td.get_text(strip=True) for td in row.select('td, th')]
            if cells:  # Skip empty rows
                rows.append(cells)
        
        # Create DataFrame
        if headers and rows:
            return pd.DataFrame(rows, columns=headers)
        elif rows:
            return pd.DataFrame(rows)
        else:
            return pd.DataFrame()

# Example: Scraping a hypothetical data table
def scrape_sample_data():
    """Demonstrate web scraping techniques."""
    
    # Create mock HTML content for demonstration
    mock_html = """
    <html>
    <body>
        <table id="data-table">
            <thead>
                <tr>
                    <th>Product</th>
                    <th>Price</th>
                    <th>Rating</th>
                </tr>
            </thead>
            <tbody>
                <tr>
                    <td>Laptop A</td>
                    <td>$999</td>
                    <td>4.5</td>
                </tr>
                <tr>
                    <td>Laptop B</td>
                    <td>$1299</td>
                    <td>4.2</td>
                </tr>
            </tbody>
        </table>
    </body>
    </html>
    """
    
    soup = BeautifulSoup(mock_html, 'html.parser')
    scraper = WebScraper()
    
    # Extract table data
    df = scraper.extract_table_data(soup, '#data-table')
    print("Scraped Data:")
    print(df)
    
    return df

scraped_df = scrape_sample_data()

# Note: Requires 'pip install selenium' and appropriate webdriver
try:
    from selenium import webdriver
    from selenium.webdriver.common.by import By
    from selenium.webdriver.support.ui import WebDriverWait
    from selenium.webdriver.support import expected_conditions as EC
    from selenium.webdriver.chrome.options import Options
    
    def setup_driver(headless: bool = True):
        """Set up Chrome driver with options."""
        options = Options()
        if headless:
            options.add_argument('--headless')
        options.add_argument('--no-sandbox')
        options.add_argument('--disable-dev-shm-usage')
        
        # Note: Requires chromedriver in PATH
        return webdriver.Chrome(options=options)
    
    def scrape_dynamic_content(url: str) -> List[Dict]:
        """Scrape content that loads dynamically."""
        driver = setup_driver()
        
        try:
            driver.get(url)
            
            # Wait for content to load
            wait = WebDriverWait(driver, 10)
            
            # Example: Wait for specific elements to appear
            elements = wait.until(
                EC.presence_of_all_elements_located((By.CLASS_NAME, "data-item"))
            )
            
            # Extract data from elements
            data = []
            for element in elements:
                item_data = {
                    'text': element.text,
                    'href': element.get_attribute('href'),
                    'class': element.get_attribute('class')
                }
                data.append(item_data)
            
            return data
            
        finally:
            driver.quit()
    
    print("Selenium setup available for dynamic content scraping")
    
except ImportError:
    print("Selenium not installed - skipping dynamic content scraping")

import sqlite3
from datetime import datetime, timedelta
import logging

class DataPipeline:
    """Pipeline for collecting, processing, and storing web data."""
    
    def __init__(self, db_path: str = 'web_data.db'):
        self.db_path = db_path
        self.setup_database()
        self.setup_logging()
    
    def setup_database(self):
        """Initialize database tables."""
        with sqlite3.connect(self.db_path) as conn:
            conn.execute('''
                CREATE TABLE IF NOT EXISTS scraped_data (
                    id INTEGER PRIMARY KEY AUTOINCREMENT,
                    source TEXT NOT NULL,
                    data_type TEXT NOT NULL,
                    content TEXT NOT NULL,
                    scraped_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                    processed BOOLEAN DEFAULT FALSE
                )
            ''')
    
    def setup_logging(self):
        """Configure logging for pipeline monitoring."""
        logging.basicConfig(
            level=logging.INFO,
            format='%(asctime)s - %(levelname)s - %(message)s',
            handlers=[
                logging.FileHandler('scraping.log'),
                logging.StreamHandler()
            ]
        )
        self.logger = logging.getLogger(__name__)
    
    def collect_data(self, sources: List[Dict]):
        """Collect data from multiple sources."""
        for source in sources:
            try:
                self.logger.info(f"Collecting data from {source['name']}")
                
                if source['type'] == 'api':
                    data = self.collect_api_data(source)
                elif source['type'] == 'scrape':
                    data = self.collect_scraped_data(source)
                else:
                    self.logger.warning(f"Unknown source type: {source['type']}")
                    continue
                
                self.store_data(source['name'], source['type'], data)
                
            except Exception as e:
                self.logger.error(f"Failed to collect from {source['name']}: {e}")
    
    def collect_api_data(self, source: Dict) -> str:
        """Collect data from API source."""
        # Mock API data collection
        return json.dumps({
            'timestamp': datetime.now().isoformat(),
            'source': source['name'],
            'data': f"Mock API data from {source['url']}"
        })
    
    def collect_scraped_data(self, source: Dict) -> str:
        """Collect data from web scraping."""
        # Mock scraping data collection
        return json.dumps({
            'timestamp': datetime.now().isoformat(),
            'source': source['name'],
            'data': f"Mock scraped data from {source['url']}"
        })
    
    def store_data(self, source: str, data_type: str, content: str):
        """Store collected data in database."""
        with sqlite3.connect(self.db_path) as conn:
            conn.execute(
                'INSERT INTO scraped_data (source, data_type, content) VALUES (?, ?, ?)',
                (source, data_type, content)
            )
    
    def process_stored_data(self) -> pd.DataFrame:
        """Process and clean stored data."""
        with sqlite3.connect(self.db_path) as conn:
            df = pd.read_sql_query(
                'SELECT * FROM scraped_data WHERE processed = FALSE',
                conn
            )
        
        # Process data (clean, transform, validate)
        processed_data = []
        for _, row in df.iterrows():
            try:
                content = json.loads(row['content'])
                processed_item = {
                    'id': row['id'],
                    'source': row['source'],
                    'timestamp': content['timestamp'],
                    'processed_at': datetime.now().isoformat()
                }
                processed_data.append(processed_item)
                
            except json.JSONDecodeError:
                self.logger.warning(f"Invalid JSON in row {row['id']}")
        
        # Mark as processed
        if processed_data:
            with sqlite3.connect(self.db_path) as conn:
                ids = [item['id'] for item in processed_data]
                placeholders = ','.join(['?'] * len(ids))
                conn.execute(
                    f'UPDATE scraped_data SET processed = TRUE WHERE id IN ({placeholders})',
                    ids
                )
        
        return pd.DataFrame(processed_data)

# Example usage
pipeline = DataPipeline()

# Define data sources
sources = [
    {
        'name': 'weather_api',
        'type': 'api',
        'url': 'https://api.weather.com/v1/current'
    },
    {
        'name': 'news_site',
        'type': 'scrape',
        'url': 'https://example-news.com/headlines'
    }
]

# Collect and process data
pipeline.collect_data(sources)
processed_df = pipeline.process_stored_data()

print("Processed data:")
print(processed_df)

import pandas as pd
import sqlite3
import sqlalchemy as sa
from sqlalchemy import create_engine, text
import numpy as np
from contextlib import contextmanager

# SQLite for local development and examples
def create_sample_database():
    """Create sample database with realistic data."""
    
    # Create in-memory SQLite database
    engine = create_engine('sqlite:///sample_data.db')
    
    # Generate sample sales data
    np.random.seed(42)
    n_customers = 1000
    n_products = 50
    n_orders = 5000
    
    # Customers table
    customers = pd.DataFrame({
        'customer_id': range(1, n_customers + 1),
        'name': [f'Customer_{i}' for i in range(1, n_customers + 1)],
        'email': [f'customer_{i}@email.com' for i in range(1, n_customers + 1)],
        'city': np.random.choice(['New York', 'London', 'Tokyo', 'Paris'], n_customers),
        'signup_date': pd.date_range('2020-01-01', periods=n_customers, freq='D')[:n_customers]
    })
    
    # Products table
    products = pd.DataFrame({
        'product_id': range(1, n_products + 1),
        'product_name': [f'Product_{i}' for i in range(1, n_products + 1)],
        'category': np.random.choice(['Electronics', 'Clothing', 'Books'], n_products),
        'price': np.random.uniform(10, 500, n_products).round(2)
    })
    
    # Orders table
    orders = pd.DataFrame({
        'order_id': range(1, n_orders + 1),
        'customer_id': np.random.randint(1, n_customers + 1, n_orders),
        'product_id': np.random.randint(1, n_products + 1, n_orders),
        'quantity': np.random.randint(1, 5, n_orders),
        'order_date': pd.date_range('2020-01-01', '2023-12-31', periods=n_orders)
    })
    
    # Write to database
    customers.to_sql('customers', engine, if_exists='replace', index=False)
    products.to_sql('products', engine, if_exists='replace', index=False)
    orders.to_sql('orders', engine, if_exists='replace', index=False)
    
    return engine

# Create sample database
engine = create_sample_database()
print("Sample database created with tables: customers, products, orders")

# Test connection
with engine.connect() as conn:
    result = conn.execute(text("SELECT COUNT(*) as count FROM customers"))
    customer_count = result.fetchone()[0]
    print(f"Database contains {customer_count} customers")

# Analytical SQL queries
analytical_queries = {
    'customer_summary': """
        SELECT 
            c.city,
            COUNT(DISTINCT c.customer_id) as customer_count,
            COUNT(o.order_id) as total_orders,
            SUM(o.quantity * p.price) as total_revenue,
            AVG(o.quantity * p.price) as avg_order_value
        FROM customers c
        LEFT JOIN orders o ON c.customer_id = o.customer_id
        LEFT JOIN products p ON o.product_id = p.product_id
        GROUP BY c.city
        ORDER BY total_revenue DESC
    """,
    
    'monthly_trends': """
        SELECT 
            strftime('%Y-%m', o.order_date) as month,
            p.category,
            SUM(o.quantity * p.price) as revenue,
            COUNT(o.order_id) as order_count,
            AVG(o.quantity * p.price) as avg_order_value
        FROM orders o
        JOIN products p ON o.product_id = p.product_id
        WHERE o.order_date >= '2023-01-01'
        GROUP BY month, p.category
        ORDER BY month, revenue DESC
    """,
    
    'customer_cohorts': """
        SELECT 
            strftime('%Y-%m', c.signup_date) as cohort_month,
            COUNT(DISTINCT c.customer_id) as cohort_size,
            COUNT(DISTINCT CASE WHEN o.order_date IS NOT NULL THEN c.customer_id END) as active_customers,
            ROUND(
                COUNT(DISTINCT CASE WHEN o.order_date IS NOT NULL THEN c.customer_id END) * 100.0 / 
                COUNT(DISTINCT c.customer_id), 2
            ) as activation_rate
        FROM customers c
        LEFT JOIN orders o ON c.customer_id = o.customer_id 
            AND o.order_date BETWEEN c.signup_date AND date(c.signup_date, '+30 days')
        GROUP BY cohort_month
        ORDER BY cohort_month
    """
}

# Execute analytical queries
results = {}
for query_name, query in analytical_queries.items():
    df = pd.read_sql_query(query, engine)
    results[query_name] = df
    
    print(f"\n{query_name.replace('_', ' ').title()}:")
    print(df.head())
    print(f"Shape: {df.shape}")

# Advanced pandas-SQL integration
class DatabaseAnalyzer:
    """Class for database-driven analysis with pandas integration."""
    
    def __init__(self, engine):
        self.engine = engine
    
    def query_to_dataframe(self, query: str, params: dict = None) -> pd.DataFrame:
        """Execute SQL query and return pandas DataFrame."""
        return pd.read_sql_query(query, self.engine, params=params)
    
    def chunked_query(self, query: str, chunksize: int = 10000) -> pd.DataFrame:
        """Process large queries in chunks to manage memory."""
        chunks = []
        for chunk in pd.read_sql_query(query, self.engine, chunksize=chunksize):
            # Process each chunk (e.g., apply transformations)
            processed_chunk = self.process_chunk(chunk)
            chunks.append(processed_chunk)
        
        return pd.concat(chunks, ignore_index=True)
    
    def process_chunk(self, chunk: pd.DataFrame) -> pd.DataFrame:
        """Process individual chunks of data."""
        # Example processing: calculate derived metrics
        if 'order_date' in chunk.columns:
            chunk['order_date'] = pd.to_datetime(chunk['order_date'])
            chunk['day_of_week'] = chunk['order_date'].dt.day_name()
            chunk['month'] = chunk['order_date'].dt.month
        
        return chunk
    
    def get_customer_analysis(self, city: str = None) -> pd.DataFrame:
        """Get customer analysis with optional city filter."""
        
        base_query = """
            SELECT 
                c.customer_id,
                c.name,
                c.city,
                c.signup_date,
                COUNT(o.order_id) as total_orders,
                SUM(o.quantity * p.price) as total_spent,
                MAX(o.order_date) as last_order_date
            FROM customers c
            LEFT JOIN orders o ON c.customer_id = o.customer_id
            LEFT JOIN products p ON o.product_id = p.product_id
        """
        
        if city:
            query = base_query + " WHERE c.city = %(city)s"
            params = {'city': city}
        else:
            query = base_query
            params = None
        
        query += """
            GROUP BY c.customer_id, c.name, c.city, c.signup_date
            ORDER BY total_spent DESC
        """
        
        df = self.query_to_dataframe(query, params)
        
        # Add pandas-based calculations
        df['signup_date'] = pd.to_datetime(df['signup_date'])
        df['last_order_date'] = pd.to_datetime(df['last_order_date'])
        df['days_since_signup'] = (pd.Timestamp.now() - df['signup_date']).dt.days
        df['days_since_last_order'] = (pd.Timestamp.now() - df['last_order_date']).dt.days
        
        # Customer segmentation
        df['customer_segment'] = pd.cut(
            df['total_spent'].fillna(0),
            bins=[0, 100, 500, 1000, float('inf')],
            labels=['Low', 'Medium', 'High', 'VIP']
        )
        
        return df

# Use the analyzer
analyzer = DatabaseAnalyzer(engine)

# Get customer analysis for specific city
tokyo_customers = analyzer.get_customer_analysis(city='Tokyo')
print("Tokyo Customer Analysis:")
print(tokyo_customers.head())

# Analyze customer segments
segment_analysis = tokyo_customers.groupby('customer_segment').agg({
    'customer_id': 'count',
    'total_spent': ['mean', 'sum'],
    'total_orders': 'mean',
    'days_since_last_order': 'mean'
}).round(2)

print("\nCustomer Segment Analysis:")
print(segment_analysis)