SRE Practices for Serverless Architectures: Ensuring Reliability Without Servers

// AWS Lambda example with structured logging
const winston = require('winston');

// Configure logger
const logger = winston.createLogger({
  level: 'info',
  format: winston.format.json(),
  defaultMeta: { service: 'payment-processor' },
  transports: [
    new winston.transports.Console()
  ]
});

// Lambda handler with structured logging
exports.handler = async (event, context) => {
  // Add request context to all logs
  const requestContext = {
    awsRequestId: context.awsRequestId,
    functionVersion: context.functionVersion,
    functionName: context.functionName,
    memoryLimitInMB: context.memoryLimitInMB,
    logGroupName: context.logGroupName,
    logStreamName: context.logStreamName
  };
  
  logger.defaultMeta = { ...logger.defaultMeta, ...requestContext };
  
  try {
    // Log the incoming event with sensitive data redacted
    const sanitizedEvent = sanitizeEvent(event);
    logger.info('Processing payment request', { event: sanitizedEvent });
    
    // Business logic
    const paymentResult = await processPayment(event);
    
    // Log the result
    logger.info('Payment processed successfully', { 
      paymentId: paymentResult.id,
      amount: paymentResult.amount,
      processingTimeMs: paymentResult.processingTime
    });
    
    return {
      statusCode: 200,
      body: JSON.stringify({
        paymentId: paymentResult.id,
        status: 'success'
      })
    };
  } catch (error) {
    // Log the error with context
    logger.error('Payment processing failed', {
      errorMessage: error.message,
      errorName: error.name,
      errorStack: error.stack,
      errorCode: error.code || 'UNKNOWN_ERROR'
    });
    
    // Return appropriate error response
    return {
      statusCode: error.statusCode || 500,
      body: JSON.stringify({
        error: error.message,
        requestId: context.awsRequestId
      })
    };
  }
};

# AWS Lambda with OpenTelemetry tracing
import json
import os
from opentelemetry import trace
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.resources import Resource
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
from opentelemetry.instrumentation.botocore import BotocoreInstrumentor
import boto3

# Set up OpenTelemetry
resource = Resource.create({
    "service.name": "order-service",
    "service.version": os.environ.get("SERVICE_VERSION", "unknown"),
    "deployment.environment": os.environ.get("ENVIRONMENT", "dev")
})

trace.set_tracer_provider(TracerProvider(resource=resource))
tracer = trace.get_tracer(__name__)

# Configure exporter
otlp_exporter = OTLPSpanExporter(
    endpoint=os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector:4317")
)
span_processor = BatchSpanProcessor(otlp_exporter)
trace.get_tracer_provider().add_span_processor(span_processor)

# Instrument AWS SDK
BotocoreInstrumentor().instrument()

# Initialize AWS clients
dynamodb = boto3.resource('dynamodb')
sns = boto3.client('sns')
orders_table = dynamodb.Table(os.environ['ORDERS_TABLE'])
notification_topic = os.environ['NOTIFICATION_TOPIC']

def lambda_handler(event, context):
    # Extract trace context from event headers if present
    propagator = TraceContextTextMapPropagator()
    headers = event.get('headers', {})
    ctx = propagator.extract(headers)
    
    with tracer.start_as_current_span("process_order", context=ctx) as span:
        # Add event information to span
        span.set_attribute("function.name", context.function_name)
        span.set_attribute("function.version", context.function_version)
        span.set_attribute("cold_start", context.aws_request_id == "__first_request__")
        
        try:
            # Parse order data
            body = json.loads(event.get('body', '{}'))
            order_id = body.get('orderId')
            
            span.set_attribute("order.id", order_id)
            
            with tracer.start_as_current_span("fetch_order_details"):
                # Get order details from DynamoDB
                order = orders_table.get_item(Key={'orderId': order_id})
                
                if 'Item' not in order:
                    span.set_attribute("error", True)
                    span.set_attribute("error.message", f"Order {order_id} not found")
                    return {
                        'statusCode': 404,
                        'body': json.dumps({'error': f"Order {order_id} not found"})
                    }
            
            # Process order and return response
            return {
                'statusCode': 200,
                'body': json.dumps({
                    'orderId': order_id,
                    'status': 'PROCESSED'
                })
            }
            
        except Exception as e:
            span.record_exception(e)
            span.set_attribute("error", True)
            span.set_attribute("error.message", str(e))
            
            return {
                'statusCode': 500,
                'body': json.dumps({
                    'error': str(e),
                    'requestId': context.aws_request_id
                })
            }

// Azure Functions with custom metrics
const { CosmosClient } = require('@azure/cosmos');
const { AzureMonitorMetricExporter } = require('@azure/monitor-opentelemetry-exporter');
const { MeterProvider } = require('@opentelemetry/sdk-metrics');
const { Resource } = require('@opentelemetry/resources');
const { SemanticResourceAttributes } = require('@opentelemetry/semantic-conventions');

// Configure metrics
const resource = new Resource({
  [SemanticResourceAttributes.SERVICE_NAME]: 'inventory-service',
  [SemanticResourceAttributes.SERVICE_VERSION]: '1.0.0',
  [SemanticResourceAttributes.DEPLOYMENT_ENVIRONMENT]: process.env.ENVIRONMENT || 'development'
});

const metricExporter = new AzureMonitorMetricExporter({
  connectionString: process.env.APPLICATIONINSIGHTS_CONNECTION_STRING
});

const meterProvider = new MeterProvider({
  resource: resource,
  exporter: metricExporter,
  interval: 1000
});

const meter = meterProvider.getMeter('inventory-operations');

// Define metrics
const functionInvocations = meter.createCounter('function.invocations', {
  description: 'Number of function invocations'
});

const functionDuration = meter.createHistogram('function.duration', {
  description: 'Function execution duration in milliseconds',
  unit: 'ms'
});

// Azure Function implementation
module.exports = async function(context, req) {
  const startTime = Date.now();
  
  // Record function invocation
  functionInvocations.add(1, {
    operation: 'getInventory',
    region: process.env.REGION || 'unknown'
  });
  
  try {
    // Function implementation
    // ...
    
    context.res = {
      status: 200,
      body: { /* result */ }
    };
  } catch (error) {
    context.log.error("Error in function", error);
    context.res = {
      status: 500,
      body: { error: "Internal server error" }
    };
  } finally {
    // Record function duration
    const duration = Date.now() - startTime;
    functionDuration.record(duration, {
      operation: 'getInventory',
      status: context.res.status
    });
  }
};

// Circuit breaker implementation for serverless functions
import { DynamoDBClient, PutItemCommand } from '@aws-sdk/client-dynamodb';
import { marshall } from '@aws-sdk/util-dynamodb';

interface CircuitBreakerState {
  serviceName: string;
  status: 'CLOSED' | 'OPEN' | 'HALF_OPEN';
  failureCount: number;
  lastFailureTime: number;
  nextAttemptTime: number;
}

class CircuitBreaker {
  private dynamoClient: DynamoDBClient;
  private tableName: string;
  private serviceName: string;
  private failureThreshold: number;
  private resetTimeout: number;
  
  constructor(config: {
    region: string;
    tableName: string;
    serviceName: string;
    failureThreshold?: number;
    resetTimeout?: number;
  }) {
    this.dynamoClient = new DynamoDBClient({ region: config.region });
    this.tableName = config.tableName;
    this.serviceName = config.serviceName;
    this.failureThreshold = config.failureThreshold || 5;
    this.resetTimeout = config.resetTimeout || 60000; // 1 minute
  }
  
  async executeWithCircuitBreaker<T>(operation: () => Promise<T>): Promise<T> {
    const state = await this.getState();
    const now = Date.now();
    
    // Check if circuit is open
    if (state.status === 'OPEN') {
      if (now < state.nextAttemptTime) {
        throw new Error(`Circuit is OPEN for service ${this.serviceName}`);
      } else {
        // Move to half-open state
        state.status = 'HALF_OPEN';
        await this.saveState(state);
      }
    }
    
    try {
      // Execute the operation
      const result = await operation();
      
      // If successful and in HALF_OPEN state, close the circuit
      if (state.status === 'HALF_OPEN') {
        state.status = 'CLOSED';
        state.failureCount = 0;
        await this.saveState(state);
      }
      
      return result;
    } catch (error) {
      // Handle failure
      state.failureCount++;
      state.lastFailureTime = now;
      
      // If failure threshold reached, open the circuit
      if (state.failureCount >= this.failureThreshold) {
        state.status = 'OPEN';
        state.nextAttemptTime = now + this.resetTimeout;
      }
      
      await this.saveState(state);
      throw error;
    }
  }
  
  // Implementation of getState and saveState methods omitted for brevity
}

# Retry with exponential backoff in AWS Lambda
import json
import time
import random
import boto3
from botocore.exceptions import ClientError

# Initialize DynamoDB client
dynamodb = boto3.resource('dynamodb')
table = dynamodb.Table('Orders')

def retry_with_exponential_backoff(func, max_retries=3, base_delay=100, max_delay=5000):
    """
    Execute a function with exponential backoff retry logic
    
    Args:
        func: Function to execute
        max_retries: Maximum number of retries
        base_delay: Base delay in milliseconds
        max_delay: Maximum delay in milliseconds
        
    Returns:
        Result of the function execution
    """
    retries = 0
    while True:
        try:
            return func()
        except ClientError as e:
            # Only retry on throttling errors or transient failures
            if e.response['Error']['Code'] not in ['ProvisionedThroughputExceededException', 
                                                  'ThrottlingException',
                                                  'InternalServerError']:
                raise
            
            if retries >= max_retries:
                raise
            
            # Calculate delay with exponential backoff and jitter
            delay = min(max_delay, base_delay * (2 ** retries))
            jitter = random.uniform(0, 0.1 * delay)  # 10% jitter
            sleep_time = (delay + jitter) / 1000.0  # Convert to seconds
            
            print(f"Request throttled, retrying in {sleep_time:.2f} seconds (retry {retries + 1}/{max_retries})")
            time.sleep(sleep_time)
            retries += 1

def lambda_handler(event, context):
    order_id = event.get('orderId')
    
    if not order_id:
        return {
            'statusCode': 400,
            'body': json.dumps({'error': 'Order ID is required'})
        }
    
    try:
        # Use retry logic for DynamoDB operations
        def get_order():
            return table.get_item(Key={'orderId': order_id})
        
        response = retry_with_exponential_backoff(get_order)
        
        if 'Item' not in response:
            return {
                'statusCode': 404,
                'body': json.dumps({'error': f'Order {order_id} not found'})
            }
        
        return {
            'statusCode': 200,
            'body': json.dumps(response['Item'])
        }
    except Exception as e:
        print(f"Error processing request: {str(e)}")
        return {
            'statusCode': 500,
            'body': json.dumps({
                'error': 'Internal server error',
                'requestId': context.aws_request_id
            })
        }

# AWS SAM template with DLQ configuration
AWSTemplateFormatVersion: '2010-09-09'
Transform: AWS::Serverless-2016-10-31
Resources:
  ProcessOrderFunction:
    Type: AWS::Serverless::Function
    Properties:
      CodeUri: ./src/
      Handler: process-order.handler
      Runtime: nodejs14.x
      MemorySize: 256
      Timeout: 30
      Environment:
        Variables:
          ORDER_TABLE: !Ref OrderTable
          PAYMENT_SERVICE_URL: https://payment.example.com/api
      Policies:
        - DynamoDBCrudPolicy:
            TableName: !Ref OrderTable
      Events:
        OrderCreatedEvent:
          Type: SQS
          Properties:
            Queue: !GetAtt OrderQueue.Arn
            BatchSize: 10
            MaximumBatchingWindowInSeconds: 5
            # Configure DLQ for the event source mapping
            DestinationConfig:
              OnFailure:
                Destination: !GetAtt OrderDLQ.Arn
      # Configure DLQ for the function itself
      DeadLetterQueue:
        Type: SQS
        TargetArn: !GetAtt FunctionDLQ.Arn

  # Queue for new orders
  OrderQueue:
    Type: AWS::SQS::Queue
    Properties:
      VisibilityTimeout: 60
      RedrivePolicy:
        deadLetterTargetArn: !GetAtt OrderDLQ.Arn
        maxReceiveCount: 3

  # DLQ for the order queue
  OrderDLQ:
    Type: AWS::SQS::Queue
    Properties:
      MessageRetentionPeriod: 1209600  # 14 days

// Cold start optimization in AWS Lambda
// Lambda handler file: index.js

// Global scope - executed once per container initialization
// Import dependencies outside the handler
const AWS = require('aws-sdk');
const dynamoDB = new AWS.DynamoDB.DocumentClient();
const sns = new AWS.SNS();

// Initialize expensive resources outside the handler
const connectionPool = initializeConnectionPool();
const configCache = loadConfiguration();

// Warm-up function data
let isWarmedUp = false;
const WARM_UP_EVENT = 'serverless-plugin-warmup';

// Handler function
exports.handler = async (event, context) => {
  // Check if this is a warm-up event
  if (event.source === WARM_UP_EVENT) {
    console.log('WarmUp - Lambda is warm!');
    return 'Lambda is warm!';
  }
  
  // Record cold start
  const isColStart = !isWarmedUp;
  if (!isWarmedUp) {
    console.log('Cold start detected');
    isWarmedUp = true;
  }
  
  // Main function logic
  try {
    // Use pre-initialized resources
    const result = await processEvent(event, connectionPool, configCache);
    
    // Add cold start information to the response
    return {
      ...result,
      coldStart: isColStart,
      initializationTime: isColStart ? process.uptime() * 1000 : 0
    };
  } catch (error) {
    console.error('Error processing event', error);
    throw error;
  }
};

# Prometheus recording rules for serverless SLOs
groups:
- name: serverless_slos
  rules:
  # Availability SLO - 99.9% of function invocations should be successful
  - record: serverless:availability:ratio
    expr: sum(rate(aws_lambda_invocations_total{function_name=~".*",status="success"}[1h])) / sum(rate(aws_lambda_invocations_total{function_name=~".*"}[1h]))
  
  # Latency SLO - 99% of function executions should complete within 1 second
  - record: serverless:latency:ratio
    expr: sum(rate(aws_lambda_duration_seconds_bucket{function_name=~".*",le="1"}[1h])) / sum(rate(aws_lambda_duration_seconds_count{function_name=~".*"}[1h]))
  
  # Cold Start SLO - 95% of function invocations should not experience cold starts
  - record: serverless:cold_start:ratio
    expr: (sum(rate(aws_lambda_invocations_total{function_name=~".*"}[1h])) - sum(rate(aws_lambda_cold_starts_total{function_name=~".*"}[1h]))) / sum(rate(aws_lambda_invocations_total{function_name=~".*"}[1h]))
  
  # Error Budget Burn Rate
  - record: serverless:error_budget:burn_rate
    expr: (1 - serverless:availability:ratio) / (1 - 0.999) # 99.9% availability target

// Local testing with Jest and serverless-offline
const { handler } = require('../src/functions/process-order');
const AWS = require('aws-sdk-mock');
const event = require('./fixtures/order-event.json');

describe('Process Order Function', () => {
  beforeAll(() => {
    // Mock AWS services
    AWS.mock('DynamoDB.DocumentClient', 'get', (params, callback) => {
      if (params.Key.orderId === 'valid-order-id') {
        callback(null, {
          Item: {
            orderId: 'valid-order-id',
            customerId: 'customer-123',
            amount: 99.99,
            status: 'PENDING'
          }
        });
      } else {
        callback(null, {});
      }
    });
    
    AWS.mock('DynamoDB.DocumentClient', 'update', (params, callback) => {
      callback(null, { Attributes: { status: 'PROCESSING' } });
    });
    
    AWS.mock('SNS', 'publish', (params, callback) => {
      callback(null, { MessageId: 'mock-message-id' });
    });
  });
  
  afterAll(() => {
    AWS.restore();
  });
  
  test('Successfully processes a valid order', async () => {
    // Arrange
    const validEvent = {
      ...event,
      pathParameters: {
        orderId: 'valid-order-id'
      }
    };
    
    const context = {
      awsRequestId: 'mock-request-id',
      functionName: 'process-order'
    };
    
    // Act
    const result = await handler(validEvent, context);
    
    // Assert
    expect(result.statusCode).toBe(200);
    const body = JSON.parse(result.body);
    expect(body.status).toBe('PROCESSING');
    expect(body.orderId).toBe('valid-order-id');
  });
  
  test('Returns 404 for non-existent order', async () => {
    // Arrange
    const invalidEvent = {
      ...event,
      pathParameters: {
        orderId: 'non-existent-order'
      }
    };
    
    const context = {
      awsRequestId: 'mock-request-id',
      functionName: 'process-order'
    };
    
    // Act
    const result = await handler(invalidEvent, context);
    
    // Assert
    expect(result.statusCode).toBe(404);
    const body = JSON.parse(result.body);
    expect(body.error).toContain('not found');
  });
});