Real-World Docker Compose Projects and Implementation

This final section demonstrates complete production-ready implementations, combining all concepts learned throughout this guide into real-world systems with full CI/CD integration and operational excellence.

Project 1: Enterprise SaaS Platform

Complete Multi-Tenant Architecture

# docker-compose.prod.yml
version: '3.8'

services:
  # Load Balancer with SSL Termination
  traefik:
    image: traefik:v2.9
    ports:
      - "80:80"
      - "443:443"
      - "8080:8080"
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock:ro
      - ./traefik/traefik.yml:/etc/traefik/traefik.yml:ro
      - ./traefik/dynamic:/etc/traefik/dynamic:ro
      - traefik_certs:/certs
    networks:
      - frontend
      - monitoring
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.dashboard.rule=Host(`traefik.${DOMAIN}`)"
      - "traefik.http.routers.dashboard.tls.certresolver=letsencrypt"

  # Frontend Application
  frontend:
    image: ${REGISTRY}/frontend:${VERSION}
    deploy:
      replicas: 3
      resources:
        limits:
          memory: 512M
        reservations:
          memory: 256M
    networks:
      - frontend
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.frontend.rule=Host(`${DOMAIN}`) || Host(`www.${DOMAIN}`)"
      - "traefik.http.routers.frontend.tls.certresolver=letsencrypt"
      - "traefik.http.services.frontend.loadbalancer.server.port=80"
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost/health"]
      interval: 30s
      timeout: 10s
      retries: 3

  # API Gateway
  api-gateway:
    image: ${REGISTRY}/api-gateway:${VERSION}
    deploy:
      replicas: 2
      resources:
        limits:
          memory: 1G
        reservations:
          memory: 512M
    networks:
      - frontend
      - backend
    environment:
      - JWT_SECRET_FILE=/run/secrets/jwt_secret
      - DATABASE_URL=postgresql://api_user:${API_DB_PASSWORD}@postgres-master:5432/api_db
      - REDIS_URL=redis://redis-cluster:6379/0
    secrets:
      - jwt_secret
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.api.rule=Host(`api.${DOMAIN}`)"
      - "traefik.http.routers.api.tls.certresolver=letsencrypt"
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 3

  # User Service
  user-service:
    image: ${REGISTRY}/user-service:${VERSION}
    deploy:
      replicas: 2
    networks:
      - backend
      - database
    environment:
      - DATABASE_URL=postgresql://user_svc:${USER_DB_PASSWORD}@postgres-master:5432/users_db
      - REDIS_URL=redis://redis-cluster:6379/1
      - ENCRYPTION_KEY_FILE=/run/secrets/encryption_key
    secrets:
      - encryption_key
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8001/health"]
      interval: 30s
      timeout: 10s
      retries: 3

  # Tenant Service
  tenant-service:
    image: ${REGISTRY}/tenant-service:${VERSION}
    deploy:
      replicas: 2
    networks:
      - backend
      - database
    environment:
      - DATABASE_URL=postgresql://tenant_svc:${TENANT_DB_PASSWORD}@postgres-master:5432/tenants_db
      - REDIS_URL=redis://redis-cluster:6379/2
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8002/health"]
      interval: 30s
      timeout: 10s
      retries: 3

  # Billing Service
  billing-service:
    image: ${REGISTRY}/billing-service:${VERSION}
    networks:
      - backend
      - database
    environment:
      - DATABASE_URL=postgresql://billing_svc:${BILLING_DB_PASSWORD}@postgres-master:5432/billing_db
      - STRIPE_SECRET_KEY_FILE=/run/secrets/stripe_secret
      - WEBHOOK_SECRET_FILE=/run/secrets/stripe_webhook_secret
    secrets:
      - stripe_secret
      - stripe_webhook_secret
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8003/health"]
      interval: 30s
      timeout: 10s
      retries: 3

  # Analytics Service
  analytics-service:
    image: ${REGISTRY}/analytics-service:${VERSION}
    networks:
      - backend
      - database
    environment:
      - CLICKHOUSE_URL=http://clickhouse:8123
      - KAFKA_BROKERS=kafka:9092
    depends_on:
      - clickhouse
      - kafka

  # Database Cluster
  postgres-master:
    image: postgres:14
    environment:
      - POSTGRES_REPLICATION_MODE=master
      - POSTGRES_REPLICATION_USER=replicator
      - POSTGRES_REPLICATION_PASSWORD_FILE=/run/secrets/replication_password
      - POSTGRES_MULTIPLE_DATABASES=api_db,users_db,tenants_db,billing_db
      - POSTGRES_MULTIPLE_USERS=api_user,user_svc,tenant_svc,billing_svc
    volumes:
      - postgres_master_data:/var/lib/postgresql/data
      - ./postgres/init-multiple-databases.sh:/docker-entrypoint-initdb.d/init-multiple-databases.sh
      - ./postgres/postgresql.conf:/etc/postgresql/postgresql.conf
    networks:
      - database
    secrets:
      - replication_password
    command: postgres -c config_file=/etc/postgresql/postgresql.conf

  postgres-replica:
    image: postgres:14
    environment:
      - POSTGRES_REPLICATION_MODE=slave
      - POSTGRES_MASTER_HOST=postgres-master
      - POSTGRES_REPLICATION_USER=replicator
      - POSTGRES_REPLICATION_PASSWORD_FILE=/run/secrets/replication_password
    volumes:
      - postgres_replica_data:/var/lib/postgresql/data
    networks:
      - database
    secrets:
      - replication_password
    depends_on:
      - postgres-master

  # Redis Cluster
  redis-cluster:
    image: redis:7-alpine
    command: |
      redis-server
      --cluster-enabled yes
      --cluster-config-file nodes.conf
      --cluster-node-timeout 5000
      --appendonly yes
      --maxmemory 1gb
      --maxmemory-policy allkeys-lru
    volumes:
      - redis_data:/data
    networks:
      - backend

  # ClickHouse for Analytics
  clickhouse:
    image: clickhouse/clickhouse-server:latest
    environment:
      - CLICKHOUSE_DB=analytics
      - CLICKHOUSE_USER=analytics_user
      - CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD}
    volumes:
      - clickhouse_data:/var/lib/clickhouse
    networks:
      - database

  # Kafka for Event Streaming
  kafka:
    image: confluentinc/cp-kafka:latest
    environment:
      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092
      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
      - KAFKA_AUTO_CREATE_TOPICS_ENABLE=true
    volumes:
      - kafka_data:/var/lib/kafka/data
    networks:
      - backend
    depends_on:
      - zookeeper

  zookeeper:
    image: confluentinc/cp-zookeeper:latest
    environment:
      - ZOOKEEPER_CLIENT_PORT=2181
      - ZOOKEEPER_TICK_TIME=2000
    volumes:
      - zookeeper_data:/var/lib/zookeeper/data
    networks:
      - backend

  # Background Workers
  worker:
    image: ${REGISTRY}/worker:${VERSION}
    deploy:
      replicas: 3
    networks:
      - backend
      - database
    environment:
      - CELERY_BROKER_URL=redis://redis-cluster:6379/3
      - DATABASE_URL=postgresql://worker:${WORKER_DB_PASSWORD}@postgres-master:5432/jobs_db
    command: celery -A app worker -l info -c 4

  scheduler:
    image: ${REGISTRY}/worker:${VERSION}
    networks:
      - backend
    environment:
      - CELERY_BROKER_URL=redis://redis-cluster:6379/3
    command: celery -A app beat -l info

networks:
  frontend:
    driver: bridge
  backend:
    driver: bridge
    internal: true
  database:
    driver: bridge
    internal: true
  monitoring:
    driver: bridge

volumes:
  traefik_certs:
  postgres_master_data:
  postgres_replica_data:
  redis_data:
  clickhouse_data:
  kafka_data:
  zookeeper_data:

secrets:
  jwt_secret:
    external: true
  encryption_key:
    external: true
  stripe_secret:
    external: true
  stripe_webhook_secret:
    external: true
  replication_password:
    external: true

Monitoring and Observability Stack

# docker-compose.monitoring.yml
version: '3.8'

services:
  # Prometheus
  prometheus:
    image: prom/prometheus:latest
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
      - ./prometheus/rules:/etc/prometheus/rules
      - prometheus_data:/prometheus
    networks:
      - monitoring
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--storage.tsdb.retention.time=30d'
      - '--web.enable-lifecycle'
      - '--web.enable-admin-api'

  # Grafana
  grafana:
    image: grafana/grafana:latest
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD}
      - GF_USERS_ALLOW_SIGN_UP=false
      - GF_INSTALL_PLUGINS=grafana-piechart-panel,grafana-worldmap-panel
    volumes:
      - grafana_data:/var/lib/grafana
      - ./grafana/provisioning:/etc/grafana/provisioning
    networks:
      - monitoring

  # AlertManager
  alertmanager:
    image: prom/alertmanager:latest
    ports:
      - "9093:9093"
    volumes:
      - ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml
      - alertmanager_data:/alertmanager
    networks:
      - monitoring

  # Jaeger for Distributed Tracing
  jaeger:
    image: jaegertracing/all-in-one:latest
    ports:
      - "16686:16686"
      - "14268:14268"
    environment:
      - COLLECTOR_ZIPKIN_HTTP_PORT=9411
    networks:
      - monitoring

  # ELK Stack for Logging
  elasticsearch:
    image: elasticsearch:7.17.0
    environment:
      - discovery.type=single-node
      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
      - xpack.security.enabled=false
    volumes:
      - elasticsearch_data:/usr/share/elasticsearch/data
    networks:
      - monitoring

  logstash:
    image: logstash:7.17.0
    volumes:
      - ./logstash/pipeline:/usr/share/logstash/pipeline
      - ./logstash/config:/usr/share/logstash/config
    networks:
      - monitoring
    depends_on:
      - elasticsearch

  kibana:
    image: kibana:7.17.0
    ports:
      - "5601:5601"
    environment:
      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
    networks:
      - monitoring
    depends_on:
      - elasticsearch

networks:
  monitoring:
    external: true

volumes:
  prometheus_data:
  grafana_data:
  alertmanager_data:
  elasticsearch_data:

Project 2: CI/CD Pipeline with GitOps

Complete CI/CD Configuration

# .gitlab-ci.yml
stages:
  - build
  - test
  - security
  - package
  - deploy-staging
  - integration-tests
  - deploy-production

variables:
  DOCKER_DRIVER: overlay2
  DOCKER_TLS_CERTDIR: "/certs"
  REGISTRY: $CI_REGISTRY_IMAGE
  COMPOSE_PROJECT_NAME: $CI_PROJECT_NAME-$CI_COMMIT_REF_SLUG

services:
  - docker:20.10.16-dind

before_script:
  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY

# Build Stage
build:
  stage: build
  script:
    - |
      # Build all services
      docker-compose -f docker-compose.build.yml build
      
      # Tag and push images
      for service in frontend api-gateway user-service tenant-service billing-service analytics-service worker; do
        docker tag ${COMPOSE_PROJECT_NAME}_${service}:latest $REGISTRY/$service:$CI_COMMIT_SHA
        docker push $REGISTRY/$service:$CI_COMMIT_SHA
        docker tag $REGISTRY/$service:$CI_COMMIT_SHA $REGISTRY/$service:latest
        docker push $REGISTRY/$service:latest
      done

# Test Stage
test:
  stage: test
  script:
    - |
      # Run unit tests
      docker-compose -f docker-compose.test.yml up --build --abort-on-container-exit
      
      # Run integration tests
      docker-compose -f docker-compose.integration.yml up --build --abort-on-container-exit
  artifacts:
    reports:
      junit: test-results.xml
      coverage: coverage.xml

# Security Scanning
security-scan:
  stage: security
  script:
    - |
      # Scan images for vulnerabilities
      for service in frontend api-gateway user-service tenant-service billing-service; do
        docker run --rm -v /var/run/docker.sock:/var/run/docker.sock \
          aquasec/trivy:latest image --exit-code 1 --severity HIGH,CRITICAL \
          $REGISTRY/$service:$CI_COMMIT_SHA
      done
      
      # SAST scanning
      docker run --rm -v $PWD:/code \
        registry.gitlab.com/gitlab-org/security-products/sast:latest /analyzer run

# Package Stage
package:
  stage: package
  script:
    - |
      # Create deployment package
      envsubst < docker-compose.prod.template.yml > docker-compose.prod.yml
      tar czf deployment-$CI_COMMIT_SHA.tar.gz \
        docker-compose.prod.yml \
        docker-compose.monitoring.yml \
        traefik/ \
        prometheus/ \
        grafana/ \
        scripts/
  artifacts:
    paths:
      - deployment-$CI_COMMIT_SHA.tar.gz
    expire_in: 1 week

# Staging Deployment
deploy-staging:
  stage: deploy-staging
  script:
    - |
      # Deploy to staging environment
      export VERSION=$CI_COMMIT_SHA
      export DOMAIN=staging.example.com
      export REGISTRY=$CI_REGISTRY_IMAGE
      
      # Update staging environment
      docker-compose -f docker-compose.staging.yml down
      docker-compose -f docker-compose.staging.yml pull
      docker-compose -f docker-compose.staging.yml up -d
      
      # Wait for services to be healthy
      ./scripts/wait-for-health.sh staging
  environment:
    name: staging
    url: https://staging.example.com
  only:
    - develop

# Integration Tests
integration-tests:
  stage: integration-tests
  script:
    - |
      # Run end-to-end tests against staging
      docker run --rm \
        -e BASE_URL=https://staging.example.com \
        -v $PWD/e2e:/tests \
        cypress/included:latest
  artifacts:
    when: always
    paths:
      - e2e/screenshots/
      - e2e/videos/
  only:
    - develop

# Production Deployment
deploy-production:
  stage: deploy-production
  script:
    - |
      # Blue-Green Deployment
      export VERSION=$CI_COMMIT_SHA
      export DOMAIN=example.com
      export REGISTRY=$CI_REGISTRY_IMAGE
      
      # Determine current and target environments
      CURRENT=$(curl -s https://api.example.com/deployment/current)
      TARGET=$([ "$CURRENT" = "blue" ] && echo "green" || echo "blue")
      
      echo "Deploying to $TARGET environment"
      
      # Deploy to target environment
      docker-compose -f docker-compose.$TARGET.yml down
      docker-compose -f docker-compose.$TARGET.yml pull
      docker-compose -f docker-compose.$TARGET.yml up -d
      
      # Health check
      ./scripts/wait-for-health.sh $TARGET
      
      # Switch traffic
      ./scripts/switch-traffic.sh $TARGET
      
      # Cleanup old environment after successful switch
      sleep 300  # Wait 5 minutes
      OTHER=$([ "$TARGET" = "blue" ] && echo "green" || echo "blue")
      docker-compose -f docker-compose.$OTHER.yml down
  environment:
    name: production
    url: https://example.com
  when: manual
  only:
    - main

Deployment Scripts

scripts/wait-for-health.sh:

#!/bin/bash
ENVIRONMENT=$1
MAX_ATTEMPTS=30
ATTEMPT=0

echo "Waiting for $ENVIRONMENT environment to be healthy..."

while [ $ATTEMPT -lt $MAX_ATTEMPTS ]; do
    HEALTHY=true
    
    # Check each service health
    for service in frontend api-gateway user-service tenant-service billing-service; do
        if ! curl -f -s "http://$service-$ENVIRONMENT:8000/health" > /dev/null; then
            HEALTHY=false
            break
        fi
    done
    
    if [ "$HEALTHY" = true ]; then
        echo "All services are healthy!"
        exit 0
    fi
    
    echo "Attempt $((ATTEMPT + 1))/$MAX_ATTEMPTS - Services not ready yet..."
    sleep 10
    ATTEMPT=$((ATTEMPT + 1))
done

echo "Health check failed after $MAX_ATTEMPTS attempts"
exit 1

scripts/switch-traffic.sh:

#!/bin/bash
TARGET=$1

echo "Switching traffic to $TARGET environment..."

# Update load balancer configuration
cat > /etc/nginx/conf.d/upstream.conf << EOF
upstream backend {
    server app-$TARGET:8000;
}
EOF

# Reload nginx
nginx -s reload

# Update deployment marker
echo "$TARGET" > /var/www/html/deployment/current

echo "Traffic switched to $TARGET environment"

Project 3: Disaster Recovery and Backup System

# docker-compose.backup.yml
version: '3.8'

services:
  # Backup Orchestrator
  backup-orchestrator:
    build: ./backup
    environment:
      - BACKUP_SCHEDULE=0 2 * * *
      - S3_BUCKET=${BACKUP_S3_BUCKET}
      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
      - ENCRYPTION_KEY_FILE=/run/secrets/backup_encryption_key
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - backup_temp:/tmp/backup
    secrets:
      - backup_encryption_key
    networks:
      - backup

  # Database Backup
  postgres-backup:
    image: postgres:14
    environment:
      - PGPASSWORD_FILE=/run/secrets/postgres_password
    volumes:
      - backup_temp:/backup
      - ./scripts/postgres-backup.sh:/backup.sh
    secrets:
      - postgres_password
    networks:
      - backup
      - database
    profiles:
      - backup
    command: /backup.sh

  # Volume Backup
  volume-backup:
    image: alpine
    volumes:
      - redis_data:/source/redis:ro
      - clickhouse_data:/source/clickhouse:ro
      - backup_temp:/backup
    profiles:
      - backup
    command: |
      sh -c "
        tar czf /backup/volumes_$(date +%Y%m%d_%H%M%S).tar.gz -C /source .
        echo 'Volume backup completed'
      "

  # Disaster Recovery Test
  dr-test:
    build: ./dr-test
    environment:
      - TEST_ENVIRONMENT=dr-test
      - RESTORE_FROM_BACKUP=latest
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - backup_temp:/backup:ro
    profiles:
      - disaster-recovery
    command: |
      sh -c "
        echo 'Starting disaster recovery test...'
        ./restore-and-test.sh
      "

networks:
  backup:
    driver: bridge
  database:
    external: true

volumes:
  backup_temp:
  redis_data:
    external: true
  clickhouse_data:
    external: true

secrets:
  backup_encryption_key:
    external: true
  postgres_password:
    external: true

Summary

This comprehensive final section demonstrated:

Enterprise-Grade Implementation

  • Multi-Tenant SaaS Platform: Complete production architecture with load balancing, service mesh, and multi-database setup
  • Monitoring Stack: Full observability with Prometheus, Grafana, Jaeger, and ELK stack
  • Security Integration: Secrets management, network isolation, and vulnerability scanning

DevOps Excellence

  • CI/CD Pipeline: Complete GitLab CI pipeline with security scanning, testing, and blue-green deployment
  • GitOps Workflow: Infrastructure as code with automated deployment and rollback capabilities
  • Quality Gates: Comprehensive testing including unit, integration, and end-to-end tests

Operational Resilience

  • Disaster Recovery: Automated backup systems with encryption and cloud storage
  • High Availability: Database replication, service redundancy, and health monitoring
  • Performance Optimization: Caching layers, connection pooling, and resource management

Key Achievements

Throughout this Docker Compose guide, you’ve mastered:

  1. Fundamental Concepts: Service orchestration, networking, and volume management
  2. Advanced Patterns: Service mesh integration, event-driven architecture, and CQRS
  3. Production Readiness: Security hardening, performance optimization, and monitoring
  4. Real-World Implementation: Complete enterprise systems with CI/CD and disaster recovery

Congratulations! You now have the expertise to design, implement, and operate production-grade Docker Compose applications. You can confidently tackle complex multi-service architectures, implement robust CI/CD pipelines, and ensure operational excellence in containerized environments.

Continue Your Learning

This is part 6 of 6 in the comprehensive guide.

← Previous Best Practices and Optimization Guide Overview See all 6 parts

Guide Complete!

You've finished all 6 parts of this guide.

Explore More Browse other guides