Docker Compose Core Concepts and Fundamentals

This section explores the fundamental concepts that make Docker Compose a powerful orchestration tool, covering advanced service configuration, networking patterns, and volume management strategies.

Service Configuration Deep Dive

Build Context and Arguments

version: '3.8'

services:
  web:
    build:
      context: ./web
      dockerfile: Dockerfile.prod
      args:
        - NODE_ENV=production
        - API_VERSION=v2
      target: production
      cache_from:
        - node:16-alpine
        - myapp:latest
    image: myapp:${TAG:-latest}

Multi-stage Dockerfile with build args:

# Dockerfile.prod
ARG NODE_ENV=development
ARG API_VERSION=v1

FROM node:16-alpine AS base
WORKDIR /app
COPY package*.json ./
RUN npm ci --only=production

FROM base AS development
RUN npm ci
COPY . .
CMD ["npm", "run", "dev"]

FROM base AS production
COPY . .
RUN npm run build
EXPOSE 3000
CMD ["npm", "start"]

Resource Constraints

version: '3.8'

services:
  web:
    image: nginx
    deploy:
      resources:
        limits:
          cpus: '0.50'
          memory: 512M
        reservations:
          cpus: '0.25'
          memory: 256M
      restart_policy:
        condition: on-failure
        delay: 5s
        max_attempts: 3
        window: 120s

  worker:
    image: myapp:worker
    deploy:
      replicas: 3
      update_config:
        parallelism: 1
        delay: 10s
        failure_action: rollback
      rollback_config:
        parallelism: 1
        delay: 5s

Environment Variable Patterns

version: '3.8'

services:
  app:
    image: myapp
    environment:
      # Direct assignment
      - NODE_ENV=production
      - PORT=3000
      
      # From host environment
      - SECRET_KEY=${SECRET_KEY}
      - DATABASE_URL=${DATABASE_URL:-postgresql://localhost:5432/myapp}
      
      # Computed values
      - APP_URL=https://${DOMAIN:-localhost}:${PORT:-3000}
    
    env_file:
      - .env
      - .env.local
      - .env.${NODE_ENV:-development}

.env file structure:

# .env
NODE_ENV=development
DEBUG=1
LOG_LEVEL=info

# Database
DATABASE_HOST=db
DATABASE_PORT=5432
DATABASE_NAME=myapp
DATABASE_USER=user
DATABASE_PASSWORD=password

# Redis
REDIS_HOST=redis
REDIS_PORT=6379
REDIS_DB=0

# External Services
SMTP_HOST=smtp.gmail.com
SMTP_PORT=587
SMTP_USER=[email protected]
SMTP_PASSWORD=app-password

Advanced Networking

Custom Network Configuration

version: '3.8'

services:
  web:
    image: nginx
    networks:
      frontend:
        aliases:
          - web-server
          - nginx-proxy
      backend:
        ipv4_address: 172.20.0.10

  api:
    image: node:16
    networks:
      - backend
      - database

  db:
    image: postgres:13
    networks:
      database:
        aliases:
          - postgres-server

networks:
  frontend:
    driver: bridge
    ipam:
      config:
        - subnet: 172.19.0.0/16
          gateway: 172.19.0.1

  backend:
    driver: bridge
    ipam:
      config:
        - subnet: 172.20.0.0/16

  database:
    driver: bridge
    internal: true  # No external access
    ipam:
      config:
        - subnet: 172.21.0.0/16

Network Isolation Patterns

version: '3.8'

services:
  # Public-facing services
  nginx:
    image: nginx
    ports:
      - "80:80"
      - "443:443"
    networks:
      - frontend

  # Application services
  web:
    build: ./web
    networks:
      - frontend
      - backend
    depends_on:
      - api

  api:
    build: ./api
    networks:
      - backend
      - database
    depends_on:
      - db
      - redis

  # Data services (isolated)
  db:
    image: postgres:13
    networks:
      - database
    volumes:
      - postgres_data:/var/lib/postgresql/data

  redis:
    image: redis:7-alpine
    networks:
      - database
    volumes:
      - redis_data:/data

networks:
  frontend:
    driver: bridge
  backend:
    driver: bridge
  database:
    driver: bridge
    internal: true

volumes:
  postgres_data:
  redis_data:

Service Discovery and Load Balancing

version: '3.8'

services:
  nginx:
    image: nginx
    ports:
      - "80:80"
    volumes:
      - ./nginx.conf:/etc/nginx/nginx.conf
    depends_on:
      - web
    networks:
      - frontend

  web:
    build: ./web
    deploy:
      replicas: 3
    networks:
      - frontend
      - backend
    environment:
      - API_URL=http://api:8000

  api:
    build: ./api
    deploy:
      replicas: 2
    networks:
      - backend
    environment:
      - DATABASE_URL=postgresql://user:pass@db:5432/myapp

networks:
  frontend:
  backend:

nginx.conf for load balancing:

events {
    worker_connections 1024;
}

http {
    upstream web_servers {
        server web:3000;
        # Docker Compose automatically load balances across replicas
    }

    server {
        listen 80;
        
        location / {
            proxy_pass http://web_servers;
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        }
    }
}

Volume Management Strategies

Volume Types and Use Cases

version: '3.8'

services:
  web:
    image: nginx
    volumes:
      # Named volume for persistent data
      - web_data:/var/www/html
      
      # Bind mount for development
      - ./src:/var/www/html:ro
      
      # Tmpfs for temporary files
      - type: tmpfs
        target: /tmp
        tmpfs:
          size: 100M
      
      # Volume with specific options
      - type: volume
        source: web_logs
        target: /var/log/nginx
        volume:
          nocopy: true

  db:
    image: postgres:13
    volumes:
      # Database data persistence
      - postgres_data:/var/lib/postgresql/data
      
      # Configuration files
      - ./postgres/postgresql.conf:/etc/postgresql/postgresql.conf:ro
      
      # Initialization scripts
      - ./postgres/init:/docker-entrypoint-initdb.d:ro
      
      # Backup location
      - backup_data:/backup

volumes:
  web_data:
    driver: local
  web_logs:
    driver: local
    driver_opts:
      type: none
      o: bind
      device: /host/logs/web
  postgres_data:
    driver: local
    driver_opts:
      type: none
      o: bind
      device: /data/postgres
  backup_data:
    external: true

Volume Backup and Restore

version: '3.8'

services:
  app:
    image: myapp
    volumes:
      - app_data:/data

  backup:
    image: alpine
    volumes:
      - app_data:/source:ro
      - backup_storage:/backup
    command: |
      sh -c "
        tar czf /backup/app_data_$$(date +%Y%m%d_%H%M%S).tar.gz -C /source .
        find /backup -name 'app_data_*.tar.gz' -mtime +7 -delete
      "
    profiles:
      - backup

  restore:
    image: alpine
    volumes:
      - app_data:/target
      - backup_storage:/backup
    command: |
      sh -c "
        if [ -f /backup/restore.tar.gz ]; then
          cd /target && tar xzf /backup/restore.tar.gz
        else
          echo 'No restore file found'
        fi
      "
    profiles:
      - restore

volumes:
  app_data:
  backup_storage:

Run backup/restore:

# Create backup
docker-compose --profile backup run --rm backup

# Restore from backup
cp backup_file.tar.gz backup_storage/restore.tar.gz
docker-compose --profile restore run --rm restore

Configuration Management

Secrets Management

version: '3.8'

services:
  web:
    image: myapp
    secrets:
      - db_password
      - api_key
    environment:
      - DB_PASSWORD_FILE=/run/secrets/db_password
      - API_KEY_FILE=/run/secrets/api_key

  db:
    image: postgres:13
    secrets:
      - db_password
    environment:
      - POSTGRES_PASSWORD_FILE=/run/secrets/db_password

secrets:
  db_password:
    file: ./secrets/db_password.txt
  api_key:
    external: true

Configuration Files

version: '3.8'

services:
  nginx:
    image: nginx
    configs:
      - source: nginx_config
        target: /etc/nginx/nginx.conf
        mode: 0644
      - source: ssl_cert
        target: /etc/ssl/certs/server.crt
        mode: 0644

  app:
    image: myapp
    configs:
      - source: app_config
        target: /app/config.json

configs:
  nginx_config:
    file: ./config/nginx.conf
  ssl_cert:
    file: ./certs/server.crt
  app_config:
    external: true

Template-based Configuration

docker-compose.template.yml:

version: '3.8'

services:
  web:
    image: ${WEB_IMAGE}:${WEB_TAG}
    ports:
      - "${WEB_PORT}:3000"
    environment:
      - NODE_ENV=${NODE_ENV}
      - API_URL=${API_URL}

  db:
    image: postgres:${POSTGRES_VERSION}
    environment:
      - POSTGRES_DB=${DB_NAME}
      - POSTGRES_USER=${DB_USER}
      - POSTGRES_PASSWORD=${DB_PASSWORD}

Generate configuration script:

#!/bin/bash
# generate-compose.sh

export WEB_IMAGE="myapp"
export WEB_TAG="${1:-latest}"
export WEB_PORT="${2:-3000}"
export NODE_ENV="${3:-production}"
export API_URL="https://api.${DOMAIN}"
export POSTGRES_VERSION="13"
export DB_NAME="myapp"
export DB_USER="user"
export DB_PASSWORD="$(openssl rand -base64 32)"

envsubst < docker-compose.template.yml > docker-compose.yml

Service Dependencies and Health Checks

Advanced Dependency Management

version: '3.8'

services:
  web:
    build: ./web
    depends_on:
      db:
        condition: service_healthy
      redis:
        condition: service_healthy
      migration:
        condition: service_completed_successfully

  migration:
    build: ./web
    command: python manage.py migrate
    depends_on:
      db:
        condition: service_healthy
    restart: "no"

  db:
    image: postgres:13
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"]
      interval: 10s
      timeout: 5s
      retries: 5
      start_period: 30s

  redis:
    image: redis:7-alpine
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 10s
      timeout: 3s
      retries: 3

Custom Health Check Scripts

health-check.sh:

#!/bin/bash
# Custom health check for web application

# Check if application is responding
if curl -f http://localhost:3000/health > /dev/null 2>&1; then
    echo "Application is healthy"
    exit 0
else
    echo "Application health check failed"
    exit 1
fi

version: '3.8'

services:
  web:
    build: ./web
    healthcheck:
      test: ["CMD", "/app/health-check.sh"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 60s

Wait Strategies

wait-for-it.sh integration:

version: '3.8'

services:
  web:
    build: ./web
    command: ["./wait-for-it.sh", "db:5432", "--", "python", "app.py"]
    depends_on:
      - db

  api:
    build: ./api
    command: |
      sh -c "
        ./wait-for-it.sh db:5432 --timeout=60 --strict -- \
        ./wait-for-it.sh redis:6379 --timeout=30 --strict -- \
        python manage.py migrate && \
        python manage.py runserver 0.0.0.0:8000
      "
    depends_on:
      - db
      - redis

Scaling and Performance

Horizontal Scaling

version: '3.8'

services:
  web:
    build: ./web
    deploy:
      replicas: 3
      update_config:
        parallelism: 1
        delay: 10s
        failure_action: rollback
        monitor: 60s
      restart_policy:
        condition: on-failure
        delay: 5s
        max_attempts: 3

  worker:
    build: ./worker
    deploy:
      replicas: 5
      resources:
        limits:
          cpus: '0.5'
          memory: 512M
        reservations:
          cpus: '0.25'
          memory: 256M

Scale services dynamically:

# Scale web service to 5 replicas
docker-compose up --scale web=5

# Scale multiple services
docker-compose up --scale web=3 --scale worker=10

Performance Optimization

version: '3.8'

services:
  web:
    build: ./web
    # Optimize container startup
    init: true
    # Limit log size
    logging:
      driver: "json-file"
      options:
        max-size: "10m"
        max-file: "3"
    # Resource limits
    mem_limit: 512m
    memswap_limit: 512m
    cpu_count: 2
    cpu_percent: 50

  nginx:
    image: nginx:alpine
    # Use tmpfs for temporary files
    tmpfs:
      - /var/cache/nginx:noexec,nosuid,size=100m
      - /tmp:noexec,nosuid,size=50m
    # Optimize shared memory
    shm_size: 128m

Summary

In this section, you’ve mastered:

Advanced Service Configuration

  • Build Contexts: Multi-stage builds with arguments and caching
  • Resource Management: CPU and memory limits with deployment strategies
  • Environment Patterns: Complex variable management and file structures

Networking Mastery

  • Custom Networks: IPAM configuration and network isolation
  • Service Discovery: Load balancing and inter-service communication
  • Security Patterns: Network segmentation and access control

Volume Strategies

  • Volume Types: Named volumes, bind mounts, and tmpfs usage
  • Data Management: Backup, restore, and migration strategies
  • Performance: Optimized volume configurations

Configuration Management

  • Secrets: Secure credential handling
  • Templates: Dynamic configuration generation
  • Health Checks: Advanced dependency and readiness management

Next Steps: In Part 3, we’ll explore practical applications including real-world multi-service architectures, development workflows, and production deployment patterns that demonstrate these concepts in action.

Continue Your Learning

This is part 2 of 6 in the comprehensive guide.

← Previous Introduction and Setup Guide Overview See all 6 parts Next → Practical Applications and Examples