Production Performance Monitoring
Monitoring performance in production is essential for maintaining optimal application efficiency over time.
Continuous Profiling Setup
Setting up continuous profiling allows you to monitor performance in production:
package main
import (
"fmt"
"log"
"net/http"
_ "net/http/pprof" // Import for side effects
"os"
"runtime"
"runtime/pprof"
"time"
)
// Configuration for profiling
type ProfilingConfig struct {
EnableHTTPProfiling bool
CPUProfileInterval time.Duration
MemProfileInterval time.Duration
BlockProfileRate int
MutexProfileRate int
OutputDir string
}
// ProfileManager handles periodic profiling
type ProfileManager struct {
config ProfilingConfig
stopCh chan struct{}
}
// NewProfileManager creates a new profile manager
func NewProfileManager(config ProfilingConfig) *ProfileManager {
return &ProfileManager{
config: config,
stopCh: make(chan struct{}),
}
}
// Start begins the profiling routines
func (pm *ProfileManager) Start() {
// Create output directory if it doesn't exist
if pm.config.OutputDir != "" {
if err := os.MkdirAll(pm.config.OutputDir, 0755); err != nil {
log.Printf("Failed to create profile output directory: %v", err)
return
}
}
// Configure profiling rates
if pm.config.BlockProfileRate > 0 {
runtime.SetBlockProfileRate(pm.config.BlockProfileRate)
}
if pm.config.MutexProfileRate > 0 {
runtime.SetMutexProfileFraction(pm.config.MutexProfileRate)
}
// Start HTTP server for pprof if enabled
if pm.config.EnableHTTPProfiling {
go func() {
log.Println("Starting pprof HTTP server on :6060")
if err := http.ListenAndServe(":6060", nil); err != nil {
log.Printf("pprof HTTP server failed: %v", err)
}
}()
}
// Start periodic CPU profiling if enabled
if pm.config.CPUProfileInterval > 0 {
go pm.startPeriodicCPUProfiling()
}
// Start periodic memory profiling if enabled
if pm.config.MemProfileInterval > 0 {
go pm.startPeriodicMemProfiling()
}
}
// Stop stops all profiling routines
func (pm *ProfileManager) Stop() {
close(pm.stopCh)
}
// startPeriodicCPUProfiling captures CPU profiles at regular intervals
func (pm *ProfileManager) startPeriodicCPUProfiling() {
ticker := time.NewTicker(pm.config.CPUProfileInterval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
pm.captureCPUProfile()
case <-pm.stopCh:
return
}
}
}
// startPeriodicMemProfiling captures memory profiles at regular intervals
func (pm *ProfileManager) startPeriodicMemProfiling() {
ticker := time.NewTicker(pm.config.MemProfileInterval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
pm.captureMemProfile()
case <-pm.stopCh:
return
}
}
}
// captureCPUProfile captures a CPU profile
func (pm *ProfileManager) captureCPUProfile() {
timestamp := time.Now().Format("20060102-150405")
filename := fmt.Sprintf("%s/cpu-%s.prof", pm.config.OutputDir, timestamp)
f, err := os.Create(filename)
if err != nil {
log.Printf("Failed to create CPU profile file: %v", err)
return
}
defer f.Close()
log.Printf("Capturing CPU profile to %s", filename)
if err := pprof.StartCPUProfile(f); err != nil {
log.Printf("Failed to start CPU profile: %v", err)
return
}
// Profile for 30 seconds
time.Sleep(30 * time.Second)
pprof.StopCPUProfile()
log.Printf("CPU profile captured")
}
// captureMemProfile captures a memory profile
func (pm *ProfileManager) captureMemProfile() {
timestamp := time.Now().Format("20060102-150405")
filename := fmt.Sprintf("%s/mem-%s.prof", pm.config.OutputDir, timestamp)
f, err := os.Create(filename)
if err != nil {
log.Printf("Failed to create memory profile file: %v", err)
return
}
defer f.Close()
log.Printf("Capturing memory profile to %s", filename)
// Run GC before profiling to get accurate memory usage
runtime.GC()
if err := pprof.WriteHeapProfile(f); err != nil {
log.Printf("Failed to write memory profile: %v", err)
return
}
log.Printf("Memory profile captured")
}
// simulateLoad generates some CPU and memory load
func simulateLoad() {
// CPU load
go func() {
for {
for i := 0; i < 1000000; i++ {
_ = i * i
}
time.Sleep(100 * time.Millisecond)
}
}()
// Memory load
go func() {
var slices [][]byte
for {
// Allocate memory
slice := make([]byte, 1024*1024) // 1MB
for i := range slice {
slice[i] = byte(i % 256)
}
slices = append(slices, slice)
// Release some memory occasionally
if len(slices) > 10 {
slices = slices[5:]
}
time.Sleep(500 * time.Millisecond)
}
}()
}
func main() {
// Configure profiling
config := ProfilingConfig{
EnableHTTPProfiling: true,
CPUProfileInterval: 5 * time.Minute,
MemProfileInterval: 5 * time.Minute,
BlockProfileRate: 1,
MutexProfileRate: 1,
OutputDir: "./profiles",
}
// Create and start profile manager
pm := NewProfileManager(config)
pm.Start()
// Simulate application load
simulateLoad()
// Keep the application running
fmt.Println("Application running with continuous profiling...")
fmt.Println("Access pprof web interface at http://localhost:6060/debug/pprof/")
fmt.Println("Press Ctrl+C to exit")
// Wait indefinitely
select {}
}
Performance Metrics Collection
Collecting and analyzing performance metrics helps identify trends and issues:
package main
import (
"fmt"
"log"
"math/rand"
"net/http"
"runtime"
"sync"
"time"
)
// simulateAPIEndpoint simulates an API endpoint with variable response times
func simulateAPIEndpoint(endpoint string, minLatency, maxLatency time.Duration, errorRate float64) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
start := time.Now()
// Simulate processing time
processingTime := minLatency + time.Duration(rand.Float64()*float64(maxLatency-minLatency))
time.Sleep(processingTime)
// Simulate errors
if rand.Float64() < errorRate {
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
// Successful response
fmt.Fprintf(w, "Response from %s\n", endpoint)
// Log performance metrics
elapsed := time.Since(start)
log.Printf("%s - %s - %v", endpoint, r.Method, elapsed)
}
}
// collectRuntimeMetrics periodically collects and logs runtime metrics
func collectRuntimeMetrics(interval time.Duration) {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for range ticker.C {
var m runtime.MemStats
runtime.ReadMemStats(&m)
log.Printf("Goroutines: %d", runtime.NumGoroutine())
log.Printf("Memory: Alloc=%v MiB, Sys=%v MiB, NumGC=%v",
m.Alloc/1024/1024, m.Sys/1024/1024, m.NumGC)
}
}
// simulateLoad generates artificial load on the server
func simulateLoad(apiURL string, concurrency int, requestsPerSecond float64) {
// Calculate delay between requests
delay := time.Duration(float64(time.Second) / requestsPerSecond)
// Create a worker pool
var wg sync.WaitGroup
requestCh := make(chan struct{})
// Start workers
for i := 0; i < concurrency; i++ {
wg.Add(1)
go func() {
defer wg.Done()
client := &http.Client{
Timeout: 5 * time.Second,
}
for range requestCh {
resp, err := client.Get(apiURL)
if err != nil {
log.Printf("Request error: %v", err)
continue
}
resp.Body.Close()
}
}()
}
// Generate requests at the specified rate
ticker := time.NewTicker(delay)
defer ticker.Stop()
log.Printf("Generating load: %f requests/second with %d concurrent clients",
requestsPerSecond, concurrency)
for range ticker.C {
select {
case requestCh <- struct{}{}:
// Request sent to worker
default:
// All workers busy, skip this request
log.Println("Overloaded, skipping request")
}
}
close(requestCh)
wg.Wait()
}
func main() {
// Seed random number generator
rand.Seed(time.Now().UnixNano())
// Start runtime metrics collection
go collectRuntimeMetrics(5 * time.Second)
// Set up API endpoints
http.HandleFunc("/api/fast", simulateAPIEndpoint("fast", 10*time.Millisecond, 50*time.Millisecond, 0.01))
http.HandleFunc("/api/medium", simulateAPIEndpoint("medium", 50*time.Millisecond, 200*time.Millisecond, 0.05))
http.HandleFunc("/api/slow", simulateAPIEndpoint("slow", 200*time.Millisecond, 1000*time.Millisecond, 0.10))
// Start load generation
go simulateLoad("http://localhost:8080/api/fast", 10, 50)
go simulateLoad("http://localhost:8080/api/medium", 5, 20)
go simulateLoad("http://localhost:8080/api/slow", 2, 5)
// Start HTTP server
log.Println("Starting server on :8080")
if err := http.ListenAndServe(":8080", nil); err != nil {
log.Fatalf("Server failed: %v", err)
}
}
Takeaway Points
Performance profiling and optimization are essential skills for Go developers building applications that need to operate efficiently at scale. By mastering Go’s comprehensive suite of profiling tools and applying systematic optimization techniques, you can identify and eliminate bottlenecks before they impact your users.
The key takeaways from this guide include:
-
Start with clear performance objectives: Define specific, measurable performance goals before beginning optimization work.
-
Profile before optimizing: Use Go’s profiling tools to identify actual bottlenecks rather than optimizing based on assumptions.
-
Focus on the critical path: Optimize the parts of your code that have the greatest impact on overall performance.
-
Measure the impact: Quantify the effect of your optimizations through benchmarking and profiling.
-
Monitor in production: Set up continuous profiling and metrics collection to catch performance regressions early.
Remember that premature optimization can lead to more complex, harder-to-maintain code without meaningful performance benefits. The most effective approach is to write clean, idiomatic Go code first, then use profiling to guide targeted optimizations where they matter most.
By applying the techniques covered in this guide—from CPU and memory profiling to advanced concurrency patterns and compiler optimizations—you’ll be well-equipped to build Go applications that are not just functionally correct, but blazingly fast and resource-efficient.