Advanced pprof Usage
Go’s pprof tool provides powerful memory profiling capabilities:
package main
import (
"fmt"
"net/http"
_ "net/http/pprof" // Import for side effects
"os"
"runtime"
"runtime/pprof"
"time"
)
// A function that leaks memory
func leakyFunction() {
// Global variable that keeps growing
var leakySlice [][]byte
for i := 0; i < 100; i++ {
// Allocate 1MB
data := make([]byte, 1024*1024)
for j := range data {
data[j] = byte(j % 256)
}
leakySlice = append(leakySlice, data)
// Simulate some work
time.Sleep(100 * time.Millisecond)
}
}
// A function with temporary allocations
func temporaryAllocations() {
for i := 0; i < 100; i++ {
// Allocate 1MB that will be garbage collected
data := make([]byte, 1024*1024)
for j := range data {
data[j] = byte(j % 256)
}
// Use the data somehow
_ = data[1024]
// Simulate some work
time.Sleep(100 * time.Millisecond)
}
}
func main() {
// Start pprof HTTP server
go func() {
fmt.Println("Starting pprof server on :6060")
fmt.Println("Visit http://localhost:6060/debug/pprof/ in your browser")
http.ListenAndServe(":6060", nil)
}()
// Create CPU profile
cpuFile, err := os.Create("cpu_profile.prof")
if err != nil {
fmt.Printf("Could not create CPU profile: %v\n", err)
}
defer cpuFile.Close()
if err := pprof.StartCPUProfile(cpuFile); err != nil {
fmt.Printf("Could not start CPU profile: %v\n", err)
}
defer pprof.StopCPUProfile()
// Run the leaky function
fmt.Println("Running leaky function...")
leakyFunction()
// Create memory profile
memFile, err := os.Create("mem_profile.prof")
if err != nil {
fmt.Printf("Could not create memory profile: %v\n", err)
}
defer memFile.Close()
// Force GC before taking memory profile
runtime.GC()
if err := pprof.WriteHeapProfile(memFile); err != nil {
fmt.Printf("Could not write memory profile: %v\n", err)
}
// Run function with temporary allocations
fmt.Println("Running function with temporary allocations...")
temporaryAllocations()
fmt.Println("Profiles written to cpu_profile.prof and mem_profile.prof")
fmt.Println("Analyze with: go tool pprof cpu_profile.prof")
fmt.Println("Analyze with: go tool pprof mem_profile.prof")
// Keep server running for a while to allow profile inspection
fmt.Println("Server running. Press Ctrl+C to exit.")
time.Sleep(5 * time.Minute)
}
Analyzing Memory Profiles
Once you’ve collected memory profiles, you need to know how to analyze them effectively:
package main
import (
"flag"
"fmt"
"os"
"runtime"
"runtime/pprof"
"time"
)
// Different allocation patterns to analyze
func allocPattern1() {
// Large number of small allocations
var objects []*struct{ x int }
for i := 0; i < 1000000; i++ {
objects = append(objects, &struct{ x int }{i})
}
_ = objects
}
func allocPattern2() {
// Small number of large allocations
var slices [][]byte
for i := 0; i < 100; i++ {
slices = append(slices, make([]byte, 10*1024*1024))
}
_ = slices
}
func allocPattern3() {
// Mix of temporary and persistent allocations
persistent := make([][]byte, 0, 10)
for i := 0; i < 1000; i++ {
// Temporary allocations that should be GC'd
temp := make([]byte, 100*1024)
for j := range temp {
temp[j] = byte(j % 256)
}
// Only keep some allocations
if i%100 == 0 {
persistent = append(persistent, temp)
}
}
_ = persistent
}
func main() {
patternPtr := flag.Int("pattern", 1, "Allocation pattern to test (1, 2, or 3)")
flag.Parse()
// Create memory profile before allocations
beforeFile, err := os.Create("heap_before.prof")
if err != nil {
fmt.Printf("Could not create profile: %v\n", err)
return
}
runtime.GC()
if err := pprof.WriteHeapProfile(beforeFile); err != nil {
fmt.Printf("Could not write profile: %v\n", err)
}
beforeFile.Close()
// Run the selected allocation pattern
start := time.Now()
switch *patternPtr {
case 1:
fmt.Println("Running pattern 1: Many small allocations")
allocPattern1()
case 2:
fmt.Println("Running pattern 2: Few large allocations")
allocPattern2()
case 3:
fmt.Println("Running pattern 3: Mixed allocation patterns")
allocPattern3()
default:
fmt.Println("Invalid pattern selected")
return
}
duration := time.Since(start)
// Create memory profile after allocations
afterFile, err := os.Create("heap_after.prof")
if err != nil {
fmt.Printf("Could not create profile: %v\n", err)
return
}
runtime.GC()
if err := pprof.WriteHeapProfile(afterFile); err != nil {
fmt.Printf("Could not write profile: %v\n", err)
}
afterFile.Close()
// Print memory stats
var stats runtime.MemStats
runtime.ReadMemStats(&stats)
fmt.Printf("Execution time: %v\n", duration)
fmt.Printf("HeapAlloc: %d MB\n", stats.HeapAlloc/1024/1024)
fmt.Printf("HeapObjects: %d\n", stats.HeapObjects)
fmt.Printf("GC cycles: %d\n", stats.NumGC)
fmt.Println("\nProfiles written to heap_before.prof and heap_after.prof")
fmt.Println("Compare with: go tool pprof -base heap_before.prof heap_after.prof")
}
Continuous Memory Monitoring
For production systems, continuous memory monitoring is essential:
package main
import (
"encoding/json"
"fmt"
"net/http"
"os"
"runtime"
"time"
)
// MemoryStats represents the memory statistics we want to track
type MemoryStats struct {
Timestamp time.Time
HeapAlloc uint64
HeapSys uint64
HeapIdle uint64
HeapInuse uint64
HeapReleased uint64
HeapObjects uint64
StackInuse uint64
StackSys uint64
MSpanInuse uint64
MSpanSys uint64
MCacheInuse uint64
MCacheSys uint64
GCSys uint64
OtherSys uint64
NextGC uint64
LastGC uint64
PauseTotalNs uint64
NumGC uint32
GCCPUFraction float64
EnableGC bool
DebugGC bool
BySize []struct{ Size, Mallocs, Frees uint64 }
}
// Collect memory stats at regular intervals
func memoryMonitor(interval time.Duration, outputFile string) {
file, err := os.Create(outputFile)
if err != nil {
fmt.Printf("Error creating output file: %v\n", err)
return
}
defer file.Close()
encoder := json.NewEncoder(file)
ticker := time.NewTicker(interval)
defer ticker.Stop()
fmt.Printf("Memory monitoring started. Writing to %s every %v\n",
outputFile, interval)
for range ticker.C {
var stats runtime.MemStats
runtime.ReadMemStats(&stats)
memStats := MemoryStats{
Timestamp: time.Now(),
HeapAlloc: stats.HeapAlloc,
HeapSys: stats.HeapSys,
HeapIdle: stats.HeapIdle,
HeapInuse: stats.HeapInuse,
HeapReleased: stats.HeapReleased,
HeapObjects: stats.HeapObjects,
StackInuse: stats.StackInuse,
StackSys: stats.StackSys,
MSpanInuse: stats.MSpanInuse,
MSpanSys: stats.MSpanSys,
MCacheInuse: stats.MCacheInuse,
MCacheSys: stats.MCacheSys,
GCSys: stats.GCSys,
OtherSys: stats.OtherSys,
NextGC: stats.NextGC,
LastGC: stats.LastGC,
PauseTotalNs: stats.PauseTotalNs,
NumGC: stats.NumGC,
GCCPUFraction: stats.GCCPUFraction,
EnableGC: stats.EnableGC,
DebugGC: stats.DebugGC,
}
if err := encoder.Encode(memStats); err != nil {
fmt.Printf("Error encoding stats: %v\n", err)
}
// Also print to console
fmt.Printf("HeapAlloc: %d MB, Objects: %d, GC: %d\n",
stats.HeapAlloc/1024/1024, stats.HeapObjects, stats.NumGC)
}
}
func simulateLoad() {
// Simulate a service with varying memory usage patterns
var data [][]byte
for {
// Allocate some memory
for i := 0; i < 100; i++ {
data = append(data, make([]byte, 1024*1024))
}
// Simulate work
time.Sleep(2 * time.Second)
// Free some memory
if len(data) > 500 {
data = data[100:]
}
// Simulate work
time.Sleep(1 * time.Second)
}
}
func main() {
// Start HTTP server for pprof
go func() {
http.ListenAndServe(":6060", nil)
}()
// Start memory monitoring
go memoryMonitor(5*time.Second, "memory_stats.json")
// Simulate application load
simulateLoad()
}
Production Best Practices
Let’s explore best practices for memory management in production Go applications.
Memory Budgeting
Establishing memory budgets for different parts of your application is crucial:
package main
import (
"fmt"
"runtime"
"sync"
"time"
)
// MemoryBudget represents a memory allocation budget for a component
type MemoryBudget struct {
name string
maxBytes int64
currentBytes int64
mu sync.Mutex
warningHandler func(string, int64, int64)
}
// NewMemoryBudget creates a new memory budget
func NewMemoryBudget(name string, maxBytes int64, warningHandler func(string, int64, int64)) *MemoryBudget {
return &MemoryBudget{
name: name,
maxBytes: maxBytes,
warningHandler: warningHandler,
}
}
// Allocate attempts to allocate bytes within the budget
func (b *MemoryBudget) Allocate(bytes int64) bool {
b.mu.Lock()
defer b.mu.Unlock()
if b.currentBytes+bytes > b.maxBytes {
if b.warningHandler != nil {
b.warningHandler(b.name, bytes, b.maxBytes-b.currentBytes)
}
return false
}
b.currentBytes += bytes
return true
}
// Release frees bytes from the budget
func (b *MemoryBudget) Release(bytes int64) {
b.mu.Lock()
defer b.mu.Unlock()
b.currentBytes -= bytes
if b.currentBytes < 0 {
b.currentBytes = 0
}
}
// Usage returns the current usage percentage
func (b *MemoryBudget) Usage() float64 {
b.mu.Lock()
defer b.mu.Unlock()
return float64(b.currentBytes) / float64(b.maxBytes) * 100
}
func main() {
// Define warning handler
warningHandler := func(component string, requested, available int64) {
fmt.Printf("WARNING: %s exceeded memory budget. Requested: %d bytes, Available: %d bytes\n",
component, requested, available)
}
// Create budgets for different components
cacheBudget := NewMemoryBudget("Cache", 100*1024*1024, warningHandler) // 100 MB
queueBudget := NewMemoryBudget("Queue", 50*1024*1024, warningHandler) // 50 MB
processingBudget := NewMemoryBudget("Processing", 200*1024*1024, warningHandler) // 200 MB
// Simulate cache usage
go func() {
var cacheData [][]byte
for i := 0; i < 150; i++ {
// Try to allocate 1MB
size := int64(1 * 1024 * 1024)
if cacheBudget.Allocate(size) {
data := make([]byte, size)
cacheData = append(cacheData, data)
fmt.Printf("Cache allocated %d MB, usage: %.1f%%\n",
i+1, cacheBudget.Usage())
} else {
fmt.Println("Cache allocation denied, evicting oldest entries")
// Evict some entries
if len(cacheData) > 10 {
for j := 0; j < 10; j++ {
cacheBudget.Release(int64(len(cacheData[j])))
}
cacheData = cacheData[10:]
}
}
time.Sleep(100 * time.Millisecond)
}
}()
// Simulate queue usage
go func() {
var queueData [][]byte
for i := 0; i < 100; i++ {
// Try to allocate 0.5MB
size := int64(512 * 1024)
if queueBudget.Allocate(size) {
data := make([]byte, size)
queueData = append(queueData, data)
fmt.Printf("Queue allocated %.1f MB, usage: %.1f%%\n",
float64(i+1)/2, queueBudget.Usage())
} else {
fmt.Println("Queue allocation denied, processing backlog")
// Process some entries
if len(queueData) > 5 {
for j := 0; j < 5; j++ {
queueBudget.Release(int64(len(queueData[j])))
}
queueData = queueData[5:]
}
}
time.Sleep(200 * time.Millisecond)
}
}()
// Monitor overall memory usage
for i := 0; i < 30; i++ {
var stats runtime.MemStats
runtime.ReadMemStats(&stats)
fmt.Printf("\n--- Memory Stats #%d ---\n", i+1)
fmt.Printf("HeapAlloc: %d MB\n", stats.HeapAlloc/1024/1024)
fmt.Printf("HeapSys: %d MB\n", stats.HeapSys/1024/1024)
fmt.Printf("HeapObjects: %d\n", stats.HeapObjects)
fmt.Printf("GC cycles: %d\n", stats.NumGC)
time.Sleep(1 * time.Second)
}
}
Graceful Degradation Under Memory Pressure
Design your application to handle memory pressure gracefully:
package main
import (
"fmt"
"runtime"
"sync"
"time"
)
// MemoryPressureLevel indicates the current memory pressure
type MemoryPressureLevel int
const (
MemoryPressureNormal MemoryPressureLevel = iota
MemoryPressureModerate
MemoryPressureHigh
MemoryPressureCritical
)
// MemoryMonitor tracks memory usage and notifies listeners of pressure changes
type MemoryMonitor struct {
pressureLevel MemoryPressureLevel
listeners []func(MemoryPressureLevel)
mu sync.Mutex
normalThreshold float64 // % of max heap
moderateThreshold float64
highThreshold float64
criticalThreshold float64
}
// NewMemoryMonitor creates a new memory monitor
func NewMemoryMonitor() *MemoryMonitor {
return &MemoryMonitor{
pressureLevel: MemoryPressureNormal,
listeners: make([]func(MemoryPressureLevel), 0),
normalThreshold: 50.0,
moderateThreshold: 70.0,
highThreshold: 85.0,
criticalThreshold: 95.0,
}
}
// AddListener registers a function to be called when pressure level changes
func (m *MemoryMonitor) AddListener(listener func(MemoryPressureLevel)) {
m.mu.Lock()
defer m.mu.Unlock()
m.listeners = append(m.listeners, listener)
}
// Start begins monitoring memory usage
func (m *MemoryMonitor) Start(interval time.Duration) {
go func() {
for {
m.checkMemoryPressure()
time.Sleep(interval)
}
}()
}
// checkMemoryPressure checks current memory usage and notifies listeners if pressure level changes
func (m *MemoryMonitor) checkMemoryPressure() {
var stats runtime.MemStats
runtime.ReadMemStats(&stats)
// Calculate heap usage percentage
heapUsage := float64(stats.HeapAlloc) / float64(stats.HeapSys) * 100
m.mu.Lock()
defer m.mu.Unlock()
var newLevel MemoryPressureLevel
if heapUsage >= m.criticalThreshold {
newLevel = MemoryPressureCritical
} else if heapUsage >= m.highThreshold {
newLevel = MemoryPressureHigh
} else if heapUsage >= m.moderateThreshold {
newLevel = MemoryPressureModerate
} else {
newLevel = MemoryPressureNormal
}
// If pressure level changed, notify listeners
if newLevel != m.pressureLevel {
m.pressureLevel = newLevel
for _, listener := range m.listeners {
go listener(newLevel)
}
}
}
// Cache with memory pressure awareness
type PressureAwareCache struct {
data map[string][]byte
mu sync.RWMutex
currentPressure MemoryPressureLevel
}
// NewPressureAwareCache creates a new cache that responds to memory pressure
func NewPressureAwareCache() *PressureAwareCache {
return &PressureAwareCache{
data: make(map[string][]byte),
currentPressure: MemoryPressureNormal,
}
}
// HandleMemoryPressureChange adjusts cache behavior based on memory pressure
func (c *PressureAwareCache) HandleMemoryPressureChange(level MemoryPressureLevel) {
c.mu.Lock()
defer c.mu.Unlock()
c.currentPressure = level
switch level {
case MemoryPressureNormal:
fmt.Println("Cache: Normal memory pressure - operating normally")
case MemoryPressureModerate:
fmt.Println("Cache: Moderate memory pressure - clearing low-priority items")
// Simulate clearing 20% of cache
c.evictPercentage(20)
case MemoryPressureHigh:
fmt.Println("Cache: High memory pressure - clearing most items")
// Simulate clearing 60% of cache
c.evictPercentage(60)
case MemoryPressureCritical:
fmt.Println("Cache: CRITICAL memory pressure - clearing almost everything")
// Simulate clearing 90% of cache
c.evictPercentage(90)
}
}
// evictPercentage removes the specified percentage of cache entries
func (c *PressureAwareCache) evictPercentage(percentage int) {
if len(c.data) == 0 {
return
}
// Calculate how many items to remove
removeCount := len(c.data) * percentage / 100
if removeCount == 0 {
removeCount = 1
}
// Remove items
count := 0
for key := range c.data {
delete(c.data, key)
count++
if count >= removeCount {
break
}
}
fmt.Printf("Cache: Evicted %d items (%.1f%% of cache)\n",
count, float64(count)/float64(len(c.data)+count)*100)
}
// Set adds an item to the cache if memory pressure allows
func (c *PressureAwareCache) Set(key string, value []byte) bool {
c.mu.Lock()
defer c.mu.Unlock()
// Under critical pressure, only allow small items
if c.currentPressure == MemoryPressureCritical && len(value) > 1024 {
return false
}
// Under high pressure, only allow medium items
if c.currentPressure == MemoryPressureHigh && len(value) > 10*1024 {
return false
}
c.data[key] = value
return true
}
// Get retrieves an item from the cache
func (c *PressureAwareCache) Get(key string) ([]byte, bool) {
c.mu.RLock()
defer c.mu.RUnlock()
value, exists := c.data[key]
return value, exists
}
func main() {
// Create memory monitor
monitor := NewMemoryMonitor()
// Create cache
cache := NewPressureAwareCache()
// Register cache as listener for memory pressure changes
monitor.AddListener(cache.HandleMemoryPressureChange)
// Start monitoring
monitor.Start(1 * time.Second)
// Simulate application workload
go func() {
// Generate increasing memory pressure
data := make([][]byte, 0)
for i := 0; i < 20; i++ {
// Allocate memory in chunks
chunk := make([]byte, 10*1024*1024) // 10MB
for j := range chunk {
chunk[j] = byte(j % 256)
}
data = append(data, chunk)
// Try to add to cache
key := fmt.Sprintf("item-%d", i)
cacheItem := make([]byte, 1024*1024) // 1MB
if cache.Set(key, cacheItem) {
fmt.Printf("Added %s to cache\n", key)
} else {
fmt.Printf("Rejected %s due to memory pressure\n", key)
}
time.Sleep(500 * time.Millisecond)
}
// Release memory gradually
for i := 0; i < len(data); i += 2 {
data[i] = nil
time.Sleep(1 * time.Second)
}
}()
// Monitor and print memory stats
for i := 0; i < 30; i++ {
var stats runtime.MemStats
runtime.ReadMemStats(&stats)
fmt.Printf("\n--- Memory Stats #%d ---\n", i+1)
fmt.Printf("HeapAlloc: %d MB\n", stats.HeapAlloc/1024/1024)
fmt.Printf("HeapSys: %d MB\n", stats.HeapSys/1024/1024)
fmt.Printf("HeapObjects: %d\n", stats.HeapObjects)
fmt.Printf("GC cycles: %d\n", stats.NumGC)
time.Sleep(1 * time.Second)
}
}
Memory Leak Detection
Implement strategies to detect and address memory leaks:
package main
import (
"fmt"
"runtime"
"time"
)
// LeakDetector monitors memory usage to detect potential leaks
type LeakDetector struct {
sampleInterval time.Duration
alertThreshold float64 // percentage growth over baseline
baselineUsage uint64
samples []uint64
maxSamples int
}
// NewLeakDetector creates a new leak detector
func NewLeakDetector(sampleInterval time.Duration, alertThreshold float64, maxSamples int) *LeakDetector {
return &LeakDetector{
sampleInterval: sampleInterval,
alertThreshold: alertThreshold,
maxSamples: maxSamples,
samples: make([]uint64, 0, maxSamples),
}
}
// Start begins monitoring for memory leaks
func (d *LeakDetector) Start() {
// Take initial measurement after GC
runtime.GC()
var stats runtime.MemStats
runtime.ReadMemStats(&stats)
d.baselineUsage = stats.HeapAlloc
fmt.Printf("Leak detector started. Baseline memory usage: %d MB\n",
d.baselineUsage/1024/1024)
go func() {
ticker := time.NewTicker(d.sampleInterval)
defer ticker.Stop()
for range ticker.C {
d.takeSample()
}
}()
}
// takeSample takes a memory sample and analyzes for leaks
func (d *LeakDetector) takeSample() {
// Force GC to get accurate measurement
runtime.GC()
var stats runtime.MemStats
runtime.ReadMemStats(&stats)
// Add sample
d.samples = append(d.samples, stats.HeapAlloc)
if len(d.samples) > d.maxSamples {
d.samples = d.samples[1:]
}
// Calculate growth
currentUsage := stats.HeapAlloc
growthPercent := float64(currentUsage-d.baselineUsage) / float64(d.baselineUsage) * 100
fmt.Printf("Memory sample: %d MB (%.2f%% growth from baseline)\n",
currentUsage/1024/1024, growthPercent)
// Check for consistent growth pattern indicating a leak
if len(d.samples) >= 5 && growthPercent > d.alertThreshold {
isGrowing := true
for i := 1; i < len(d.samples); i++ {
if d.samples[i] <= d.samples[i-1] {
isGrowing = false
break
}
}
if isGrowing {
fmt.Printf("ALERT: Potential memory leak detected! Memory has grown by %.2f%%\n",
growthPercent)
fmt.Printf("Last %d samples (MB): ", len(d.samples))
for _, sample := range d.samples {
fmt.Printf("%.1f ", float64(sample)/1024/1024)
}
fmt.Println()
}
}
}
// Simulate a function with a memory leak
func leakyFunction() {
// Global slice that keeps growing
var leakyData [][]byte
for {
// Allocate memory that never gets freed
data := make([]byte, 1*1024*1024) // 1MB
leakyData = append(leakyData, data)
time.Sleep(500 * time.Millisecond)
}
}
// Simulate a function with normal memory usage
func normalFunction() {
for {
// Local variable that gets cleaned up
data := make([]byte, 10*1024*1024) // 10MB
// Use the data somehow
for i := 0; i < len(data); i += 1024 {
data[i] = byte(i % 256)
}
time.Sleep(500 * time.Millisecond)
}
}
func main() {
// Create and start leak detector
detector := NewLeakDetector(2*time.Second, 50.0, 10)
detector.Start()
// Uncomment to simulate a leak
// go leakyFunction()
// Normal memory usage
go normalFunction()
// Keep main goroutine running
time.Sleep(60 * time.Second)
}
Key Takeaways
Mastering Go’s memory management and garbage collector is essential for building high-performance applications that can scale effectively. By understanding the underlying mechanisms and applying advanced techniques, you can significantly improve your application’s performance and resource utilization.
The techniques we’ve explored in this article—from custom memory pools and zero-allocation strategies to GC tuning and sophisticated profiling—provide a comprehensive toolkit for optimizing memory usage in Go applications. Remember that memory optimization is often a balancing act between performance, complexity, and maintainability. The right approach depends on your specific requirements and constraints.
As Go continues to evolve, its memory management capabilities will likely improve further. However, the fundamental principles and techniques discussed here will remain valuable for developers seeking to push the boundaries of performance in their Go applications.
By applying these advanced memory management techniques and continuously monitoring your application’s behavior, you can build Go systems that not only perform well under normal conditions but also remain stable and responsive under heavy load—truly mastering performance at scale.