Building Code Generation Tools
While the previous sections explored individual techniques for metaprogramming in Go, building production-ready code generation tools requires combining these approaches into cohesive, maintainable packages. In this section, we’ll examine how to design and implement robust code generators that can be used in real-world projects.
Designing a Code Generator
Effective code generators follow a clear architecture that separates concerns and promotes maintainability:
- Input Processing: Parsing and validating input (source code, configuration files, annotations)
- Model Building: Constructing an intermediate representation of the data
- Template Rendering: Generating output code using templates or AST manipulation
- Output Management: Writing files, formatting code, and managing dependencies
Let’s implement a complete code generator that follows this architecture:
package main
import (
"bytes"
"encoding/json"
"fmt"
"go/format"
"os"
"path/filepath"
"strings"
"text/template"
)
// ModelDefinition represents a data model definition
type ModelDefinition struct {
Name string `json:"name"`
Fields []FieldDefinition `json:"fields"`
}
// FieldDefinition represents a field in a model
type FieldDefinition struct {
Name string `json:"name"`
Type string `json:"type"`
Tags map[string]string `json:"tags"`
Required bool `json:"required"`
}
// Generator orchestrates the code generation process
type Generator struct {
Models []ModelDefinition
PackageName string
OutputDir string
Templates map[string]*template.Template
}
// NewGenerator creates a new code generator
func NewGenerator(packageName, outputDir string) *Generator {
return &Generator{
PackageName: packageName,
OutputDir: outputDir,
Templates: make(map[string]*template.Template),
}
}
// LoadModels loads model definitions from a JSON file
func (g *Generator) LoadModels(filename string) error {
data, err := os.ReadFile(filename)
if err != nil {
return fmt.Errorf("error reading model file: %w", err)
}
if err := json.Unmarshal(data, &g.Models); err != nil {
return fmt.Errorf("error parsing model file: %w", err)
}
// Validate models
for i, model := range g.Models {
if model.Name == "" {
return fmt.Errorf("model #%d has no name", i+1)
}
for j, field := range model.Fields {
if field.Name == "" {
return fmt.Errorf("field #%d in model %s has no name", j+1, model.Name)
}
if field.Type == "" {
return fmt.Errorf("field %s in model %s has no type", field.Name, model.Name)
}
}
}
return nil
}
// LoadTemplate loads a template from a file
func (g *Generator) LoadTemplate(name, filename string) error {
tmpl, err := template.New(name).Funcs(template.FuncMap{
"toLower": strings.ToLower,
"toUpper": strings.ToUpper,
"title": strings.Title,
"join": strings.Join,
}).ParseFiles(filename)
if err != nil {
return fmt.Errorf("error loading template %s: %w", name, err)
}
g.Templates[name] = tmpl
return nil
}
// GenerateModels generates model files
func (g *Generator) GenerateModels() error {
// Ensure output directory exists
if err := os.MkdirAll(g.OutputDir, 0755); err != nil {
return fmt.Errorf("error creating output directory: %w", err)
}
// Generate model files
for _, model := range g.Models {
if err := g.generateModelFile(model); err != nil {
return fmt.Errorf("error generating model %s: %w", model.Name, err)
}
}
return nil
}
// generateModelFile generates a single model file
func (g *Generator) generateModelFile(model ModelDefinition) error {
// Prepare template data
data := struct {
Package string
Model ModelDefinition
}{
Package: g.PackageName,
Model: model,
}
// Generate model code
var buf bytes.Buffer
if err := g.Templates["model"].Execute(&buf, data); err != nil {
return fmt.Errorf("error executing template: %w", err)
}
// Format the generated code
formattedCode, err := format.Source(buf.Bytes())
if err != nil {
// If formatting fails, still write the unformatted code for debugging
fmt.Printf("Warning: formatting failed for %s: %v\n", model.Name, err)
formattedCode = buf.Bytes()
}
// Write the file
filename := filepath.Join(g.OutputDir, strings.ToLower(model.Name)+".go")
if err := os.WriteFile(filename, formattedCode, 0644); err != nil {
return fmt.Errorf("error writing file: %w", err)
}
fmt.Printf("Generated %s\n", filename)
return nil
}
func main() {
if len(os.Args) != 4 {
fmt.Println("Usage: modelgen <package-name> <models-file> <output-dir>")
os.Exit(1)
}
packageName := os.Args[1]
modelsFile := os.Args[2]
outputDir := os.Args[3]
// Create generator
generator := NewGenerator(packageName, outputDir)
// Load models
if err := generator.LoadModels(modelsFile); err != nil {
fmt.Printf("Error loading models: %v\n", err)
os.Exit(1)
}
// Load templates
templateFile := "model_template.tmpl"
if err := generator.LoadTemplate("model", templateFile); err != nil {
fmt.Printf("Error loading template: %v\n", err)
os.Exit(1)
}
// Generate code
if err := generator.GenerateModels(); err != nil {
fmt.Printf("Error generating models: %v\n", err)
os.Exit(1)
}
fmt.Println("Code generation completed successfully")
}
This generator reads model definitions from a JSON file and generates Go struct definitions using templates. Let’s look at an example model definition file:
[
{
"name": "User",
"fields": [
{
"name": "ID",
"type": "string",
"tags": {
"json": "id",
"db": "id"
},
"required": true
},
{
"name": "Name",
"type": "string",
"tags": {
"json": "name",
"db": "name"
},
"required": true
},
{
"name": "Email",
"type": "string",
"tags": {
"json": "email",
"db": "email"
},
"required": true
},
{
"name": "CreatedAt",
"type": "time.Time",
"tags": {
"json": "created_at",
"db": "created_at"
},
"required": false
}
]
},
{
"name": "Product",
"fields": [
{
"name": "ID",
"type": "string",
"tags": {
"json": "id",
"db": "id"
},
"required": true
},
{
"name": "Name",
"type": "string",
"tags": {
"json": "name",
"db": "name"
},
"required": true
},
{
"name": "Price",
"type": "float64",
"tags": {
"json": "price",
"db": "price"
},
"required": true
},
{
"name": "Description",
"type": "string",
"tags": {
"json": "description,omitempty",
"db": "description"
},
"required": false
}
]
}
]
And a template file for generating model structs:
// Code generated by modelgen; DO NOT EDIT.
package {{.Package}}
import (
"time"
)
// {{.Model.Name}} represents a {{toLower .Model.Name}} entity
type {{.Model.Name}} struct {
{{- range .Model.Fields}}
{{.Name}} {{.Type}} `{{range $key, $value := .Tags}}{{$key}}:"{{$value}}" {{end}}`
{{- end}}
}
// Validate checks if the {{.Model.Name}} is valid
func (m *{{.Model.Name}}) Validate() error {
{{- range .Model.Fields}}
{{- if .Required}}
if {{if eq .Type "string"}}m.{{.Name}} == ""{{else if eq .Type "int"}}m.{{.Name}} == 0{{else if eq .Type "float64"}}m.{{.Name}} == 0{{else}}m.{{.Name}} == nil{{end}} {
return fmt.Errorf("{{.Name}} is required")
}
{{- end}}
{{- end}}
return nil
}
// ToMap converts the {{.Model.Name}} to a map
func (m *{{.Model.Name}}) ToMap() map[string]interface{} {
return map[string]interface{}{
{{- range .Model.Fields}}
"{{index .Tags "json"}}": m.{{.Name}},
{{- end}}
}
}
Running the generator would produce Go files for each model, complete with validation and utility methods.
Integrating with Build Systems
To make code generation a seamless part of the development workflow, it should be integrated with the build system. For Go projects, this typically means using go generate directives:
//go:generate go run github.com/example/modelgen models models.json ./generated
For more complex scenarios, a Makefile can orchestrate the generation process:
.PHONY: generate
generate:
@echo "Generating models..."
@go run ./cmd/modelgen models models.json ./generated
@echo "Generating API clients..."
@go run ./cmd/apigen api api.yaml ./generated/api
@echo "Code generation complete"
.PHONY: build
build: generate
@echo "Building application..."
@go build -o app ./cmd/app
Combining Multiple Generation Techniques
Real-world code generators often combine multiple techniques for maximum flexibility:
package main
import (
"bytes"
"fmt"
"go/ast"
"go/format"
"go/parser"
"go/token"
"os"
"path/filepath"
"strings"
"text/template"
)
// Generator combines AST parsing and template rendering
type Generator struct {
SourceDir string
OutputDir string
Templates map[string]*template.Template
ParsedASTs map[string]*ast.File
FileSet *token.FileSet
}
// NewGenerator creates a new generator
func NewGenerator(sourceDir, outputDir string) *Generator {
return &Generator{
SourceDir: sourceDir,
OutputDir: outputDir,
Templates: make(map[string]*template.Template),
ParsedASTs: make(map[string]*ast.File),
FileSet: token.NewFileSet(),
}
}
// ParseSourceFiles parses Go source files in the source directory
func (g *Generator) ParseSourceFiles() error {
return filepath.Walk(g.SourceDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
// Skip directories and non-Go files
if info.IsDir() || !strings.HasSuffix(path, ".go") {
return nil
}
// Parse the file
file, err := parser.ParseFile(g.FileSet, path, nil, parser.ParseComments)
if err != nil {
return fmt.Errorf("error parsing %s: %w", path, err)
}
// Store the AST
relPath, _ := filepath.Rel(g.SourceDir, path)
g.ParsedASTs[relPath] = file
return nil
})
}
// LoadTemplate loads a template from a file
func (g *Generator) LoadTemplate(name, filename string) error {
tmpl, err := template.New(name).Funcs(template.FuncMap{
"toLower": strings.ToLower,
"toUpper": strings.ToUpper,
"title": strings.Title,
}).ParseFiles(filename)
if err != nil {
return fmt.Errorf("error loading template %s: %w", name, err)
}
g.Templates[name] = tmpl
return nil
}
// FindInterfaces finds all interfaces in the parsed files
func (g *Generator) FindInterfaces() map[string]*ast.InterfaceType {
interfaces := make(map[string]*ast.InterfaceType)
for _, file := range g.ParsedASTs {
ast.Inspect(file, func(n ast.Node) bool {
// Look for type declarations
if typeSpec, ok := n.(*ast.TypeSpec); ok {
// Check if it's an interface
if ifaceType, ok := typeSpec.Type.(*ast.InterfaceType); ok {
interfaces[typeSpec.Name.Name] = ifaceType
}
}
return true
})
}
return interfaces
}
// GenerateMocks generates mock implementations for interfaces
func (g *Generator) GenerateMocks() error {
// Ensure output directory exists
if err := os.MkdirAll(g.OutputDir, 0755); err != nil {
return fmt.Errorf("error creating output directory: %w", err)
}
// Find interfaces
interfaces := g.FindInterfaces()
// Generate mock for each interface
for name, iface := range interfaces {
if err := g.generateMock(name, iface); err != nil {
return fmt.Errorf("error generating mock for %s: %w", name, err)
}
}
return nil
}
// generateMock generates a mock implementation for an interface
func (g *Generator) generateMock(name string, iface *ast.InterfaceType) error {
// Extract methods from the interface
methods := make([]Method, 0)
for _, method := range iface.Methods.List {
if len(method.Names) == 0 {
continue // Skip embedded interfaces for simplicity
}
methodName := method.Names[0].Name
funcType, ok := method.Type.(*ast.FuncType)
if !ok {
continue
}
// Extract parameters
params := make([]Parameter, 0)
if funcType.Params != nil {
for i, param := range funcType.Params.List {
paramType := g.FileSet.Position(param.Type.Pos()).String()
paramName := fmt.Sprintf("arg%d", i)
if len(param.Names) > 0 {
paramName = param.Names[0].Name
}
params = append(params, Parameter{
Name: paramName,
Type: paramType,
})
}
}
// Extract results
results := make([]Parameter, 0)
if funcType.Results != nil {
for i, result := range funcType.Results.List {
resultType := g.FileSet.Position(result.Type.Pos()).String()
resultName := fmt.Sprintf("result%d", i)
if len(result.Names) > 0 {
resultName = result.Names[0].Name
}
results = append(results, Parameter{
Name: resultName,
Type: resultType,
})
}
}
methods = append(methods, Method{
Name: methodName,
Params: params,
Results: results,
})
}
// Prepare template data
data := struct {
Name string
Methods []Method
}{
Name: name,
Methods: methods,
}
// Generate mock code
var buf bytes.Buffer
if err := g.Templates["mock"].Execute(&buf, data); err != nil {
return fmt.Errorf("error executing template: %w", err)
}
// Format the generated code
formattedCode, err := format.Source(buf.Bytes())
if err != nil {
// If formatting fails, still write the unformatted code for debugging
fmt.Printf("Warning: formatting failed for %s: %v\n", name, err)
formattedCode = buf.Bytes()
}
// Write the file
filename := filepath.Join(g.OutputDir, strings.ToLower(name)+"_mock.go")
if err := os.WriteFile(filename, formattedCode, 0644); err != nil {
return fmt.Errorf("error writing file: %w", err)
}
fmt.Printf("Generated mock for %s: %s\n", name, filename)
return nil
}
// Method represents a method in an interface
type Method struct {
Name string
Params []Parameter
Results []Parameter
}
// Parameter represents a parameter or result in a method
type Parameter struct {
Name string
Type string
}
func main() {
if len(os.Args) != 4 {
fmt.Println("Usage: mockgen <source-dir> <template-file> <output-dir>")
os.Exit(1)
}
sourceDir := os.Args[1]
templateFile := os.Args[2]
outputDir := os.Args[3]
// Create generator
generator := NewGenerator(sourceDir, outputDir)
// Parse source files
if err := generator.ParseSourceFiles(); err != nil {
fmt.Printf("Error parsing source files: %v\n", err)
os.Exit(1)
}
// Load template
if err := generator.LoadTemplate("mock", templateFile); err != nil {
fmt.Printf("Error loading template: %v\n", err)
os.Exit(1)
}
// Generate mocks
if err := generator.GenerateMocks(); err != nil {
fmt.Printf("Error generating mocks: %v\n", err)
os.Exit(1)
}
fmt.Println("Mock generation completed successfully")
}
This generator combines AST parsing to extract interface definitions with template rendering to generate mock implementations. It demonstrates how different metaprogramming techniques can be combined to create powerful code generation tools.
Testing Generated Code
Generated code should be tested just like hand-written code. There are several approaches to testing code generators:
- Unit testing the generator: Test the generator’s logic to ensure it produces the expected output for various inputs.
func TestGenerateMock(t *testing.T) {
// Create a temporary directory for output
tempDir, err := os.MkdirTemp("", "mockgen-test")
if err != nil {
t.Fatalf("Failed to create temp dir: %v", err)
}
defer os.RemoveAll(tempDir)
// Create a generator with test inputs
generator := NewGenerator("./testdata", tempDir)
// Parse source files
if err := generator.ParseSourceFiles(); err != nil {
t.Fatalf("Failed to parse source files: %v", err)
}
// Load template
if err := generator.LoadTemplate("mock", "./testdata/mock_template.tmpl"); err != nil {
t.Fatalf("Failed to load template: %v", err)
}
// Generate mocks
if err := generator.GenerateMocks(); err != nil {
t.Fatalf("Failed to generate mocks: %v", err)
}
// Check that the expected files were generated
expectedFiles := []string{"repository_mock.go", "service_mock.go"}
for _, file := range expectedFiles {
path := filepath.Join(tempDir, file)
if _, err := os.Stat(path); os.IsNotExist(err) {
t.Errorf("Expected file %s was not generated", file)
}
}
// Check the content of generated files
for _, file := range expectedFiles {
path := filepath.Join(tempDir, file)
content, err := os.ReadFile(path)
if err != nil {
t.Fatalf("Failed to read generated file %s: %v", file, err)
}
// Verify that the file compiles
cmd := exec.Command("go", "build", "-o", "/dev/null", path)
if output, err := cmd.CombinedOutput(); err != nil {
t.Errorf("Generated file %s does not compile: %v\n%s", file, err, output)
}
// Verify specific content (simplified example)
if !bytes.Contains(content, []byte("// Code generated by mockgen")) {
t.Errorf("Generated file %s is missing the generated code comment", file)
}
}
}
- Golden file testing: Compare generated output to known-good “golden” files.
func TestGenerateModelAgainstGolden(t *testing.T) {
// Create a temporary directory for output
tempDir, err := os.MkdirTemp("", "modelgen-test")
if err != nil {
t.Fatalf("Failed to create temp dir: %v", err)
}
defer os.RemoveAll(tempDir)
// Create a generator with test inputs
generator := NewGenerator("models", tempDir)
// Load models
if err := generator.LoadModels("./testdata/test_models.json"); err != nil {
t.Fatalf("Failed to load models: %v", err)
}
// Load template
if err := generator.LoadTemplate("model", "./testdata/model_template.tmpl"); err != nil {
t.Fatalf("Failed to load template: %v", err)
}
// Generate models
if err := generator.GenerateModels(); err != nil {
t.Fatalf("Failed to generate models: %v", err)
}
// Compare with golden files
files, err := filepath.Glob(filepath.Join(tempDir, "*.go"))
if err != nil {
t.Fatalf("Failed to list generated files: %v", err)
}
for _, file := range files {
baseName := filepath.Base(file)
goldenFile := filepath.Join("./testdata/golden", baseName)
// Read generated file
generated, err := os.ReadFile(file)
if err != nil {
t.Fatalf("Failed to read generated file %s: %v", file, err)
}
// Read golden file
golden, err := os.ReadFile(goldenFile)
if err != nil {
t.Fatalf("Failed to read golden file %s: %v", goldenFile, err)
}
// Compare content
if !bytes.Equal(generated, golden) {
t.Errorf("Generated file %s does not match golden file", baseName)
// For detailed diff, use a diff library or write the diff to a file
}
}
}
- Integration testing: Test the generated code in the context of the application.
func TestGeneratedRepositories(t *testing.T) {
// This test assumes the code has already been generated
// Set up a test database
db, err := setupTestDatabase()
if err != nil {
t.Fatalf("Failed to set up test database: %v", err)
}
defer db.Close()
// Create instances of the generated repositories
userRepo := NewUserRepository(db)
productRepo := NewProductRepository(db)
// Test the generated repositories
t.Run("UserRepository", func(t *testing.T) {
// Create a user
user := &User{
Name: "Test User",
Email: "[email protected]",
}
// Test Create method
id, err := userRepo.Create(context.Background(), user)
if err != nil {
t.Fatalf("Failed to create user: %v", err)
}
// Test Get method
retrieved, err := userRepo.Get(context.Background(), id)
if err != nil {
t.Fatalf("Failed to get user: %v", err)
}
if retrieved.Name != user.Name || retrieved.Email != user.Email {
t.Errorf("Retrieved user does not match created user")
}
})
// Similar tests for other repositories...
}
Real-World Code Generation Examples
Many popular Go projects use code generation extensively:
-
Protocol Buffers and gRPC: The
protoc-gen-gotool generates Go code from.protofiles, including message types, serialization code, and gRPC client/server interfaces. -
go-swagger: Generates Go server and client code from OpenAPI/Swagger specifications.
-
sqlc: Generates type-safe Go code from SQL queries, providing a compile-time checked database layer.
-
mockgen: Generates mock implementations of interfaces for testing.
-
stringer: Generates String() methods for enum types.
These tools demonstrate the power of code generation for reducing boilerplate, ensuring consistency, and improving type safety in Go applications.
Building effective code generation tools requires careful design, thorough testing, and seamless integration with the development workflow. When done right, code generation can significantly improve developer productivity and code quality.