Performance: optimise file writes

cmd/goa/gen.go

@@ -13,6 +13,7 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
+	"time"
 
 	"goa.design/goa/v3/codegen"
 	"golang.org/x/tools/go/packages"
@@ -44,7 +45,7 @@ type Generator struct {
 }
 
 // NewGenerator creates a Generator.
-func NewGenerator(cmd, path, output string) *Generator {
+func NewGenerator(cmd, path, output string, debug bool) *Generator {
 	bin := "goa"
 	if runtime.GOOS == "windows" {
 		bin += ".exe"
@@ -55,7 +56,11 @@ func NewGenerator(cmd, path, output string) *Generator {
 	{
 		version = 2
 		matched := false
+		startPkgLoad := time.Now()
 		pkgs, _ := packages.Load(&packages.Config{Mode: packages.NeedFiles | packages.NeedModule}, path)
+		if debug {
+			fmt.Fprintf(os.Stderr, "[TIMING]   packages.Load (design files) took %v\n", time.Since(startPkgLoad))
+		}
 		fset := token.NewFileSet()
 		p := regexp.MustCompile(`goa.design/goa/v(\d+)/dsl`)
 		for _, pkg := range pkgs {
@@ -132,6 +137,7 @@ func (g *Generator) Write(_ bool) error {
 			codegen.SimpleImport("sort"),
 			codegen.SimpleImport("strconv"),
 			codegen.SimpleImport("strings"),
+			codegen.SimpleImport("time"),
 			codegen.SimpleImport("goa.design/goa/" + ver + "codegen"),
 			codegen.SimpleImport("goa.design/goa/" + ver + "codegen/generator"),
 			codegen.SimpleImport("goa.design/goa/" + ver + "eval"),
@@ -154,23 +160,36 @@ func (g *Generator) Write(_ bool) error {
 }
 
 // Compile compiles the generator.
-func (g *Generator) Compile() error {
+func (g *Generator) Compile(debug bool) error {
 	// We first need to go get the generated package to make sure that all
 	// dependencies are added to go.sum prior to compiling.
+	startLoad := time.Now()
 	pkgs, err := packages.Load(&packages.Config{Mode: packages.NeedName}, fmt.Sprintf(".%c%s", filepath.Separator, g.tmpDir))
 	if err != nil {
 		return err
 	}
 	if len(pkgs) != 1 {
 		return fmt.Errorf("expected to find one package in %s", g.tmpDir)
 	}
+	if debug {
+		fmt.Fprintf(os.Stderr, "[TIMING]   packages.Load (temp dir) took %v\n", time.Since(startLoad))
+	}
+
 	if !g.hasVendorDirectory {
+		startGet := time.Now()
 		if err := g.runGoCmd("get", pkgs[0].PkgPath); err != nil {
 			return err
 		}
+		if debug {
+			fmt.Fprintf(os.Stderr, "[TIMING]   go get took %v\n", time.Since(startGet))
+		}
 	}
 
+	startBuild := time.Now()
 	err = g.runGoCmd("build", "-o", g.bin)
+	if debug {
+		fmt.Fprintf(os.Stderr, "[TIMING]   go build took %v\n", time.Since(startBuild))
+	}
 
 	// If we're in vendor context we check the error string to see if it's an issue of unsatisfied dependencies
 	if err != nil && g.hasVendorDirectory {
@@ -183,7 +202,7 @@ func (g *Generator) Compile() error {
 }
 
 // Run runs the compiled binary and return the output lines.
-func (g *Generator) Run() ([]string, error) {
+func (g *Generator) Run(debug bool) ([]string, error) {
 	var cmdl string
 	{
 		args := make([]string, len(os.Args)-1)
@@ -210,7 +229,7 @@ func (g *Generator) Run() ([]string, error) {
 		cmdl = fmt.Sprintf("$ %s%s", rawcmd, cmdl)
 	}
 
-	args := []string{"--version=" + strconv.Itoa(g.DesignVersion), "--output=" + g.Output, "--cmd=" + cmdl}
+	args := []string{"--version=" + strconv.Itoa(g.DesignVersion), "--output=" + g.Output, "--cmd=" + cmdl, "--debug=" + strconv.FormatBool(debug)}
 	cmd := exec.Command(filepath.Join(g.tmpDir, g.bin), args...)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
@@ -291,6 +310,7 @@ const mainT = `func main() {
 		out     = flag.String("output", "", "")
 		version = flag.String("version", "", "")
 		cmdl    = flag.String("cmd", "", "")
+		debug   = flag.Bool("debug", false, "")
 		ver int
 	)
 	{
@@ -311,15 +331,31 @@ const mainT = `func main() {
 		ver = v
 	}
 
+	startBinary := time.Now()
+	if *debug {
+		fmt.Fprintf(os.Stderr, "[TIMING]   [binary] Starting generated binary execution\n")
+	}
+
 	if ver > goa.Major {
 		fail("cannot run goa %s on design using goa v%s\n", goa.Version(), *version)
 	}
+
+	startCheckErrors := time.Now()
 	if err := eval.Context.Errors; err != nil {
 		fail(err.Error())
 	}
+	if *debug {
+		fmt.Fprintf(os.Stderr, "[TIMING]   [binary] Check eval.Context.Errors took %v\n", time.Since(startCheckErrors))
+	}
+
+	startRunDSL := time.Now()
 	if err := eval.RunDSL(); err != nil {
 		fail(err.Error())
 	}
+	if *debug {
+		fmt.Fprintf(os.Stderr, "[TIMING]   [binary] eval.RunDSL() took %v\n", time.Since(startRunDSL))
+	}
+
 {{- range .CleanupDirs }}
 	if err := os.RemoveAll({{ printf "%q" . }}); err != nil {
 		fail(err.Error())
@@ -328,11 +364,16 @@ const mainT = `func main() {
 {{- if gt .DesignVersion 2 }}
 	codegen.DesignVersion = ver
 {{- end }}
-	outputs, err := generator.Generate(*out, {{ printf "%q" .Command }})
+
+	startGenerate := time.Now()
+	outputs, err := generator.Generate(*out, {{ printf "%q" .Command }}, *debug)
 	if err != nil {
 		fail(err.Error())
 	}
-
+	if *debug {
+		fmt.Fprintf(os.Stderr, "[TIMING]   [binary] generator.Generate() took %v\n", time.Since(startGenerate))
+		fmt.Fprintf(os.Stderr, "[TIMING]   [binary] Total binary execution took %v\n", time.Since(startBinary))
+	}
 	fmt.Println(strings.Join(outputs, "\n"))
 }
 

cmd/goa/main.go

@@ -5,6 +5,7 @@ import (
 	"go/build"
 	"os"
 	"strings"
+	"time"
 
 	"flag"
 
@@ -77,29 +78,55 @@ var (
 
 func generate(cmd, path, output string, debug bool) error {
 	var (
-		files []string
-		err   error
-		tmp   *Generator
+		files                                                            []string
+		err                                                              error
+		tmp                                                              *Generator
+		startTotal, startImport, startNewGen, startWrite, startCompile, startRun time.Time
 	)
 
+	startTotal = time.Now()
+	if debug {
+		fmt.Fprintf(os.Stderr, "[TIMING] Starting goa generation\n")
+	}
+
+	startImport = time.Now()
 	if _, err = build.Import(path, ".", 0); err != nil {
 		goto fail
 	}
+	if debug {
+		fmt.Fprintf(os.Stderr, "[TIMING] build.Import took %v\n", time.Since(startImport))
+	}
 
-	tmp = NewGenerator(cmd, path, output)
+	startNewGen = time.Now()
+	tmp = NewGenerator(cmd, path, output, debug)
+	if debug {
+		fmt.Fprintf(os.Stderr, "[TIMING] NewGenerator took %v\n", time.Since(startNewGen))
+	}
 
+	startWrite = time.Now()
 	if err = tmp.Write(debug); err != nil {
 		goto fail
 	}
+	if debug {
+		fmt.Fprintf(os.Stderr, "[TIMING] Write (generate main.go) took %v\n", time.Since(startWrite))
+	}
 
-	if err = tmp.Compile(); err != nil {
+	startCompile = time.Now()
+	if err = tmp.Compile(debug); err != nil {
 		goto fail
 	}
+	if debug {
+		fmt.Fprintf(os.Stderr, "[TIMING] Compile (go get + go build) took %v\n", time.Since(startCompile))
+	}
 
-	if files, err = tmp.Run(); err != nil {
+	startRun = time.Now()
+	if files, err = tmp.Run(debug); err != nil {
 		goto fail
 	}
-
+	if debug {
+		fmt.Fprintf(os.Stderr, "[TIMING] Run (execute binary) took %v\n", time.Since(startRun))
+		fmt.Fprintf(os.Stderr, "[TIMING] Total generation time: %v\n", time.Since(startTotal))
+	}
 	fmt.Println(strings.Join(files, "\n"))
 	if !debug {
 		tmp.Remove()

codegen/file.go

@@ -85,30 +85,28 @@ func (f *File) Render(dir string) (string, error) {
 		return "", err
 	}
 
-	file, err := os.OpenFile(
-		path,
-		os.O_CREATE|os.O_TRUNC|os.O_WRONLY,
-		0600,
-	)
-	if err != nil {
-		return "", err
-	}
+	// Render all sections to a buffer instead of directly to file
+	var buf bytes.Buffer
 	for _, s := range f.SectionTemplates {
-		if err := s.Write(file); err != nil {
+		if err := s.Write(&buf); err != nil {
 			return "", err
 		}
 	}
-	if err := file.Close(); err != nil {
-		return "", err
-	}
 
-	// Format Go source files
+	// For Go files, process everything in memory
+	content := buf.Bytes()
 	if filepath.Ext(path) == ".go" {
-		if err := finalizeGoSource(path); err != nil {
+		content, err = finalizeGoSource(path, content)
+		if err != nil {
 			return "", err
 		}
 	}
 
+	// Write the final content exactly once
+	if err := os.WriteFile(path, content, 0644); err != nil {
+		return "", err
+	}
+
 	// Run finalizer if any
 	if f.FinalizeFunc != nil {
 		if err := f.FinalizeFunc(path); err != nil {
@@ -127,47 +125,38 @@ func (s *SectionTemplate) Write(w io.Writer) error {
 	return tmpl.Execute(w, s.Data)
 }
 
-// finalizeGoSource removes unneeded imports from the given Go source file and
-// runs go fmt on it.
-func finalizeGoSource(path string) error {
-	// Make sure file parses and print content if it does not.
+// finalizeGoSource processes Go source entirely in memory without file I/O
+func finalizeGoSource(path string, content []byte) ([]byte, error) {
+	// Parse the content
 	fset := token.NewFileSet()
-	file, err := parser.ParseFile(fset, path, nil, parser.ParseComments)
+	file, err := parser.ParseFile(fset, path, content, parser.ParseComments)
 	if err != nil {
-		content, _ := os.ReadFile(path)
 		var buf bytes.Buffer
 		scanner.PrintError(&buf, err)
-		return fmt.Errorf("%s\n========\nContent:\n%s", buf.String(), content)
+		return nil, fmt.Errorf("%s\n========\nContent:\n%s", buf.String(), content)
 	}
 
 	// Clean unused imports using optimized single-pass detection
 	impMap := buildImportMap(file)
 	detectUsedImports(file, impMap)
 	removeUnusedImports(fset, file, impMap)
 	ast.SortImports(fset, file)
-	w, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
-	if err != nil {
-		return err
-	}
-	if err := format.Node(w, fset, file); err != nil {
-		return err
-	}
-	if err := w.Close(); err != nil {
-		return err
-	}
 
-	// Format code using goimport standard
-	bs, err := os.ReadFile(path)
-	if err != nil {
-		return err
+	// Format the AST back to bytes
+	var formatted bytes.Buffer
+	if err := format.Node(&formatted, fset, file); err != nil {
+		return nil, err
 	}
+
+	// Apply goimports formatting
 	opt := imports.Options{
 		Comments:   true,
 		FormatOnly: true,
 	}
-	bs, err = imports.Process(path, bs, &opt)
+	result, err := imports.Process(path, formatted.Bytes(), &opt)
 	if err != nil {
-		return err
+		return nil, err
 	}
-	return os.WriteFile(path, bs, os.ModePerm)
+
+	return result, nil
 }