1// Copyright 2019 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5//go:generate go run . -execute 6 7package main 8 9import ( 10 "bytes" 11 "flag" 12 "fmt" 13 "go/format" 14 "io/ioutil" 15 "os" 16 "os/exec" 17 "path" 18 "path/filepath" 19 "regexp" 20 "sort" 21 "strconv" 22 "strings" 23 24 gengo "google.golang.org/protobuf/cmd/protoc-gen-go/internal_gengo" 25 "google.golang.org/protobuf/compiler/protogen" 26 "google.golang.org/protobuf/internal/detrand" 27) 28 29func init() { 30 // Determine repository root path. 31 out, err := exec.Command("git", "rev-parse", "--show-toplevel").CombinedOutput() 32 check(err) 33 repoRoot = strings.TrimSpace(string(out)) 34 35 // Determine the module path. 36 cmd := exec.Command("go", "list", "-m", "-f", "{{.Path}}") 37 cmd.Dir = repoRoot 38 out, err = cmd.CombinedOutput() 39 check(err) 40 modulePath = strings.TrimSpace(string(out)) 41 42 // When the environment variable RUN_AS_PROTOC_PLUGIN is set, 43 // we skip running main and instead act as a protoc plugin. 44 // This allows the binary to pass itself to protoc. 45 if plugin := os.Getenv("RUN_AS_PROTOC_PLUGIN"); plugin != "" { 46 // Disable deliberate output instability for generated files. 47 // This is reasonable since we fully control the output. 48 detrand.Disable() 49 50 protogen.Options{}.Run(func(gen *protogen.Plugin) error { 51 for _, file := range gen.Files { 52 if file.Generate { 53 gengo.GenerateVersionMarkers = false 54 gengo.GenerateFile(gen, file) 55 generateIdentifiers(gen, file) 56 generateSourceContextStringer(gen, file) 57 } 58 } 59 gen.SupportedFeatures = gengo.SupportedFeatures 60 return nil 61 }) 62 os.Exit(0) 63 } 64} 65 66var ( 67 run bool 68 protoRoot string 69 repoRoot string 70 modulePath string 71 72 generatedPreamble = []string{ 73 "// Copyright 2019 The Go Authors. All rights reserved.", 74 "// Use of this source code is governed by a BSD-style", 75 "// license that can be found in the LICENSE file.", 76 "", 77 "// Code generated by generate-protos. DO NOT EDIT.", 78 "", 79 } 80) 81 82func main() { 83 flag.BoolVar(&run, "execute", false, "Write generated files to destination.") 84 flag.StringVar(&protoRoot, "protoroot", os.Getenv("PROTOBUF_ROOT"), "The root of the protobuf source tree.") 85 flag.Parse() 86 if protoRoot == "" { 87 panic("protobuf source root is not set") 88 } 89 90 generateLocalProtos() 91 generateRemoteProtos() 92} 93 94func generateLocalProtos() { 95 tmpDir, err := ioutil.TempDir(repoRoot, "tmp") 96 check(err) 97 defer os.RemoveAll(tmpDir) 98 99 // Generate all local proto files (except version-locked files). 100 dirs := []struct { 101 path string 102 pkgPaths map[string]string // mapping of .proto path to Go package path 103 annotate map[string]bool // .proto files to annotate 104 exclude map[string]bool // .proto files to exclude from generation 105 }{{ 106 path: "cmd/protoc-gen-go/testdata", 107 pkgPaths: map[string]string{"cmd/protoc-gen-go/testdata/nopackage/nopackage.proto": "google.golang.org/protobuf/cmd/protoc-gen-go/testdata/nopackage"}, 108 annotate: map[string]bool{"cmd/protoc-gen-go/testdata/annotations/annotations.proto": true}, 109 }, { 110 path: "internal/testprotos", 111 exclude: map[string]bool{"internal/testprotos/irregular/irregular.proto": true}, 112 }} 113 excludeRx := regexp.MustCompile(`legacy/.*/`) 114 for _, d := range dirs { 115 subDirs := map[string]bool{} 116 117 srcDir := filepath.Join(repoRoot, filepath.FromSlash(d.path)) 118 filepath.Walk(srcDir, func(srcPath string, _ os.FileInfo, _ error) error { 119 if !strings.HasSuffix(srcPath, ".proto") || excludeRx.MatchString(srcPath) { 120 return nil 121 } 122 relPath, err := filepath.Rel(repoRoot, srcPath) 123 check(err) 124 125 srcRelPath, err := filepath.Rel(srcDir, srcPath) 126 check(err) 127 subDirs[filepath.Dir(srcRelPath)] = true 128 129 if d.exclude[filepath.ToSlash(relPath)] { 130 return nil 131 } 132 133 opts := "module=" + modulePath 134 for protoPath, goPkgPath := range d.pkgPaths { 135 opts += fmt.Sprintf(",M%v=%v", protoPath, goPkgPath) 136 } 137 if d.annotate[filepath.ToSlash(relPath)] { 138 opts += ",annotate_code" 139 } 140 protoc("-I"+filepath.Join(protoRoot, "src"), "-I"+repoRoot, "--go_out="+opts+":"+tmpDir, relPath) 141 return nil 142 }) 143 144 // For directories in testdata, generate a test that links in all 145 // generated packages to ensure that it builds and initializes properly. 146 // This is done because "go build ./..." does not build sub-packages 147 // under testdata. 148 if filepath.Base(d.path) == "testdata" { 149 var imports []string 150 for sd := range subDirs { 151 imports = append(imports, fmt.Sprintf("_ %q", path.Join(modulePath, d.path, filepath.ToSlash(sd)))) 152 } 153 sort.Strings(imports) 154 155 s := strings.Join(append(generatedPreamble, []string{ 156 "package main", 157 "", 158 "import (" + strings.Join(imports, "\n") + ")", 159 }...), "\n") 160 b, err := format.Source([]byte(s)) 161 check(err) 162 check(ioutil.WriteFile(filepath.Join(tmpDir, filepath.FromSlash(d.path+"/gen_test.go")), b, 0664)) 163 } 164 } 165 166 syncOutput(repoRoot, tmpDir) 167} 168 169func generateRemoteProtos() { 170 tmpDir, err := ioutil.TempDir(repoRoot, "tmp") 171 check(err) 172 defer os.RemoveAll(tmpDir) 173 174 // Generate all remote proto files. 175 files := []struct{ prefix, path, goPkgPath string }{ 176 // Well-known protos. 177 {"src", "google/protobuf/any.proto", ""}, 178 {"src", "google/protobuf/api.proto", ""}, 179 {"src", "google/protobuf/duration.proto", ""}, 180 {"src", "google/protobuf/empty.proto", ""}, 181 {"src", "google/protobuf/field_mask.proto", ""}, 182 {"src", "google/protobuf/source_context.proto", ""}, 183 {"src", "google/protobuf/struct.proto", ""}, 184 {"src", "google/protobuf/timestamp.proto", ""}, 185 {"src", "google/protobuf/type.proto", ""}, 186 {"src", "google/protobuf/wrappers.proto", ""}, 187 188 // Compiler protos. 189 {"src", "google/protobuf/compiler/plugin.proto", ""}, 190 {"src", "google/protobuf/descriptor.proto", ""}, 191 192 // Conformance protos. 193 {"", "conformance/conformance.proto", "google.golang.org/protobuf/internal/testprotos/conformance;conformance"}, 194 {"src", "google/protobuf/test_messages_proto2.proto", "google.golang.org/protobuf/internal/testprotos/conformance;conformance"}, 195 {"src", "google/protobuf/test_messages_proto3.proto", "google.golang.org/protobuf/internal/testprotos/conformance;conformance"}, 196 197 // Benchmark protos. 198 // TODO: The protobuf repo no longer includes benchmarks. 199 // CL removing them says they are superceded by google/fleetbench: 200 // https://github.com/protocolbuffers/protobuf/commit/83c499de86224538e5d59adc3d0fa7fdb45b2c72 201 // But that project's proto benchmark files are very different: 202 // https://github.com/google/fleetbench/tree/main/fleetbench/proto 203 // For now, commenting these out until benchmarks in this repo can be 204 // reconciled with new fleetbench stuff. 205 //{"benchmarks", "benchmarks.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks;benchmarks"}, 206 //{"benchmarks", "datasets/google_message1/proto2/benchmark_message1_proto2.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message1/proto2;proto2"}, 207 //{"benchmarks", "datasets/google_message1/proto3/benchmark_message1_proto3.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message1/proto3;proto3"}, 208 //{"benchmarks", "datasets/google_message2/benchmark_message2.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message2;google_message2"}, 209 //{"benchmarks", "datasets/google_message3/benchmark_message3.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"}, 210 //{"benchmarks", "datasets/google_message3/benchmark_message3_1.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"}, 211 //{"benchmarks", "datasets/google_message3/benchmark_message3_2.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"}, 212 //{"benchmarks", "datasets/google_message3/benchmark_message3_3.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"}, 213 //{"benchmarks", "datasets/google_message3/benchmark_message3_4.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"}, 214 //{"benchmarks", "datasets/google_message3/benchmark_message3_5.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"}, 215 //{"benchmarks", "datasets/google_message3/benchmark_message3_6.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"}, 216 //{"benchmarks", "datasets/google_message3/benchmark_message3_7.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"}, 217 //{"benchmarks", "datasets/google_message3/benchmark_message3_8.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"}, 218 //{"benchmarks", "datasets/google_message4/benchmark_message4.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message4;google_message4"}, 219 //{"benchmarks", "datasets/google_message4/benchmark_message4_1.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message4;google_message4"}, 220 //{"benchmarks", "datasets/google_message4/benchmark_message4_2.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message4;google_message4"}, 221 //{"benchmarks", "datasets/google_message4/benchmark_message4_3.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message4;google_message4"}, 222 } 223 224 opts := "module=" + modulePath 225 for _, file := range files { 226 if file.goPkgPath != "" { 227 opts += fmt.Sprintf(",M%v=%v", file.path, file.goPkgPath) 228 } 229 } 230 for _, f := range files { 231 protoc("-I"+filepath.Join(protoRoot, f.prefix), "--go_out="+opts+":"+tmpDir, f.path) 232 } 233 234 syncOutput(repoRoot, tmpDir) 235} 236 237func protoc(args ...string) { 238 // TODO: Remove --experimental_allow_proto3_optional flag. 239 cmd := exec.Command("protoc", "--plugin=protoc-gen-go="+os.Args[0], "--experimental_allow_proto3_optional") 240 cmd.Args = append(cmd.Args, args...) 241 cmd.Env = append(os.Environ(), "RUN_AS_PROTOC_PLUGIN=1") 242 out, err := cmd.CombinedOutput() 243 if err != nil { 244 fmt.Printf("executing: %v\n%s\n", strings.Join(cmd.Args, " "), out) 245 } 246 check(err) 247} 248 249// generateIdentifiers generates an internal package for descriptor.proto 250// and well-known types. 251func generateIdentifiers(gen *protogen.Plugin, file *protogen.File) { 252 if file.Desc.Package() != "google.protobuf" { 253 return 254 } 255 256 importPath := modulePath + "/internal/genid" 257 base := strings.TrimSuffix(path.Base(file.Desc.Path()), ".proto") 258 g := gen.NewGeneratedFile(importPath+"/"+base+"_gen.go", protogen.GoImportPath(importPath)) 259 for _, s := range generatedPreamble { 260 g.P(s) 261 } 262 g.P("package ", path.Base(importPath)) 263 g.P() 264 265 g.P("const ", file.GoDescriptorIdent.GoName, " = ", strconv.Quote(file.Desc.Path())) 266 g.P() 267 268 var processEnums func([]*protogen.Enum) 269 var processMessages func([]*protogen.Message) 270 const protoreflectPackage = protogen.GoImportPath("google.golang.org/protobuf/reflect/protoreflect") 271 processEnums = func(enums []*protogen.Enum) { 272 for _, enum := range enums { 273 g.P("// Full and short names for ", enum.Desc.FullName(), ".") 274 g.P("const (") 275 g.P(enum.GoIdent.GoName, "_enum_fullname = ", strconv.Quote(string(enum.Desc.FullName()))) 276 g.P(enum.GoIdent.GoName, "_enum_name = ", strconv.Quote(string(enum.Desc.Name()))) 277 g.P(")") 278 g.P() 279 } 280 } 281 processMessages = func(messages []*protogen.Message) { 282 for _, message := range messages { 283 g.P("// Names for ", message.Desc.FullName(), ".") 284 g.P("const (") 285 g.P(message.GoIdent.GoName, "_message_name ", protoreflectPackage.Ident("Name"), " = ", strconv.Quote(string(message.Desc.Name()))) 286 g.P(message.GoIdent.GoName, "_message_fullname ", protoreflectPackage.Ident("FullName"), " = ", strconv.Quote(string(message.Desc.FullName()))) 287 g.P(")") 288 g.P() 289 290 if len(message.Fields) > 0 { 291 g.P("// Field names for ", message.Desc.FullName(), ".") 292 g.P("const (") 293 for _, field := range message.Fields { 294 g.P(message.GoIdent.GoName, "_", field.GoName, "_field_name ", protoreflectPackage.Ident("Name"), " = ", strconv.Quote(string(field.Desc.Name()))) 295 } 296 g.P() 297 for _, field := range message.Fields { 298 g.P(message.GoIdent.GoName, "_", field.GoName, "_field_fullname ", protoreflectPackage.Ident("FullName"), " = ", strconv.Quote(string(field.Desc.FullName()))) 299 } 300 g.P(")") 301 g.P() 302 303 g.P("// Field numbers for ", message.Desc.FullName(), ".") 304 g.P("const (") 305 for _, field := range message.Fields { 306 g.P(message.GoIdent.GoName, "_", field.GoName, "_field_number ", protoreflectPackage.Ident("FieldNumber"), " = ", field.Desc.Number()) 307 } 308 g.P(")") 309 g.P() 310 } 311 312 if len(message.Oneofs) > 0 { 313 g.P("// Oneof names for ", message.Desc.FullName(), ".") 314 g.P("const (") 315 for _, oneof := range message.Oneofs { 316 g.P(message.GoIdent.GoName, "_", oneof.GoName, "_oneof_name ", protoreflectPackage.Ident("Name"), " = ", strconv.Quote(string(oneof.Desc.Name()))) 317 } 318 g.P() 319 for _, oneof := range message.Oneofs { 320 g.P(message.GoIdent.GoName, "_", oneof.GoName, "_oneof_fullname ", protoreflectPackage.Ident("FullName"), " = ", strconv.Quote(string(oneof.Desc.FullName()))) 321 } 322 g.P(")") 323 g.P() 324 } 325 326 processEnums(message.Enums) 327 processMessages(message.Messages) 328 } 329 } 330 processEnums(file.Enums) 331 processMessages(file.Messages) 332} 333 334// generateSourceContextStringer generates the implementation for the 335// protoreflect.SourcePath.String method by using information present 336// in the descriptor.proto. 337func generateSourceContextStringer(gen *protogen.Plugin, file *protogen.File) { 338 if file.Desc.Path() != "google/protobuf/descriptor.proto" { 339 return 340 } 341 342 importPath := modulePath + "/reflect/protoreflect" 343 g := gen.NewGeneratedFile(importPath+"/source_gen.go", protogen.GoImportPath(importPath)) 344 for _, s := range generatedPreamble { 345 g.P(s) 346 } 347 g.P("package ", path.Base(importPath)) 348 g.P() 349 350 var messages []*protogen.Message 351 for _, message := range file.Messages { 352 if message.Desc.Name() == "FileDescriptorProto" { 353 messages = append(messages, message) 354 } 355 } 356 seen := make(map[*protogen.Message]bool) 357 358 for len(messages) > 0 { 359 m := messages[0] 360 messages = messages[1:] 361 if seen[m] { 362 continue 363 } 364 seen[m] = true 365 366 g.P("func (p *SourcePath) append", m.GoIdent.GoName, "(b []byte) []byte {") 367 g.P("if len(*p) == 0 { return b }") 368 g.P("switch (*p)[0] {") 369 for _, f := range m.Fields { 370 g.P("case ", f.Desc.Number(), ":") 371 var cardinality string 372 switch { 373 case f.Desc.IsMap(): 374 panic("maps are not supported") 375 case f.Desc.IsList(): 376 cardinality = "Repeated" 377 default: 378 cardinality = "Singular" 379 } 380 nextAppender := "nil" 381 if f.Message != nil { 382 nextAppender = "(*SourcePath).append" + f.Message.GoIdent.GoName 383 messages = append(messages, f.Message) 384 } 385 g.P("b = p.append", cardinality, "Field(b, ", strconv.Quote(string(f.Desc.Name())), ", ", nextAppender, ")") 386 } 387 g.P("}") 388 g.P("return b") 389 g.P("}") 390 g.P() 391 } 392} 393 394func syncOutput(dstDir, srcDir string) { 395 filepath.Walk(srcDir, func(srcPath string, _ os.FileInfo, _ error) error { 396 if !strings.HasSuffix(srcPath, ".go") && !strings.HasSuffix(srcPath, ".meta") { 397 return nil 398 } 399 relPath, err := filepath.Rel(srcDir, srcPath) 400 check(err) 401 dstPath := filepath.Join(dstDir, relPath) 402 403 if run { 404 if copyFile(dstPath, srcPath) { 405 fmt.Println("#", relPath) 406 } 407 } else { 408 cmd := exec.Command("diff", dstPath, srcPath, "-N", "-u") 409 cmd.Stdout = os.Stdout 410 cmd.Run() 411 } 412 return nil 413 }) 414} 415 416func copyFile(dstPath, srcPath string) (changed bool) { 417 src, err := ioutil.ReadFile(srcPath) 418 check(err) 419 check(os.MkdirAll(filepath.Dir(dstPath), 0775)) 420 dst, _ := ioutil.ReadFile(dstPath) 421 if bytes.Equal(src, dst) { 422 return false 423 } 424 check(ioutil.WriteFile(dstPath, src, 0664)) 425 return true 426} 427 428func check(err error) { 429 if err != nil { 430 panic(err) 431 } 432} 433