docs: restructure and add more samples

Author: fischor

.gitignore

@@ -1,3 +1,6 @@
+output_root
+/acme/
+
 testout/
 .vscode/
 .vim/

samples/protoc-gen-dataclasses/README.md

@@ -0,0 +1,22 @@
+# protoc-gen-dataclasses sample plugin
+
+This sample demonstrates the use of `py_import_func` and shows how to signal support for and handle proto3 optionals.
+
+## Run
+
+Ensure a folder called `output_root` exists:
+
+```
+mkdir output_root
+rm -r output_root/acme
+```
+
+generate
+
+```sh
+protoc \
+    --plugin=protoc-gen-dataclasses=samples/protoc-gen-dataclasses/plugin.py \
+    --dataclasses_out=output_root \
+    --proto_path samples/protos \
+    samples/protos/acme/**/*.proto
+`` 

samples/protoc-gen-dataclasses/plugin.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+
+import protogen
+import google.protobuf.compiler.plugin_pb2
+
+dataclassesPackage = protogen.PyImportPath("dataclasses")
+
+
+def generate(gen: protogen.Plugin):
+    for f in gen.files_to_generate:
+        # This plugin generates a file with suffix `_dataclasses.py` file for
+        # each `.proto' file to generate. For a proto file "a/b/library.proto"
+        # a python file "a/b/library_dataclasses.py" will be generated. The
+        # python module name of this file is "a.b.library_dataclasses". This is
+        # different from the behaviour assumed by default, which would generate
+        # a "a/b/library_pb2.py" file that has a module name of "a.b.library_pb2".
+        #
+        # This changed module naming behaviour changes the way messages and need
+        # to be imported between two python files: instead of importing the
+        # "a.b.library" module when a message from the "a/b/library.proto" file
+        # is referenced, the "a.b.library_dataclasses" module needs to be
+        # imported.
+        #
+        # To import the correct modules, the python identifiers of the messages
+        # the plugin has to work with (see `protogen.Message.py_ident`) must be
+        # adjusted accordingly. This is done by providing the
+        # `datalasses_py_import_func` to the `protogen.Options` below. With that
+        # all messages will use the
+        #
+        #   ```
+        #   proto_filename.replace(".proto", "_dataclasses").replace("/", ".")
+        #   ```
+        #
+        # as the python import path.
+        g = gen.new_generated_file(
+            f.proto.name.replace(".proto", "_dataclasses.py"),
+            f.py_import_path,
+        )
+        g.P("# Autogenerated code. DO NOT EDIT.")
+        g.P(f'"""This is an module docstring."""')
+        g.P()
+        g.print_import()
+        g.P()
+
+        for message in f.messages:
+            g.P("@", dataclassesPackage.ident("dataclass"))
+            g.P(f"class {message.proto.name}:")
+            g.P(f'  """{message.location.leading_comments}"""')
+            g.P()
+
+            for field in message.fields:
+                if field.kind == protogen.Kind.MESSAGE:
+                    g.P("  ", field.py_name, ": ", field.message.py_ident)
+                elif field.proto.proto3_optional:
+                    g.P(f"  {field.py_name}: str | None")  # use str for simplicity
+                else:
+                    g.P(f"  {field.py_name}: str")
+
+            g.P()
+
+
+def dataclasses_py_import_func(proto_filename: str, proto_package: str):
+    return protogen.PyImportPath(
+        proto_filename.replace(".proto", "_dataclasses").replace("/", ".")
+    )
+
+
+opts = protogen.Options(
+    py_import_func=dataclasses_py_import_func,
+    # To indicate to be able to handle proto3 optionals a protoc plugin must set
+    # `google.protobuf.compiler.plugin_pb2.CodeGeneratorResponse.Feature.FEATURE_PROTO3_OPTIONAL`
+    # in the `supported_features` list. This enum value will be delegated to
+    # protoc via the CodeGeneratorResponse.supported_features field.
+    # See also https://github.com/protocolbuffers/protobuf/blob/fe0a809d5ef4/src/google/protobuf/compiler/plugin.proto#L107-L109).
+    supported_features=[
+        google.protobuf.compiler.plugin_pb2.CodeGeneratorResponse.Feature.FEATURE_PROTO3_OPTIONAL
+    ],
+)
+opts.run(generate)

samples/protoc-gen-empty/README.md

@@ -0,0 +1,22 @@
+# protoc-gen-empty sample
+
+This is a basic example of a protoc plugin.
+
+## Run
+
+Ensure a folder called `output_root` exists:
+
+```
+mkdir output_root
+rm -r output_root/acme
+```
+
+Generate code for the `samples/protos` protobuf definitions with the protoc-gen-empty plugin:
+
+```sh
+protoc \
+    --plugin=protoc-gen-empty=samples/protoc-gen-empty/plugin.py \
+    --empty_out=output_root \
+    --proto_path samples/protos \
+    samples/protos/acme/**/*.proto
+`` 

samples/empty/__init__.py renamed to samples/protoc-gen-empty/plugin.py

@@ -9,7 +9,7 @@ def generate(gen: protogen.Plugin):
     for f in gen.files_to_generate:
         if f.generate:
             g = gen.new_generated_file(
-                f.proto.name.replace(".proto", ".py"), 
+                f.proto.name.replace(".proto", "_pb2.py"),
                 f.py_import_path,
             )
             g.P("# Autogenerated code. DO NOT EDIT.")
@@ -19,19 +19,19 @@ def generate(gen: protogen.Plugin):
             g.P()
 
             for enum in f.enums:
-                g.P(f"class {enum.py_ident.py_name}(", enumPackage.ident("Enum"), "):")
+                g.P(f"class {enum.proto.name}(", enumPackage.ident("Enum"), "):")
                 for value in enum.values:
                     g.P(f"  {value.proto.name}={value.number}")
                 g.P()
 
             for message in f.messages:
-                g.P(f"class {message.py_ident.py_name}:")
+                g.P(f"class {message.proto.name}:")
                 g.P(f"  def __init__(self, host: str):")
                 g.P(f"    self.host = host")
                 g.P()
 
             for service in f.services:
-                g.P(f"class {service.py_ident.py_name}Client:")
+                g.P(f"class {service.proto.name}Client:")
                 g.P(f"  def __init__(self, host: str):")
                 g.P(f"    self.host = host")
                 g.P()

samples/protoc-gen-extensions/README.md

@@ -0,0 +1,27 @@
+# protoc-gen-extensions sample
+
+This sample shows how to use proto3 extensions (`MethodOptions` more specifically) and the `protogen.Registry` to resolve messages.
+
+## Run
+
+For a plugin that uses extensions the extension needs to be generated first using the official protoc python compiler.
+This is necessary for the corresponding options that use e.g. in `samples/protos/acme/library/v1/library.proto` to be present and accessible for the example plugin.
+Generate python code for at least the file where the extension resides in and all its dependencies with the official Python protoc plugin.
+
+```sh
+protoc --proto_path=samples/protos --python_out=. samples/protos/acme/longrunning/operations.proto samples/protos/acme/protobuf/any.proto
+`` 
+
+Ensure a folder called `output_root` exists:
+
+```
+mkdir output_root
+rm -r output_root/acme
+```
+
+Then run plugin.
+
+```sh
+protoc --plugin=protoc-gen-extensions=samples/protoc-gen-extensions/plugin.py --extensions_out=output_root -I samples/protos samples/protos/acme/**/*.proto
+`` 
+

samples/protoc-gen-extensions/plugin.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python
+
+from dataclasses import dataclass
+from typing import Optional
+import protogen
+
+# Import the protobuf definitions that have been generated with the official
+# protobuf plugin for python.
+import acme.longrunning.operations_pb2
+
+
+@dataclass
+class OperationInfo:
+    response_type: protogen.Message
+    metadata_type: protogen.Message
+
+
+def operation_info(
+    registry: protogen.Registry, method: protogen.Method
+) -> Optional[OperationInfo]:
+    # Any options set for the extension can be found in the `options` field of
+    # the protobuf message they have been set on, here on the `Method`.
+    pb = None
+    for fd, msg in method.proto.options.ListFields():
+        if fd.number == acme.longrunning.operations_pb2.OPERATION_INFO_FIELD_NUMBER:
+            pb = msg
+            break
+
+    if pb is None:
+        # The extension has not been set on the method.
+        return None
+
+    # pb is at this point a protobuf message as generated by the offical
+    # protobuf plugin for python. In the example the extensions message type is
+    # `operations_pb2.OperationInfo`.
+    #
+    # `operations_pb2.OperationInfo` has two fields: `response_type` and
+    # `metadata_type` which themselfes reference protobuf messages by their
+    # name. These names might be fully qualified like "google.protobuf.Empty" or
+    # not fully qualified like `WriteBookRequest` which would resolve to
+    # `acme.library.v1.WritebookRequest` given the current scope is the
+    # `acme.library-v1.Library.WriteBook` service method. To resolve non
+    # fully-qualified message names given a current scope as reference, use
+    # `protogen.Registry.resolve_message_type`.
+
+    operation_info = OperationInfo(
+        registry.resolve_message_type(method.full_name, pb.response_type),
+        registry.resolve_message_type(method.full_name, pb.metadata_type),
+    )
+
+    if operation_info.response_type is None:
+        raise Exception(
+            f"message {pb.response_type} could not be resolved from base {method.full_name}"
+        )
+
+    if operation_info.metadata_type is None:
+        raise Exception(
+            f"message {pb.metadata_type} could not be resolved from base {method.full_name}"
+        )
+
+    return operation_info
+
+
+def generate(gen: protogen.Plugin):
+    for f in gen.files_to_generate:
+        g = gen.new_generated_file(
+            f.proto.name.replace(".proto", "_ext.py"),
+            f.py_import_path,
+        )
+
+        g.P("# Autogenerated code. DO NOT EDIT.")
+        g.P(f'"""This is an module docstring."""')
+        g.P()
+        g.print_import()
+        g.P()
+
+        for service in f.services:
+            g.P(f"class {service.py_ident.py_name}Client:")
+            g.P(f"  def __init__(self, host: str):")
+            g.P(f"    self.host = host")
+            g.P()
+
+            for method in service.methods:
+                # Create a new class `<MethodName>Operation` for each response type
+                # for each method that returns a acme.longrunning.Operation.
+                if method.output.full_name == "acme.longrunning.Operation":
+                    op = operation_info(gen.registry, method)
+                    g.P(f"class {method.proto.name}Operation:")
+                    g.P("  def wait() -> ", method.output.py_ident, ":")
+                    g.P("    pass")
+                    g.P()
+                    g.P("  response_type:", op.response_type.py_ident)
+                    g.P("  metadata_type:", op.metadata_type.py_ident)
+
+                # fmt: off
+                g.P(f"  def {method.py_name}(req: ", method.input.py_ident, ") -> ", method.output.py_ident, ":")
+                g.P(f"    pass")
+                g.P()
+                # fmt: on
+
+
+def options_py_import_func(proto_filename: str, proto_package: str):
+    return protogen.PyImportPath(
+        proto_filename.replace(".proto", "_ext").replace("/", ".")
+    )
+
+
+opts = protogen.Options(py_import_func=options_py_import_func)
+opts.run(generate)

samples/protos/acme/library/v1/library.proto

@@ -0,0 +1,28 @@
+syntax = "proto3";
+
+package acme.library.v1;
+
+import "acme/longrunning/operations.proto";
+
+// Note that this import is not actively used in this proto file. However the
+// `metadata_type` of this extension references the `acme.protobuf.Empty` type
+// and it should be resolved and used by the plugin.
+import "acme/protobuf/empty.proto";
+
+service Library {
+
+  rpc WriteBook(WriteBookRequest) returns (acme.longrunning.Operation) {
+    option (acme.longrunning.operation_info) = {
+      response_type : "WriteBookResponse"
+      metadata_type : "acme.protobuf.Empty"
+    };
+  }
+}
+
+message Book { string name = 1; }
+
+message WriteBookRequest { string random_field = 1; }
+
+message WriteBookResponse { string random_field = 1; }
+
+message WriteBookMetadata { string random_field = 1; }

samples/protos/acme/longrunning/operations.proto

@@ -0,0 +1,26 @@
+syntax = "proto3";
+
+package acme.longrunning;
+
+import "acme/protobuf/any.proto";
+import "google/protobuf/descriptor.proto";
+
+extend google.protobuf.MethodOptions { OperationInfo operation_info = 1049; }
+
+message Operation {
+  string name = 1;
+
+  acme.protobuf.Any metadata = 2;
+
+  bool done = 3;
+
+  oneof result {
+    string error = 4;
+    acme.protobuf.Any response = 5;
+  }
+}
+
+message OperationInfo {
+  string response_type = 1;
+  string metadata_type = 2;
+}

samples/protos/acme/protobuf/any.proto

@@ -0,0 +1,8 @@
+syntax = "proto3";
+
+package acme.protobuf;
+
+message Any {
+  string type_url = 1;
+  bytes value = 2;
+}