rework building protos with protoc changes

mix.exs

@@ -75,43 +75,59 @@ defmodule GrpcReflection.MixProject do
     ]
   end
 
-  @protoc_opts "gen_descriptors=true,plugins=grpc"
-  @protoc_opts_no_descriptor "package_prefix=NoDescriptor,plugins=grpc"
+  @protoc_opts "gen_descriptors=true,gen_proto_source=true,paths=source_relative,plugins=grpc"
+  @protoc_opts_no_descriptor "package_prefix=NoDescriptor,gen_proto_source=true,paths=source_relative,plugins=grpc"
   # Protos that set (elixirpb.file).module_prefix must be skipped here: that option
   # overrides package_prefix entirely, so the no-descriptor pass would emit modules with
   # the same name as the descriptor pass, causing a compile-time conflict.
   # Add any proto that uses the elixirpb.file module_prefix option to this list.
   @skip_no_descriptor ["custom_prefix_service.proto"]
 
   defp build_protos(_argv) do
+    {reflection, fixtures} =
+      "./priv/protos/**/*.proto"
+      |> Path.wildcard()
+      |> Enum.split_with(&String.contains?(&1, "reflection.proto"))
+
     # compile reflection protos
-    Enum.each(
-      [
-        "priv/protos/grpc/reflection/v1alpha/reflection.proto",
-        "priv/protos/grpc/reflection/v1/reflection.proto"
-      ],
-      fn reflection_proto ->
-        Mix.shell().cmd(
-          "protoc --elixir_out=#{@protoc_opts}:./lib/proto --proto_path=priv/protos/ #{reflection_proto}"
-        )
-      end
-    )
+    cmds =
+      reflection
+      |> Enum.map(fn reflection_proto ->
+        # protoc now grabs the path to the file and includes it with
+        # the message name, which for us leads to bad paths
+        i_path =
+          reflection_proto
+          |> Path.split()
+          |> List.delete_at(-1)
+          |> Path.join()
+
+        "protoc --elixir_out=#{@protoc_opts}:./lib/proto -I=#{i_path} #{reflection_proto}"
+      end)
 
     # compile test protos — once with descriptors, once without
-    "./priv/protos"
-    |> File.ls!()
-    |> Enum.filter(&Regex.match?(~r/.*.proto$/, &1))
-    |> Enum.each(fn proto ->
-      Mix.shell().cmd(
-        "protoc --elixir_out=#{@protoc_opts}:./test/support/protos -I priv/protos/ -I deps/protobuf/src #{proto}"
-      )
-
-      unless proto in @skip_no_descriptor do
-        Mix.shell().cmd(
-          "protoc --elixir_out=#{@protoc_opts_no_descriptor}:./test/support/protos/no_descriptor -I priv/protos/ -I deps/protobuf/src #{proto}"
-        )
+    fixtures
+    |> Enum.flat_map(fn proto ->
+      i_path =
+        proto
+        |> Path.split()
+        |> List.delete_at(-1)
+        |> Path.join()
+
+      with_descriptors =
+        "protoc --elixir_out=#{@protoc_opts}:./test/support/protos -I=#{i_path} -I priv/protos -I deps/protobuf/src #{proto}"
+
+      without_descriptors =
+        "protoc --elixir_out=#{@protoc_opts_no_descriptor}:./test/support/protos -I=#{i_path} -I priv/protos -I deps/protobuf/src #{proto}"
+
+      if Enum.any?(@skip_no_descriptor, &String.contains?(proto, &1)) do
+        [with_descriptors]
+      else
+        [with_descriptors, without_descriptors]
       end
     end)
+    |> Enum.concat(cmds)
+    |> Task.async_stream(&Mix.shell().cmd(&1))
+    |> Enum.to_list()
   end
 
   defp package do

mix.lock

@@ -20,7 +20,7 @@
   "makeup_erlang": {:hex, :makeup_erlang, "1.0.3", "4252d5d4098da7415c390e847c814bad3764c94a814a0b4245176215615e1035", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "953297c02582a33411ac6208f2c6e55f0e870df7f80da724ed613f10e6706afd"},
   "mint": {:hex, :mint, "1.7.1", "113fdb2b2f3b59e47c7955971854641c61f378549d73e829e1768de90fc1abf1", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "fceba0a4d0f24301ddee3024ae116df1c3f4bb7a563a731f45fdfeb9d39a231b"},
   "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"},
-  "protobuf": {:hex, :protobuf, "0.16.0", "d1878725105d49162977cf3408ccc3eac4f3532e26e5a9e250f2c624175d10f6", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "f0d0d3edd8768130f24cc2cfc41320637d32c80110e80d13f160fa699102c828"},
+  "protobuf": {:hex, :protobuf, "0.17.0", "39e24e43c9648e148feba16ed51100b5b2028ea900b55460377b0476f6e10613", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "ca6c91f6f63e2c147b47f03eefd10b80538aa6fc55ff4b12b795efb786b0152f"},
   "ranch": {:hex, :ranch, "2.2.0", "25528f82bc8d7c6152c57666ca99ec716510fe0925cb188172f41ce93117b1b0", [:make, :rebar3], [], "hexpm", "fa0b99a1780c80218a4197a59ea8d3bdae32fbff7e88527d7d8a4787eff4f8e7"},
   "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"},
 }

priv/protos/imports_test.proto → priv/protos/imports_test/imports_test.proto

@@ -3,7 +3,7 @@ syntax = "proto3";
 package imports_test;
 
 // Import from local proto file
-import "common_types.proto";
+import "common/common_types.proto";
 
 // Import from google protos
 import "google/protobuf/timestamp.proto";

priv/protos/package_b.proto → priv/protos/package_b/package_b.proto

@@ -2,7 +2,7 @@ syntax = "proto3";
 
 package package_b;
 
-import "package_a.proto";
+import "package_a/package_a.proto";
 
 message MessageB {
   string field_b = 1;

test/support/protos/common_types.pb.ex → test/support/protos/common/common_types.pb.ex

@@ -4,7 +4,7 @@ defmodule Common.Priority do
   use Protobuf,
     enum: true,
     full_name: "common.Priority",
-    protoc_gen_elixir_version: "0.16.0",
+    protoc_gen_elixir_version: "0.17.0",
     syntax: :proto3
 
   def descriptor do
@@ -16,37 +16,43 @@ defmodule Common.Priority do
           name: "PRIORITY_UNSPECIFIED",
           number: 0,
           options: nil,
-          __unknown_fields__: []
+          __unknown_fields__: [],
+          __protobuf__: true
         },
         %Google.Protobuf.EnumValueDescriptorProto{
           name: "LOW",
           number: 1,
           options: nil,
-          __unknown_fields__: []
+          __unknown_fields__: [],
+          __protobuf__: true
         },
         %Google.Protobuf.EnumValueDescriptorProto{
           name: "MEDIUM",
           number: 2,
           options: nil,
-          __unknown_fields__: []
+          __unknown_fields__: [],
+          __protobuf__: true
         },
         %Google.Protobuf.EnumValueDescriptorProto{
           name: "HIGH",
           number: 3,
           options: nil,
-          __unknown_fields__: []
+          __unknown_fields__: [],
+          __protobuf__: true
         },
         %Google.Protobuf.EnumValueDescriptorProto{
           name: "CRITICAL",
           number: 4,
           options: nil,
-          __unknown_fields__: []
+          __unknown_fields__: [],
+          __protobuf__: true
         }
       ],
       options: nil,
       reserved_range: [],
       reserved_name: [],
-      __unknown_fields__: []
+      __unknown_fields__: [],
+      __protobuf__: true
     }
   end
 
@@ -60,7 +66,11 @@ end
 defmodule Common.Address do
   @moduledoc false
 
-  use Protobuf, full_name: "common.Address", protoc_gen_elixir_version: "0.16.0", syntax: :proto3
+  use Protobuf,
+    full_name: "common.Address",
+    proto_source: "common_types.proto",
+    protoc_gen_elixir_version: "0.17.0",
+    syntax: :proto3
 
   def descriptor do
     # credo:disable-for-next-line
@@ -79,7 +89,8 @@ defmodule Common.Address do
           oneof_index: nil,
           json_name: "street",
           proto3_optional: nil,
-          __unknown_fields__: []
+          __unknown_fields__: [],
+          __protobuf__: true
         },
         %Google.Protobuf.FieldDescriptorProto{
           name: "city",
@@ -93,7 +104,8 @@ defmodule Common.Address do
           oneof_index: nil,
           json_name: "city",
           proto3_optional: nil,
-          __unknown_fields__: []
+          __unknown_fields__: [],
+          __protobuf__: true
         },
         %Google.Protobuf.FieldDescriptorProto{
           name: "state",
@@ -107,7 +119,8 @@ defmodule Common.Address do
           oneof_index: nil,
           json_name: "state",
           proto3_optional: nil,
-          __unknown_fields__: []
+          __unknown_fields__: [],
+          __protobuf__: true
         },
         %Google.Protobuf.FieldDescriptorProto{
           name: "postal_code",
@@ -121,7 +134,8 @@ defmodule Common.Address do
           oneof_index: nil,
           json_name: "postalCode",
           proto3_optional: nil,
-          __unknown_fields__: []
+          __unknown_fields__: [],
+          __protobuf__: true
         },
         %Google.Protobuf.FieldDescriptorProto{
           name: "country",
@@ -135,7 +149,8 @@ defmodule Common.Address do
           oneof_index: nil,
           json_name: "country",
           proto3_optional: nil,
-          __unknown_fields__: []
+          __unknown_fields__: [],
+          __protobuf__: true
         }
       ],
       nested_type: [],
@@ -146,7 +161,8 @@ defmodule Common.Address do
       oneof_decl: [],
       reserved_range: [],
       reserved_name: [],
-      __unknown_fields__: []
+      __unknown_fields__: [],
+      __protobuf__: true
     }
   end
 
@@ -162,7 +178,8 @@ defmodule Common.Coordinates do
 
   use Protobuf,
     full_name: "common.Coordinates",
-    protoc_gen_elixir_version: "0.16.0",
+    proto_source: "common_types.proto",
+    protoc_gen_elixir_version: "0.17.0",
     syntax: :proto3
 
   def descriptor do
@@ -182,7 +199,8 @@ defmodule Common.Coordinates do
           oneof_index: nil,
           json_name: "latitude",
           proto3_optional: nil,
-          __unknown_fields__: []
+          __unknown_fields__: [],
+          __protobuf__: true
         },
         %Google.Protobuf.FieldDescriptorProto{
           name: "longitude",
@@ -196,7 +214,8 @@ defmodule Common.Coordinates do
           oneof_index: nil,
           json_name: "longitude",
           proto3_optional: nil,
-          __unknown_fields__: []
+          __unknown_fields__: [],
+          __protobuf__: true
         },
         %Google.Protobuf.FieldDescriptorProto{
           name: "altitude",
@@ -210,7 +229,8 @@ defmodule Common.Coordinates do
           oneof_index: nil,
           json_name: "altitude",
           proto3_optional: nil,
-          __unknown_fields__: []
+          __unknown_fields__: [],
+          __protobuf__: true
         }
       ],
       nested_type: [],
@@ -221,7 +241,8 @@ defmodule Common.Coordinates do
       oneof_decl: [],
       reserved_range: [],
       reserved_name: [],
-      __unknown_fields__: []
+      __unknown_fields__: [],
+      __protobuf__: true
     }
   end
 
@@ -233,7 +254,11 @@ end
 defmodule Common.Money do
   @moduledoc false
 
-  use Protobuf, full_name: "common.Money", protoc_gen_elixir_version: "0.16.0", syntax: :proto3
+  use Protobuf,
+    full_name: "common.Money",
+    proto_source: "common_types.proto",
+    protoc_gen_elixir_version: "0.17.0",
+    syntax: :proto3
 
   def descriptor do
     # credo:disable-for-next-line
@@ -252,7 +277,8 @@ defmodule Common.Money do
           oneof_index: nil,
           json_name: "currencyCode",
           proto3_optional: nil,
-          __unknown_fields__: []
+          __unknown_fields__: [],
+          __protobuf__: true
         },
         %Google.Protobuf.FieldDescriptorProto{
           name: "units",
@@ -266,7 +292,8 @@ defmodule Common.Money do
           oneof_index: nil,
           json_name: "units",
           proto3_optional: nil,
-          __unknown_fields__: []
+          __unknown_fields__: [],
+          __protobuf__: true
         },
         %Google.Protobuf.FieldDescriptorProto{
           name: "nanos",
@@ -280,7 +307,8 @@ defmodule Common.Money do
           oneof_index: nil,
           json_name: "nanos",
           proto3_optional: nil,
-          __unknown_fields__: []
+          __unknown_fields__: [],
+          __protobuf__: true
         }
       ],
       nested_type: [],
@@ -291,7 +319,8 @@ defmodule Common.Money do
       oneof_decl: [],
       reserved_range: [],
       reserved_name: [],
-      __unknown_fields__: []
+      __unknown_fields__: [],
+      __protobuf__: true
     }
   end
 
@@ -305,7 +334,8 @@ defmodule Common.TimeRange do
 
   use Protobuf,
     full_name: "common.TimeRange",
-    protoc_gen_elixir_version: "0.16.0",
+    proto_source: "common_types.proto",
+    protoc_gen_elixir_version: "0.17.0",
     syntax: :proto3
 
   def descriptor do
@@ -325,7 +355,8 @@ defmodule Common.TimeRange do
           oneof_index: nil,
           json_name: "startTime",
           proto3_optional: nil,
-          __unknown_fields__: []
+          __unknown_fields__: [],
+          __protobuf__: true
         },
         %Google.Protobuf.FieldDescriptorProto{
           name: "end_time",
@@ -339,7 +370,8 @@ defmodule Common.TimeRange do
           oneof_index: nil,
           json_name: "endTime",
           proto3_optional: nil,
-          __unknown_fields__: []
+          __unknown_fields__: [],
+          __protobuf__: true
         }
       ],
       nested_type: [],
@@ -350,7 +382,8 @@ defmodule Common.TimeRange do
       oneof_decl: [],
       reserved_range: [],
       reserved_name: [],
-      __unknown_fields__: []
+      __unknown_fields__: [],
+      __protobuf__: true
     }
   end