A => LICENSE +21 -0
@@ 1,21 @@
+MIT License
+
+Copyright (c) 2021 Brian Duggan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
A => META6.json +27 -0
@@ 1,27 @@
+{
+ "auth": "cpan:bduggan",
+ "authors": [
+ "Brian Duggan"
+ ],
+ "description": "Utilities for managing protobufs",
+ "name": "Protobuf",
+ "license": "MIT",
+ "perl": "6.*",
+ "provides": {
+ "Protobuf": "lib/Protobuf.rakumod",
+ "Protobuf::Grammar": "lib/Protobuf/Grammar.rakumod",
+ "Protobuf::Actions": "lib/Protobuf/Actions.rakumod"
+ },
+ "repo-type": "git",
+ "source-url": "https://git.sr.ht/~bduggan/raku-protobuf",
+ "tags": [
+ "protobuf"
+ ],
+ "depends" : [
+ "Grammar::PrettyErrors"
+ ],
+ "test-depends": [
+ "Test"
+ ],
+ "version": "0.0.1"
+}
A => README.md +63 -0
@@ 1,63 @@
+# raku-protobuf
+
+Utilities for handling protobufs with Raku
+
+## Description
+
+This package contains utilities for managing protobufs with raku.
+
+Current contents:
+
+* Protobuf::Grammar -- parse a protobuf file
+* Protobuf::Actions -- generate a representation from a grammar
+* Protobuf -- provides `parse-proto` for combining the above
+* protoc.raku -- cli to parse a proto from the command line
+
+There are a few other little classes like Protobuf::Definition,
+Protobuf::Service, etc which comprise the structure that is
+built by Protobuf::Actions.
+
+## SYNOPSIS
+
+ use Protobuf;
+
+ my $p = parse-proto("my.proto".IO.slurp);
+ # returns a Protobuf::Definition
+
+ for $p.services -> $svc {
+
+ # each one is a Protobuf::Service
+ say "service name: " ~ $svc.name;
+
+ for $svc.endpoints -> $e {
+ # each one is a Protobuf::Endpoint
+ say " endpoint: " ~ $e.name;
+
+ say ' request params: ' ~ $e.request.name;
+ # Requests and responses are Protobuf::Message's
+
+ for $e.request.fields -> $f {
+ # These are Protobuf::Field's
+ say ' name : ' ~ $f.name;
+ say ' type : ' ~ $f.type;
+ }
+ say ' response params: ' ~ $e.response.name;
+ for $e.response.fields -> $f {
+ say ' name : ' ~ $f.name;
+ say ' type : ' ~ $f.type;
+ }
+ }
+ }
+
+## AUTHOR
+
+Brian Duggan
+
+## TODO
+
+* Generate code from a proto
+* A bunch of details in the grammar
+
+## SEE ALSO
+
+https://developers.google.com/protocol-buffers/docs/reference/proto3-spec
A => eg/helloworld.proto +38 -0
@@ 1,38 @@
+// Copyright 2015 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+option java_multiple_files = true;
+option java_package = "io.grpc.examples.helloworld";
+option java_outer_classname = "HelloWorldProto";
+option objc_class_prefix = "HLW";
+
+package helloworld;
+
+// The greeting service definition.
+service Greeter {
+ // Sends a greeting
+ rpc SayHello (HelloRequest) returns (HelloReply) {}
+}
+
+// The request message containing the user's name.
+message HelloRequest {
+ string name = 1;
+}
+
+// The response message containing the greetings
+message HelloReply {
+ string message = 1;
+}
A => lib/Protobuf.rakumod +11 -0
@@ 1,11 @@
+unit module Protobuf;
+
+use Protobuf::Grammar;
+use Protobuf::Actions;
+
+sub parse-proto(Str $s) is export {
+ my $a = Protobuf::Actions.new;
+ my $g = Protobuf::Grammar.new.parse($s, actions => $a);
+ return $g.made;
+}
+
A => lib/Protobuf/Actions.rakumod +70 -0
@@ 1,70 @@
+class Protobuf::Service {
+ has $.name;
+ has @.endpoints;
+}
+class Protobuf::Definition {
+ has @.services;
+ has @.messages;
+}
+class Protobuf::Endpoint {
+ has $.name;
+ has $.request is rw;
+ has $.response is rw;
+}
+class Protobuf::Message {
+ has $.name;
+ has @.fields;
+}
+class Protobuf::Field {
+ has $.name;
+ has $.type;
+}
+
+class Protobuf::Actions {
+
+ method TOP($/) {
+ my @things := $<proto>.made.List;
+ my @services = @things.grep( * ~~ Protobuf::Service );
+ my @messages = @things.grep( * ~~ Protobuf::Message );
+ my %messages = @messages.map: { .name => $_ }
+ for @services -> $s {
+ for $s.endpoints -> $e {
+ $e.request = %messages{ $e.request.name };
+ $e.response = %messages{ $e.response.name };
+ }
+ }
+ $/.make: Protobuf::Definition.new: :@services, :@messages;
+ }
+
+ method proto($/) {
+ $/.make: $<topLevelDef>.map( *.made ).grep( *.defined )
+ }
+
+ method topLevelDef($/) {
+ $/.make: $<service>.made // $<message>.made
+ }
+
+ method message($/) {
+ $/.make: Protobuf::Message.new( name => ~$<messageName>, fields => $<messageBody>.made )
+ }
+
+ method messageBody($/) {
+ $/.make: $<field>.map: *.made;
+ }
+
+ method field($/) {
+ $/.make: Protobuf::Field.new( name => ~$<fieldName>, type => ~$<type> );
+ }
+
+ method service($/) {
+ $/.make: Protobuf::Service.new: name => ~$<serviceName>, endpoints => ($<rpc>.map(*.made).grep(*.defined))
+ }
+
+ method rpc($/) {
+ $/.make: Protobuf::Endpoint.new(name => ~$<rpcName>, request => $<request>.?made, response => $<response>.?made )
+ }
+
+ method messageType($/) {
+ $/.make: Protobuf::Message.new: name => ~$<messageName>
+ }
+}
A => lib/Protobuf/Grammar.rakumod +300 -0
@@ 1,300 @@
+use Grammar::PrettyErrors;
+
+grammar Protobuf::Grammar does Grammar::PrettyErrors {
+
+token ws {
+ <!ww>
+ [
+ \s
+ | [ '//' \V* [$|\n] ]
+ ]*
+}
+
+# letter = "A" … "Z" | "a" … "z"
+regex letter { <[A..Z] + [a..z]> }
+
+# decimalDigit = "0" … "9"
+regex decimalDigit { <[0..9]> }
+
+# octalDigit = "0" … "7"
+regex octalDigit { <[0..7]> }
+
+# hexDigit = "0" … "9" | "A" … "F" | "a" … "f"
+regex hexDigit { <[0..9A..Fa..f]> }
+
+# ident = letter { letter | decimalDigit | "_" }
+regex ident {
+ <.letter>
+ [ <.letter> | <.decimalDigit> | "_" ]*
+}
+
+# fullIdent = ident { "." ident }
+regex fullIdent { <ident>+ % '.' }
+
+# messageName = ident
+# enumName = ident
+# fieldName = ident
+# oneofName = ident
+# mapName = ident
+# serviceName = ident
+# rpcName = ident
+regex messageName { <ident> }
+regex enumName { <ident> }
+regex fieldName { <ident> }
+regex oneofName { <ident> }
+regex mapName { <ident> }
+regex serviceName { <ident> }
+regex rpcName { <ident> }
+
+# messageType = [ "." ] { ident "." } messageName
+regex messageType { [ "." ]? [ <ident> '.' ]* <messageName> }
+
+# enumType = [ "." ] { ident "." } enumName
+regex enumType { [ '.' ] [ <ident> '.']+ <enumName> }
+
+# intLit = decimalLit | octalLit | hexLit
+regex intLit { <decimalLit> | <octalLit> | <hexLit> }
+
+# decimalLit = ( "1" … "9" ) { decimalDigit }
+regex decimalLit { <[1..9]> <decimalDigit>* }
+
+# octalLit = "0" { octalDigit }
+regex octalLit { '0' <octalDigit>* }
+
+# hexLit = "0" ( "x" | "X" ) hexDigit { hexDigit }
+regex hexLit { '0' <[xX]> <hexDigit>+ }
+
+# floatLit = ( decimals "." [ decimals ] [ exponent ] | decimals exponent | "."decimals [ exponent ] )
+# | "inf" | "nan"
+regex floatLit {
+ [ | <decimals> '.' <decimals>? <exponent>?
+ | <decimals> <exponent>
+ | '.' <decimals> <exponent>?
+ ]
+ | 'inf' | 'nan'
+}
+
+# decimals = decimalDigit { decimalDigit }
+regex decimals { <decimalDigit>+ }
+
+# exponent = ( "e" | "E" ) [ "+" | "-" ] decimals
+regex exponent {
+ <[eE]> <[+-]> <decimals>
+}
+
+# boolLit = "true" | "false"
+regex boolLit {
+ true|false
+}
+
+# strLit = ( "'" { charValue } "'" ) | ( '"' { charValue } '"' )
+regex strLit {
+ | "'" <.charValue>* "'"
+ | '"' <.charValue>* '"'
+}
+
+# charValue = hexEscape | octEscape | charEscape | /[^\0\n\\]/
+regex charValue {
+ <hexEscape> | <octEscape> | <charEscape> | <-[\0\n\\]>
+}
+
+# hexEscape = '\' ( "x" | "X" ) hexDigit hexDigit
+regex hexEscape {
+ '\\' <[xX]> <hexDigit>**2
+}
+
+# octEscape = '\' octalDigit octalDigit octalDigit
+regex octEscape {
+ '\\' <octalDigit>**3
+}
+
+# charEscape = '\' ( "a" | "b" | "f" | "n" | "r" | "t" | "v" | '\' | "'" | '"' )
+regex charEscape {
+ '\\' <[abfnrtv\\'"]>
+}
+
+# quote = "'" | '"'
+regex quote {
+ <['"]>
+}
+
+# emptyStatement = ";"
+regex emptyStatement {
+ ';'
+}
+
+# constant = fullIdent | ( [ "-" | "+" ] intLit ) | ( [ "-" | "+" ] floatLit ) | strLit | boolLit
+regex constant {
+ | <fullIdent>
+ | [ <[-+]>? <intLit> ]
+ | [ <[-+]>? <floatLit> ]
+ | <strLit>
+ | <boolLit>
+}
+
+# syntax = "syntax" "=" quote "proto3" quote ";"
+rule syntax {
+ syntax '=' <quote> proto3 $<quote> ';'
+}
+
+# import = "import" [ "weak" | "public" ] strLit ";" </pre>
+rule import {
+ import [ weak | public | "" ] <strLit> ';'
+}
+
+# package = "package" fullIdent ";"
+rule package {
+ package <fullIdent> ';'
+}
+
+# option = "option" optionName "=" constant ";"
+rule option {
+ option <optionName> '=' <constant> ';'
+}
+
+# optionName = ( ident | "(" fullIdent ")" ) { "." ident }
+rule optionName {
+ [ <ident> | '(' <fullIdent> ')' ]
+ [ '.' <ident> ]*
+}
+
+# type = "double" | "float" | "int32" | "int64" | "uint32" | "uint64"
+# | "sint32" | "sint64" | "fixed32" | "fixed64" | "sfixed32" | "sfixed64"
+# | "bool" | "string" | "bytes" | messageType | enumType
+regex type {
+ < double float int32 int64 uint32 uint64
+ sint32 sint64 fixed32 fixed64 sfixed32 sfixed64
+ bool string bytes> | <messageType> | <enumType>
+}
+
+# fieldNumber = intLit
+regex fieldNumber {
+ <intLit>
+}
+
+# field = [ "repeated" ] type fieldName "=" fieldNumber [ "[" fieldOptions "]" ] ";"
+rule field {
+ [ repeated ]? <type> <fieldName> '=' <fieldNumber> [ '[' <fieldOptions> ']' ]? ';'
+}
+
+# fieldOptions = fieldOption { "," fieldOption }
+regex fieldOptions {
+ <fieldOption>+ % ','
+}
+
+# fieldOption = optionName "=" constant
+regex fieldOption {
+ <optionName> '=' <constant>
+}
+
+# oneof = "oneof" oneofName "{" { oneofField | emptyStatement } "}"
+rule oneof {
+ oneof <oneofName> '{' [ <oneofField> | <emptyStatement> ]* '}'
+}
+
+# oneofField = type fieldName "=" fieldNumber [ "[" fieldOptions "]" ] ";"
+rule oneofField {
+ <type> <fieldName> '=' <fieldNumber> [ '[' <fieldOptions> ']' ]? ';'
+}
+
+# mapField="map" "<" " "," keytype type>" mapName "=" fieldNumber [ "[" fieldOptions "]" ] ";"
+rule mapField {
+ map '<' <keyType> ',' <type> '>' <mapName> '=' <fieldNumber> [ '[' <fieldOptions> ']' ]? ';'
+}
+# map<int32, string> my_map = 4;
+
+# keyType = "int32" | "int64" | "uint32" | "uint64" | "sint32" | "sint64" |
+# "fixed32" | "fixed64" | "sfixed32" | "sfixed64" | "bool" | "string"
+regex keyType {
+ < int32 int64 uint32 uint64 sint32 sint64
+ fixed32 fixed64 sfixed32 sfixed64 bool string >
+}
+
+# reserved = "reserved" ( ranges | fieldNames ) ";"
+rule reserved {
+ reserved [ <ranges> | <fieldNames> ] ';'
+}
+
+# ranges = range { "," range }
+rule ranges {
+ <range>+ % ','
+}
+
+# range = intLit [ "to" ( intLit | "max" ) ]
+rule range {
+ <intLit> [ to [ <intLit> | max ] ]?
+}
+
+# fieldNames = fieldName { "," fieldName }
+rule fieldNames {
+ <fieldName>+ % ','
+}
+
+# enum = "enum" enumName enumBody
+rule enum {
+ enum <enumName> <enumBody>
+}
+
+# enumBody = "{" { option | enumField | emptyStatement } "}"
+rule enumBody {
+ '{' [ <option> | <enumField> | <emptyStatement> ]* '}'
+}
+
+# enumField = ident "=" intLit [ "[" enumValueOption { "," enumValueOption } "]" ]";"
+rule enumField {
+ <ident> '=' <intLit>
+ [ '[' <enumValueOption>+ % ',' ']' ]?
+ ';'
+}
+
+# enumValueOption = optionName "=" constant</pre>
+rule enumValueOption {
+ <optionName> '=' <constant>
+}
+
+# message = "message" messageName messageBody
+rule message {
+ message <messageName> <messageBody>
+}
+
+# messageBody = "{" { field | enum | message | option | oneof | mapField | reserved | emptyStatement } "}"
+rule messageBody {
+ '{' [ <field> | <enum> | <message> | <option> | <oneof> | <mapField> | <reserved> | <emptyStatement> ]* '}'
+}
+
+# service = "service" serviceName "{" { option | rpc | emptyStatement } "}"
+rule service {
+ service <serviceName> '{' [
+ <option> | <rpc> | <emptyStatement>
+ ]* '}'
+}
+
+# rpc = "rpc" rpcName "(" [ "stream" ] messageType ")" "returns" "(" [ "stream" ]
+# messageType ")" (( "{" {option | emptyStatement } "}" ) | ";")
+rule rpc {
+ rpc
+ <rpcName> '(' [ stream ]? <request=messageType> ')'
+ returns
+ '(' [ stream ]? <response=messageType> ')'
+ [[ '{' [<option> | <emptyStatement>]* '}' ] | ';']
+}
+
+# proto = syntax { import | package | option | topLevelDef | emptyStatement }
+rule proto {
+ <syntax> [ <import> | <package> | <option> | <topLevelDef> | <emptyStatement> ]*
+}
+
+# topLevelDef = message | enum | service
+rule topLevelDef {
+ <message> | <enum> | <service>
+}
+
+rule TOP {
+ [ \s
+ | [ '//' \V* [$|\n] ]
+ ]*
+ <proto>
+}
+
+}
+
A => script/protoc.raku +24 -0
@@ 1,24 @@
+#!/usr/bin/env raku
+
+use Protobuf;
+
+my $p = parse-proto($*ARGFILES.slurp);
+
+for $p.services -> $svc {
+ say "service name " ~ $svc.name;
+ for $svc.endpoints -> $e {
+ say " endpoint " ~ $e.name;
+ say ' request params ' ~ $e.request.name;
+ for $e.request.fields -> $f {
+ say ' name : ' ~ $f.name;
+ say ' type : ' ~ $f.type;
+ }
+ say ' response params ' ~ $e.response.name;
+ for $e.response.fields -> $f {
+ say ' name : ' ~ $f.name;
+ say ' type : ' ~ $f.type;
+ }
+ }
+}
+
+
A => t/01-load.rakutest +8 -0
@@ 1,8 @@
+#!/usr/bin/env raku
+
+use Test;
+
+plan 1;
+
+use-ok 'Grammar::Protobuf', 'use gramar';
+
A => t/02-parse.rakutest +108 -0
@@ 1,108 @@
+#!/usr/bin/env raku
+
+use Test;
+use Grammar::Protobuf;
+
+my \protobuf = Grammar::Protobuf.new;
+
+ok protobuf.parse("package foo.bar;",:rule<package>), 'package';
+ok protobuf.parse("foo.bar",:rule<fullIdent>), 'fullIdent';
+ok protobuf.parse('string', :rule<type>), 'type';
+ok protobuf.parse('SubMessage', :rule<type>), 'type';
+ok protobuf.parse('4', :rule<intLit>), 'int';
+ok protobuf.parse('4', :rule<fieldNumber>), 'fieldNumber';
+ok protobuf.parse('string name = 4;', :rule<oneofField>), 'oneofField';
+ok protobuf.parse('SubMessage sub_message = 9;', :rule<oneofField>), 'oneofField';
+ok protobuf.parse(q:to/X/, :rule<oneof>), 'oneof';
+oneof foo {
+ string name = 4;
+ SubMessage sub_message = 9;
+}
+X
+
+ok protobuf.parse('map<int32, string> my_map = 4;',:rule<mapField>), 'mapField';
+ok protobuf.parse('map<string, Project> projects = 3;',:rule<mapField>), 'mapField';
+
+ok protobuf.parse('9 to 11',:rule<range>), 'range';
+ok protobuf.parse('reserved 2, 15, 9 to 11;',:rule<reserved>), 'reserved';
+ok protobuf.parse('2, 9 to 11',:rule<ranges>), 'ranges';
+# ok protobuf.parse('reserved "foo", "bar";',:rule<reserved>), 'reserved';
+# ok protobuf.parse('"foo", "bar"',:rule<fieldNames>), 'fieldNames';
+
+ok protobuf.parse('UNKNOWN = 0;',:rule<enumField>), 'enumField';
+ok protobuf.parse('EnumAllowingAlias',:rule<enumName>), 'enumName';
+ok protobuf.parse('"hello world"', :rule<constant>), 'constant';
+ok protobuf.parse('RUNNING = 2 [(custom_option) = "hello world"];', :rule<enumField>), 'field';
+ok protobuf.parse(q:to/X/, :rule<enum>), 'enum';
+ enum EnumAllowingAlias {
+ option allow_alias = true;
+ UNKNOWN = 0;
+ STARTED = 1;
+ RUNNING = 2 [(custom_option) = "hello world"];
+ }
+ X
+ok protobuf.parse(q:to/X/,:rule<field>), 'field';
+ foo.bar nested_message = 2;
+ X
+ok protobuf.parse(q:to/X/,:rule<field>), 'field';
+ repeated int32 samples = 4 [packed=true];
+ X
+ok protobuf.parse('foo.bar',:rule<type>), 'type';
+
+ok protobuf.parse('Outer',:rule<messageName>), 'messageName';
+
+ok protobuf.parse('option (my_option).a = true;',:rule<option>), 'option';
+
+ok protobuf.parse(q:to/X/,:rule<message>), 'message';
+message Outer {
+ option (my_option).a = true;
+ message Inner {
+ int64 ival = 1;
+ }
+ map<int32, string> my_map = 2;
+}
+X
+
+ok protobuf.parse(q:to/X/,:rule<service>), 'service';
+ service SearchService {
+ rpc Search (SearchRequest) returns (SearchResponse);
+ }
+ X
+
+ok protobuf.parse(q:to/X/,:rule<service>), 'service';
+ service SearchService {
+ rpc Search (SearchRequest) returns (SearchResponse);
+ }
+ X
+
+ok protobuf.parse('//', :rule<ws>), 'ws';
+ok protobuf.parse('// comment', :rule<ws>), 'ws';
+ok protobuf.parse(q:to/X/, :rule<package>), 'package';
+package // comment
+ foo.bar;
+X
+
+ok protobuf.parse(q:to/X/.trim, :rule<syntax>), 'syntax';
+syntax = "proto3";
+X
+
+ok protobuf.parse(q:to/PROTO/, :rule<proto>), 'proto';
+syntax = "proto3";
+import public "other.proto";
+option java_package = "com.example.foo";
+enum EnumAllowingAlias {
+ option allow_alias = true;
+ UNKNOWN = 0;
+ STARTED = 1;
+ RUNNING = 2 [(custom_option) = "hello world"];
+}
+message outer {
+ option (my_option).a = true;
+ message inner { // Level 2
+ int64 ival = 1;
+ }
+ repeated inner inner_message = 2;
+ EnumAllowingAlias enum_field =3;
+ map<int32, string> my_map = 4;
+}
+PROTO
A => t/03-actions.rakutest +11 -0
@@ 1,11 @@
+use Protobuf;
+use Test;
+
+my $m = parse-proto($?FILE.IO.parent.parent.child('eg/helloworld.proto').slurp);
+my @services = $m.services;
+is @services[0].name, 'Greeter', 'service name';
+is @services[0].endpoints[0].name, 'SayHello', 'endpoint name';
+is @services[0].endpoints[0].request.fields[0].name, 'name', 'field name';
+is @services[0].endpoints[0].response.fields[0].name, 'message', 'field name';
+
+done-testing;