~bduggan/raku-protobuf

c5d1e28f35196670967cd0740b46dd5e3b263f0d — Brian Duggan 7 months ago
init
A  => LICENSE +21 -0
@@ 1,21 @@
MIT License

Copyright (c) 2021 Brian Duggan

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

A  => META6.json +27 -0
@@ 1,27 @@
{
  "auth": "cpan:bduggan",
  "authors": [
    "Brian Duggan"
  ],
  "description": "Utilities for managing protobufs",
  "name": "Protobuf",
  "license": "MIT",
  "perl": "6.*",
  "provides": {
    "Protobuf": "lib/Protobuf.rakumod",
    "Protobuf::Grammar": "lib/Protobuf/Grammar.rakumod",
    "Protobuf::Actions": "lib/Protobuf/Actions.rakumod"
  },
  "repo-type": "git",
  "source-url": "https://git.sr.ht/~bduggan/raku-protobuf",
  "tags": [
    "protobuf"
  ],
  "depends" : [
    "Grammar::PrettyErrors"
  ],
  "test-depends": [
    "Test"
  ],
  "version": "0.0.1"
}

A  => README.md +63 -0
@@ 1,63 @@
# raku-protobuf

Utilities for handling protobufs with Raku

## Description

This package contains utilities for managing protobufs with raku.

Current contents:

* Protobuf::Grammar -- parse a protobuf file
* Protobuf::Actions -- generate a representation from a grammar
* Protobuf -- provides `parse-proto` for combining the above
* protoc.raku -- cli to parse a proto from the command line

There are a few other little classes like Protobuf::Definition,
Protobuf::Service, etc which comprise the structure that is
built by Protobuf::Actions.

## SYNOPSIS

  use Protobuf;

  my $p = parse-proto("my.proto".IO.slurp);
  # returns a Protobuf::Definition

  for $p.services -> $svc {

    # each one is a Protobuf::Service
    say "service name: " ~ $svc.name;

    for $svc.endpoints -> $e {
      # each one is a Protobuf::Endpoint
      say " endpoint:  " ~ $e.name;

      say '  request params: ' ~ $e.request.name;
      # Requests and responses are Protobuf::Message's

      for $e.request.fields -> $f {
        # These are Protobuf::Field's
        say '   name : ' ~ $f.name;
        say '   type : ' ~ $f.type;
      }
      say '  response params: ' ~ $e.response.name;
      for $e.response.fields -> $f {
        say '   name : ' ~ $f.name;
        say '   type : ' ~ $f.type;
      }
    }
  }

## AUTHOR

Brian Duggan

## TODO

* Generate code from a proto
* A bunch of details in the grammar

## SEE ALSO

https://developers.google.com/protocol-buffers/docs/reference/proto3-spec

A  => eg/helloworld.proto +38 -0
@@ 1,38 @@
// Copyright 2015 gRPC authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

option java_multiple_files = true;
option java_package = "io.grpc.examples.helloworld";
option java_outer_classname = "HelloWorldProto";
option objc_class_prefix = "HLW";

package helloworld;

// The greeting service definition.
service Greeter {
  // Sends a greeting
  rpc SayHello (HelloRequest) returns (HelloReply) {}
}

// The request message containing the user's name.
message HelloRequest {
  string name = 1;
}

// The response message containing the greetings
message HelloReply {
  string message = 1;
}

A  => lib/Protobuf.rakumod +11 -0
@@ 1,11 @@
unit module Protobuf;

use Protobuf::Grammar;
use Protobuf::Actions;

sub parse-proto(Str $s) is export {
  my $a = Protobuf::Actions.new;
  my $g = Protobuf::Grammar.new.parse($s, actions => $a);
  return $g.made;
}


A  => lib/Protobuf/Actions.rakumod +70 -0
@@ 1,70 @@
class Protobuf::Service {
  has $.name;
  has @.endpoints;
}
class Protobuf::Definition {
  has @.services;
  has @.messages;
}
class Protobuf::Endpoint {
  has $.name;
  has $.request is rw;
  has $.response is rw;
}
class Protobuf::Message {
  has $.name;
  has @.fields;
}
class Protobuf::Field {
  has $.name;
  has $.type;
}

class Protobuf::Actions {

  method TOP($/) {
    my @things := $<proto>.made.List;
    my @services = @things.grep( * ~~ Protobuf::Service );
    my @messages = @things.grep( * ~~ Protobuf::Message );
    my %messages = @messages.map: { .name => $_ }
    for @services -> $s {
      for $s.endpoints -> $e {
        $e.request = %messages{ $e.request.name };
        $e.response = %messages{ $e.response.name };
      }
    }
    $/.make: Protobuf::Definition.new: :@services, :@messages;
  }

  method proto($/) {
    $/.make: $<topLevelDef>.map( *.made ).grep( *.defined )
  }

  method topLevelDef($/) {
    $/.make: $<service>.made // $<message>.made
  }

  method message($/) {
    $/.make: Protobuf::Message.new( name => ~$<messageName>, fields => $<messageBody>.made )
  }

  method messageBody($/) {
    $/.make: $<field>.map: *.made;
  }

  method field($/) {
    $/.make: Protobuf::Field.new( name => ~$<fieldName>, type => ~$<type> );
  }

  method service($/) {
    $/.make: Protobuf::Service.new: name => ~$<serviceName>, endpoints => ($<rpc>.map(*.made).grep(*.defined))
  }

  method rpc($/) {
    $/.make: Protobuf::Endpoint.new(name => ~$<rpcName>, request => $<request>.?made, response => $<response>.?made )
  }

  method messageType($/) {
    $/.make: Protobuf::Message.new: name => ~$<messageName>
  }
}

A  => lib/Protobuf/Grammar.rakumod +300 -0
@@ 1,300 @@
use Grammar::PrettyErrors;

grammar Protobuf::Grammar does Grammar::PrettyErrors {

token ws {
  <!ww>
  [
    \s
    | [ '//' \V* [$|\n] ]
  ]*
}

# letter = "A" … "Z" | "a" … "z"
regex letter { <[A..Z] + [a..z]> }

# decimalDigit = "0" … "9"
regex decimalDigit { <[0..9]> }

# octalDigit   = "0" … "7"
regex octalDigit { <[0..7]> }

# hexDigit     = "0" … "9" | "A" … "F" | "a" … "f"
regex hexDigit { <[0..9A..Fa..f]> }

# ident = letter { letter | decimalDigit | "_" }
regex ident {
  <.letter>
  [ <.letter> | <.decimalDigit> | "_" ]*
}

# fullIdent = ident { "." ident }
regex fullIdent { <ident>+ % '.' }

# messageName = ident
# enumName = ident
# fieldName = ident
# oneofName = ident
# mapName = ident
# serviceName = ident
# rpcName = ident
regex messageName { <ident> }
regex enumName { <ident> }
regex fieldName { <ident> }
regex oneofName { <ident> }
regex mapName { <ident> }
regex serviceName { <ident> }
regex rpcName { <ident> }

# messageType = [ "." ] { ident "." } messageName
regex messageType { [ "." ]? [ <ident> '.' ]* <messageName> }

# enumType = [ "." ] { ident "." } enumName
regex enumType { [ '.' ] [ <ident> '.']+ <enumName> }

# intLit     = decimalLit | octalLit | hexLit
regex intLit { <decimalLit> | <octalLit> | <hexLit> }

# decimalLit = ( "1" … "9" ) { decimalDigit }
regex decimalLit { <[1..9]> <decimalDigit>* }

# octalLit   = "0" { octalDigit }
regex octalLit { '0' <octalDigit>* }

# hexLit = "0" ( "x" | "X" ) hexDigit { hexDigit }
regex hexLit { '0' <[xX]> <hexDigit>+ }

# floatLit = ( decimals "." [ decimals ] [ exponent ] | decimals exponent | "."decimals [ exponent ] )
#            | "inf" | "nan"
regex floatLit {
  [ | <decimals> '.' <decimals>? <exponent>?
    | <decimals> <exponent>
    | '.' <decimals> <exponent>? 
    ]
  | 'inf' | 'nan'
}

# decimals  = decimalDigit { decimalDigit }
regex decimals { <decimalDigit>+ }

# exponent  = ( "e" | "E" ) [ "+" | "-" ] decimals
regex exponent  {
  <[eE]> <[+-]> <decimals>
}

# boolLit = "true" | "false"
regex boolLit {
  true|false
}

# strLit = ( "'" { charValue } "'" ) |  ( '"' { charValue } '"' )
regex strLit {
  | "'" <.charValue>* "'"
  | '"' <.charValue>* '"'
}

# charValue = hexEscape | octEscape | charEscape | /[^\0\n\\]/
regex charValue {
  <hexEscape> | <octEscape> | <charEscape> | <-[\0\n\\]>
}

# hexEscape = '\' ( "x" | "X" ) hexDigit hexDigit
regex hexEscape {
  '\\' <[xX]> <hexDigit>**2
}

# octEscape = '\' octalDigit octalDigit octalDigit
regex octEscape {
  '\\' <octalDigit>**3
}

# charEscape = '\' ( "a" | "b" | "f" | "n" | "r" | "t" | "v" | '\' | "'" | '"' )
regex charEscape {
  '\\' <[abfnrtv\\'"]>
}

# quote = "'" | '"'
regex quote {
  <['"]>
}

# emptyStatement = ";"
regex emptyStatement {
  ';'
}

# constant = fullIdent | ( [ "-" | "+" ] intLit ) | ( [ "-" | "+" ] floatLit ) | strLit | boolLit
regex constant {
  | <fullIdent>
  | [ <[-+]>? <intLit> ]
  | [ <[-+]>? <floatLit> ]
  | <strLit>
  | <boolLit>
}

# syntax = "syntax" "=" quote "proto3" quote ";"
rule syntax {
  syntax '=' <quote> proto3 $<quote> ';'
}

# import = "import" [ "weak" | "public" ] strLit ";" </pre>
rule import {
  import [ weak | public | "" ] <strLit> ';'
}

# package = "package" fullIdent ";"
rule package {
  package <fullIdent> ';'
}

# option = "option" optionName  "=" constant ";"
rule option {
    option <optionName> '=' <constant> ';'
}

# optionName = ( ident | "(" fullIdent ")" ) { "." ident }
rule optionName {
  [ <ident> | '(' <fullIdent> ')' ]
  [ '.' <ident> ]*
}

# type = "double" | "float" | "int32" | "int64" | "uint32" | "uint64"
#       | "sint32" | "sint64" | "fixed32" | "fixed64" | "sfixed32" | "sfixed64"
#       | "bool" | "string" | "bytes" | messageType | enumType
regex type {
 < double float int32 int64 uint32 uint64
   sint32 sint64 fixed32 fixed64 sfixed32 sfixed64
   bool string bytes> | <messageType> | <enumType>
}

# fieldNumber = intLit
regex fieldNumber {
  <intLit>
}

# field = [ "repeated" ] type fieldName "=" fieldNumber [ "[" fieldOptions "]" ] ";"
rule field {
  [ repeated ]? <type> <fieldName> '=' <fieldNumber> [ '[' <fieldOptions> ']' ]? ';'
}

# fieldOptions = fieldOption { ","  fieldOption }
regex fieldOptions {
  <fieldOption>+ % ','
}

# fieldOption = optionName "=" constant
regex fieldOption {
 <optionName> '=' <constant>
}

# oneof = "oneof" oneofName "{" { oneofField | emptyStatement } "}"
rule oneof {
  oneof <oneofName> '{' [ <oneofField> | <emptyStatement> ]* '}'
}

# oneofField = type fieldName "=" fieldNumber [ "[" fieldOptions "]" ] ";"
rule oneofField {
  <type> <fieldName> '=' <fieldNumber> [ '[' <fieldOptions> ']' ]? ';'
}

# mapField="map" "<" " "," keytype type>" mapName "=" fieldNumber [ "[" fieldOptions "]" ] ";"
rule mapField {
  map '<' <keyType> ',' <type> '>' <mapName> '=' <fieldNumber> [ '[' <fieldOptions> ']' ]? ';'
}
#   map<int32, string> my_map = 4;

# keyType = "int32" | "int64" | "uint32" | "uint64" | "sint32" | "sint64" |
#           "fixed32" | "fixed64" | "sfixed32" | "sfixed64" | "bool" | "string"
regex keyType {
 < int32 int64 uint32 uint64 sint32 sint64 
   fixed32 fixed64 sfixed32 sfixed64 bool string >
}

# reserved = "reserved" ( ranges | fieldNames ) ";"
rule reserved {
 reserved [ <ranges> | <fieldNames> ] ';'
}

# ranges = range { "," range }
rule ranges {
  <range>+ % ','
}

# range =  intLit [ "to" ( intLit | "max" ) ]
rule range {
  <intLit> [ to [ <intLit> | max ] ]?
}

# fieldNames = fieldName { "," fieldName }
rule fieldNames {
  <fieldName>+ % ','
}

# enum = "enum" enumName enumBody
rule enum {
 enum <enumName> <enumBody>
}

# enumBody = "{" { option | enumField | emptyStatement } "}"
rule enumBody {
  '{' [ <option> | <enumField> | <emptyStatement> ]* '}'
}

# enumField = ident "=" intLit [ "[" enumValueOption { ","  enumValueOption } "]" ]";"
rule enumField {
  <ident> '=' <intLit>
  [ '[' <enumValueOption>+ % ',' ']' ]?
  ';'
}

# enumValueOption = optionName "=" constant</pre>
rule enumValueOption {
  <optionName> '=' <constant>
}

# message = "message" messageName messageBody
rule message {
 message <messageName> <messageBody>
}

# messageBody = "{" { field | enum | message | option | oneof | mapField | reserved | emptyStatement } "}"
rule messageBody {
  '{' [ <field> | <enum> | <message> | <option> | <oneof> | <mapField> | <reserved> | <emptyStatement> ]* '}'
}

# service = "service" serviceName "{" { option | rpc | emptyStatement } "}"
rule service {
  service <serviceName> '{' [
    <option> | <rpc> | <emptyStatement>
  ]* '}'
}

# rpc = "rpc" rpcName "(" [ "stream" ] messageType ")" "returns" "(" [ "stream" ]
#        messageType ")" (( "{" {option | emptyStatement } "}" ) | ";")
rule rpc {
  rpc
  <rpcName> '(' [ stream ]? <request=messageType> ')'
  returns
     '(' [ stream ]? <response=messageType> ')'
  [[ '{' [<option> | <emptyStatement>]* '}' ] | ';']
}

# proto = syntax { import | package | option | topLevelDef | emptyStatement }
rule proto {
  <syntax> [ <import> | <package> | <option> | <topLevelDef> | <emptyStatement> ]*
}

# topLevelDef = message | enum | service
rule topLevelDef {
 <message> | <enum> | <service>
}

rule TOP {
  [ \s
    | [ '//' \V* [$|\n] ]
  ]*
  <proto>
}

}


A  => script/protoc.raku +24 -0
@@ 1,24 @@
#!/usr/bin/env raku

use Protobuf;

my $p = parse-proto($*ARGFILES.slurp);

for $p.services -> $svc {
  say "service name " ~ $svc.name;
  for $svc.endpoints -> $e {
    say " endpoint  " ~ $e.name;
    say '  request params ' ~ $e.request.name;
    for $e.request.fields -> $f {
      say '   name : ' ~ $f.name;
      say '   type : ' ~ $f.type;
    }
    say '  response params ' ~ $e.response.name;
    for $e.response.fields -> $f {
      say '   name : ' ~ $f.name;
      say '   type : ' ~ $f.type;
    }
  }
}



A  => t/01-load.rakutest +8 -0
@@ 1,8 @@
#!/usr/bin/env raku

use Test;

plan 1;

use-ok 'Grammar::Protobuf', 'use gramar';


A  => t/02-parse.rakutest +108 -0
@@ 1,108 @@
#!/usr/bin/env raku

use Test;
use Grammar::Protobuf;

my \protobuf = Grammar::Protobuf.new;

ok protobuf.parse("package foo.bar;",:rule<package>), 'package';
ok protobuf.parse("foo.bar",:rule<fullIdent>), 'fullIdent';
ok protobuf.parse('string', :rule<type>), 'type';
ok protobuf.parse('SubMessage', :rule<type>), 'type';
ok protobuf.parse('4', :rule<intLit>), 'int';
ok protobuf.parse('4', :rule<fieldNumber>), 'fieldNumber';
ok protobuf.parse('string name = 4;', :rule<oneofField>), 'oneofField';
ok protobuf.parse('SubMessage sub_message = 9;', :rule<oneofField>), 'oneofField';
ok protobuf.parse(q:to/X/, :rule<oneof>), 'oneof';
oneof foo {
    string name = 4;
    SubMessage sub_message = 9;
}
X

ok protobuf.parse('map<int32, string> my_map = 4;',:rule<mapField>), 'mapField';
ok protobuf.parse('map<string, Project> projects = 3;',:rule<mapField>), 'mapField';

ok protobuf.parse('9 to 11',:rule<range>), 'range';
ok protobuf.parse('reserved 2, 15, 9 to 11;',:rule<reserved>), 'reserved';
ok protobuf.parse('2, 9 to 11',:rule<ranges>), 'ranges';
# ok protobuf.parse('reserved "foo", "bar";',:rule<reserved>), 'reserved';
# ok protobuf.parse('"foo", "bar"',:rule<fieldNames>), 'fieldNames';

ok protobuf.parse('UNKNOWN = 0;',:rule<enumField>), 'enumField';
ok protobuf.parse('EnumAllowingAlias',:rule<enumName>), 'enumName';
ok protobuf.parse('"hello world"', :rule<constant>), 'constant';
ok protobuf.parse('RUNNING = 2 [(custom_option) = "hello world"];', :rule<enumField>), 'field';
ok protobuf.parse(q:to/X/, :rule<enum>), 'enum';
  enum EnumAllowingAlias {
    option allow_alias = true;
    UNKNOWN = 0;
    STARTED = 1;
    RUNNING = 2 [(custom_option) = "hello world"];
  }
  X
ok protobuf.parse(q:to/X/,:rule<field>), 'field';
  foo.bar nested_message = 2;
  X
ok protobuf.parse(q:to/X/,:rule<field>), 'field';
  repeated int32 samples = 4 [packed=true];
  X
ok protobuf.parse('foo.bar',:rule<type>), 'type';

ok protobuf.parse('Outer',:rule<messageName>), 'messageName';

ok protobuf.parse('option (my_option).a = true;',:rule<option>), 'option';

ok protobuf.parse(q:to/X/,:rule<message>), 'message';
message Outer {
  option (my_option).a = true;
  message Inner {
    int64 ival = 1;
  }
  map<int32, string> my_map = 2;
}
X

ok protobuf.parse(q:to/X/,:rule<service>), 'service';
  service SearchService {
    rpc Search (SearchRequest) returns (SearchResponse);
  }
  X

ok protobuf.parse(q:to/X/,:rule<service>), 'service';
  service SearchService {
    rpc Search (SearchRequest) returns (SearchResponse);
  }
  X

ok protobuf.parse('//', :rule<ws>), 'ws';
ok protobuf.parse('// comment', :rule<ws>), 'ws';
ok protobuf.parse(q:to/X/, :rule<package>), 'package';
package  // comment
  foo.bar;
X

ok protobuf.parse(q:to/X/.trim, :rule<syntax>), 'syntax';
syntax = "proto3";
X

ok protobuf.parse(q:to/PROTO/, :rule<proto>), 'proto';
syntax = "proto3";
import public "other.proto";
option java_package = "com.example.foo";
enum EnumAllowingAlias {
  option allow_alias = true;
  UNKNOWN = 0;
  STARTED = 1;
  RUNNING = 2 [(custom_option) = "hello world"];
}
message outer {
  option (my_option).a = true;
  message inner {   // Level 2
    int64 ival = 1;
  }
  repeated inner inner_message = 2;
  EnumAllowingAlias enum_field =3;
  map<int32, string> my_map = 4;
}
PROTO

A  => t/03-actions.rakutest +11 -0
@@ 1,11 @@
use Protobuf;
use Test;

my $m = parse-proto($?FILE.IO.parent.parent.child('eg/helloworld.proto').slurp);
my @services = $m.services;
is @services[0].name, 'Greeter', 'service name';
is @services[0].endpoints[0].name, 'SayHello', 'endpoint name';
is @services[0].endpoints[0].request.fields[0].name, 'name', 'field name';
is @services[0].endpoints[0].response.fields[0].name, 'message', 'field name';

done-testing;