From 097f963a9671452ce0686af8172f2521cbdff1fa Mon Sep 17 00:00:00 2001 From: Emma <-> Date: Fri, 10 Jul 2020 00:06:06 +0200 Subject: [PATCH] Added unit tests these will need some more negative cases, but that is going to require schema validation --- README.md | 4 +- converter/parser.js | 72 +++++++++++++--------- converter/parser.test.js | 117 ++++++++++++++++++++++++++++++++++++ converter/templates.js | 10 +-- converter/templates.test.js | 112 ++++++++++++++++++++++++++++++++++ converter/tokenizer.test.js | 58 ++++++++++++++++++ 6 files changed, 336 insertions(+), 37 deletions(-) create mode 100644 converter/parser.test.js create mode 100644 converter/templates.test.js create mode 100644 converter/tokenizer.test.js diff --git a/README.md b/README.md index 5adfec3..27b7211 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,8 @@ The idea is, that the schema converter runs on node.js, while the resulting Java both in the browser and in other javascript environments. ## What is still missing - * Conversion error handling and integrity verification. - * Unit tests, these will come last. + * Schema validity and integrity verification + * Some Conversion error handling ## How to use ### Schema converter diff --git a/converter/parser.js b/converter/parser.js index 2264cec..b0f66aa 100644 --- a/converter/parser.js +++ b/converter/parser.js @@ -5,8 +5,13 @@ const ENUM_VALUE_PATTERN = RegExp(/[A-Z][A-Z0-9]*/); class TokenError extends Error { constructor(token, expected) { - super(`Unexpected token ${token[0]} (line ${token[2] + 1}:${token[3] + 1}), expected ${expected}`); + if (token) { + super(`Unexpected token ${token[0]} (line ${token[2] + 1}:${token[3] + 1}), expected ${expected}`); + } else { + super(`Unexpected end of tokens, expected ${expected}`); + } + this.name = 'TokenError'; if (Error.captureStackTrace) { Error.captureStackTrace(this, TokenError); } @@ -24,9 +29,9 @@ function parseSchema(tokenList) { function parseSchema_type(tokenList) { let token = tokenList.shift(); - if (token[0] === 'TTYPE') { + if (token && token[0] === 'TTYPE') { return parseSchema_userType(tokenList); - } else if (token[0] === 'TENUM') { + } else if (token && token[0] === 'TENUM') { return parseSchema_userEnum(tokenList); } else { throw new TokenError(token, "'type' or 'enum'"); @@ -35,7 +40,7 @@ function parseSchema_type(tokenList) { function parseSchema_userType(tokenList) { let token = tokenList.shift(); - if (token[0] !== 'TNAME') { + if (!token || token[0] !== 'TNAME') { throw new TokenError(token, 'type name'); } @@ -47,7 +52,7 @@ function parseSchema_userType(tokenList) { function parseSchema_userEnum(tokenList) { let token = tokenList.shift(); - if (token[0] !== 'TNAME') { + if (!token || token[0] !== 'TNAME') { throw new TokenError(token, 'enum name'); } @@ -59,26 +64,26 @@ function parseSchema_userEnum(tokenList) { }; token = tokenList.shift(); - if (token[0] !== 'TLBRACE') { + if (!token || token[0] !== 'TLBRACE') { throw new TokenError(token, '{'); } let nextValue = 0; for (;;) { token = tokenList.shift(); - if (token[0] === 'TRBRACE') { + if (token && token[0] === 'TRBRACE') { break; } - if (token[0] !== 'TNAME') { + if (!token || token[0] !== 'TNAME') { throw new TokenError(token, 'value name'); } let name = token[1]; - if (tokenList[0][0] === 'TEQUAL') { + if (tokenList[0] && tokenList[0][0] === 'TEQUAL') { tokenList.shift(); token = tokenList.shift(); - if (token[0] !== 'TINTEGER') { + if (!token || token[0] !== 'TINTEGER') { throw new TokenError(token, 'integer'); } nextValue = Number(token[1]); @@ -91,6 +96,9 @@ function parseSchema_userEnum(tokenList) { function parse_type(tokenList) { let token = tokenList.shift(); + if (!token) { + throw new TokenError(token, 'something'); + } switch (token[0]) { case 'TUINT': return 'BareUInt'; @@ -126,27 +134,27 @@ function parse_type(tokenList) { let optional = {}; optional.type = 'BareOptional'; token = tokenList.shift(); - if (token[0] !== 'TLANGLE') { + if (!token || token[0] !== 'TLANGLE') { throw new TokenError(token, '<'); } optional.subtype = parse_type(tokenList); token = tokenList.shift(); - if (token[0] !== 'TRANGLE') { + if (!token || token[0] !== 'TRANGLE') { throw new TokenError(token, '>'); } return optional; case 'TDATA': - if (tokenList[0][0] === 'TLANGLE') { + if (tokenList[0] && tokenList[0][0] === 'TLANGLE') { let data = {} data.type = 'BareDataFixed'; tokenList.shift(); token = tokenList.shift(); - if (token[0] !== 'TINTEGER') { + if (!token || token[0] !== 'TINTEGER') { throw new TokenError(token, 'length'); } data.len = Number(token[1]); token = tokenList.shift(); - if (token[0] !== 'TRANGLE') { + if (!token || token[0] !== 'TRANGLE') { throw new TokenError(token, '>'); } return data; @@ -158,13 +166,13 @@ function parse_type(tokenList) { case 'TLBRACKET': let array = {}; token = tokenList.shift(); - if (token[0] === 'TRBRACKET') { + if (token && token[0] === 'TRBRACKET') { array.type = 'BareArray'; - } else if (token[0] === 'TINTEGER') { + } else if (token && token[0] === 'TINTEGER') { array.type = 'BareArrayFixed'; array.len = Number(token[1]); token = tokenList.shift(); - if (token[0] !== 'TRBRACKET') { + if (!token || token[0] !== 'TRBRACKET') { throw new TokenError(token, ']'); } } else { @@ -176,12 +184,12 @@ function parse_type(tokenList) { let map = {}; map.type = 'BareMap'; token = tokenList.shift(); - if (token[0] !== 'TLBRACKET') { + if (token && token[0] !== 'TLBRACKET') { throw new TokenError(token, '['); } map.keyType = parse_type(tokenList); token = tokenList.shift(); - if (token[0] !== 'TRBRACKET') { + if (!token || token[0] !== 'TRBRACKET') { throw new TokenError(token, ']'); } map.valueType = parse_type(tokenList); @@ -194,9 +202,9 @@ function parse_type(tokenList) { for (;;) { let subtype = parse_type(tokenList); token = tokenList.shift(); - if (token[0] === 'TEQUAL') { + if (token && token[0] === 'TEQUAL') { token = tokenList.shift(); - if (token[0] !== 'TINTEGER') { + if (!token || token[0] !== 'TINTEGER') { throw new TokenError(token, 'integer'); } index = Number(token[1]); @@ -204,9 +212,9 @@ function parse_type(tokenList) { } let sub = [subtype, index]; union.subtypes.push(sub); - if (token[0] === 'TPIPE') { + if (token && token[0] === 'TPIPE') { index++; - } else if (token[0] === 'TRPAREN') { + } else if (token && token[0] === 'TRPAREN') { break; } else { throw new TokenError(token, "'|' or ')'"); @@ -214,7 +222,11 @@ function parse_type(tokenList) { } return union; case 'TNAME': - return token[1]; + if (token[1]) { + return token[1]; + } else { + throw new TokenError(token, 'a type name'); + } default: throw new TokenError(token, 'a type'); } @@ -227,21 +239,21 @@ function parse_struct(tokenList) { let token; for (;;) { token = tokenList.shift(); - if (token[0] === 'TRBRACE') { + if (token && token[0] === 'TRBRACE') { return struct; } - if (token[0] !== 'TNAME') { + if (!token || token[0] !== 'TNAME') { throw new TokenError(token, 'field name'); } let name = token[1]; let type; token = tokenList.shift(); - if (token[0] !== 'TCOLON') { + if (!token || token[0] !== 'TCOLON') { throw new TokenError(token, ':'); } - if (tokenList[0][0] === 'TNAME') { + if (tokenList[0] && tokenList[0][0] === 'TNAME') { token = tokenList.shift(); type = token[1]; } else { @@ -249,7 +261,7 @@ function parse_struct(tokenList) { } struct.entries.push([name, type]); - if (tokenList[0][0] === 'TCOMMA') { + if (tokenList[0] && tokenList[0][0] === 'TCOMMA') { tokenList.shift(); // consume optional comma } } diff --git a/converter/parser.test.js b/converter/parser.test.js new file mode 100644 index 0000000..5ea6184 --- /dev/null +++ b/converter/parser.test.js @@ -0,0 +1,117 @@ +import {strict as assert} from 'assert'; + +import parser from './parser.js'; + +function assertSchema(tokenList, typeDefinition) { + try { + assert.deepEqual(parser.parseSchema(tokenList), typeDefinition); + } catch (e) { + console.log(e.stack); + } +} + +function throwSchema(tokenList) { + try { + assert.throws(() => parser.parseSchema(tokenList), {name: 'TokenError'}); + } catch (e) { + console.log(e.stack); + } +} + +(() => { + assertSchema([], []); + + assertSchema([ + ['TENUM'], ['TNAME', 'Name'], ['TLBRACE'], + ['TNAME', 'NAME'], + ['TNAME', 'NAME'], ['TEQUAL'], ['TINTEGER', '5'], + ['TNAME', 'NAME'], ['TRBRACE'] + ], [{ + name: 'Name', type: {type: 'BareEnum', keys: [ + [0, 'NAME'], [5, 'NAME'], [6, 'NAME']]} + }]); + + assertSchema([ + ['TTYPE'], ['TNAME', 'Name'], ['TUINT'] + ], [{ + name: 'Name', type: 'BareUInt' + }]); + + assertSchema([ + ['TTYPE'], ['TNAME', 'Name'], ['TNAME', 'Name'] + ], [{ + name: 'Name', type: 'Name' + }]); + + assertSchema([ + ['TTYPE'], ['TNAME', 'Name'], ['TOPTIONAL'], ['TLANGLE'], ['TVOID'], ['TRANGLE'] + ], [{ + name: 'Name', type: {type: 'BareOptional', subtype: 'BareVoid'} + }]); + + assertSchema([ + ['TTYPE'], ['TNAME', 'Name'], ['TDATA'] + ], [{ + name: 'Name', type: 'BareData' + }]); + + assertSchema([ + ['TTYPE'], ['TNAME', 'Name'], ['TDATA'], ['TLANGLE'], ['TINTEGER', '24'], ['TRANGLE'] + ], [{ + name: 'Name', type: {type: 'BareDataFixed', len: 24} + }]); + + assertSchema([ + ['TTYPE'], ['TNAME', 'Name'], ['TLBRACE'], ['TRBRACE'] + ], [{ + name: 'Name', type: {type: 'BareStruct', entries: []} + }]); + + assertSchema([ + ['TTYPE'], ['TNAME', 'Name'], ['TLBRACE'], + ['TNAME', 'tname'], ['TCOLON'], ['TBOOL'], + ['TNAME', 'tname'], ['TCOLON'], ['TNAME', 'Vname'], + ['TNAME', 'tname'], ['TCOLON'], ['TMAP'], ['TLBRACKET'], ['TUINT'], ['TRBRACKET'], ['TNAME', 'Type'], + ['TNAME', 'tname'], ['TCOLON'], ['TLBRACE'], ['TRBRACE'], ['TRBRACE'] + ], [{ + name: 'Name', type: {type: 'BareStruct', entries: [ + ['tname', 'BareBool'], + ['tname', 'Vname'], + ['tname', {type: 'BareMap', keyType: 'BareUInt', valueType: 'Type'}], + ['tname', {type: 'BareStruct', entries: []}] + ]} + }]); + + assertSchema([ + ['TTYPE'], ['TNAME', 'Name'], ['TLBRACKET'], ['TRBRACKET'], ['TBOOL'] + ], [{ + name: 'Name', type: {type: 'BareArray', subtype: 'BareBool'} + }]); + + assertSchema([ + ['TTYPE'], ['TNAME', 'Name'], ['TLBRACKET'], ['TINTEGER', '4'], ['TRBRACKET'], ['TBOOL'] + ], [{ + name: 'Name', type: {type: 'BareArrayFixed', subtype: 'BareBool', len: 4} + }]); + + assertSchema([ + ['TTYPE'], ['TNAME', 'Name'], ['TMAP'], ['TLBRACKET'], ['TU64'], ['TRBRACKET'], ['TNAME', 'Type'] + ], [{ + name: 'Name', type: {type: 'BareMap', keyType: 'BareU64', valueType: 'Type'} + }]); + + assertSchema([ + ['TTYPE'], ['TNAME', 'Name'], ['TLPAREN'], ['TI32'], ['TPIPE'], ['TNAME', 'Type'], ['TEQUAL'], ['TINTEGER', '4'], ['TPIPE'], ['TBOOL'], ['TRPAREN'] + ], [{ + name: 'Name', type: {type: 'BareUnion', subtypes: [ + ['BareI32', 0], ['Type', 4], ['BareBool', 5]]} + }]); + + throwSchema([[]]); + + throwSchema([['TINVALID']]); + + throwSchema([['TTYPE'], ['TNAME', 'Name']]); + + throwSchema([['TTYPE'], ['TNAME', 'Name'], ['TDATA'], ['TLANGLE']]); +})(); \ No newline at end of file diff --git a/converter/templates.js b/converter/templates.js index d72b9ee..ec86925 100644 --- a/converter/templates.js +++ b/converter/templates.js @@ -1,7 +1,7 @@ -const import_statement = (libraryPath) => "import {mapEnum,mapUnion,BareUInt,BareInt," + - "BareU8,BareU16,BareU32,BareU64,BareI8,BareI16,BareI32,BareI64,BareF32,BareF64," + - "BareBool,BareEnum,BareString,BareDataFixed,BareData,BareVoid,BareOptional," + - "BareArrayFixed,BareArray,BareMap,BareUnion,BareStruct}\n" + +const import_statement = (libraryPath) => 'import {mapEnum,mapUnion,BareUInt,BareInt,' + + 'BareU8,BareU16,BareU32,BareU64,BareI8,BareI16,BareI32,BareI64,BareF32,BareF64,' + + 'BareBool,BareEnum,BareString,BareDataFixed,BareData,BareVoid,BareOptional,' + + 'BareArrayFixed,BareArray,BareMap,BareUnion,BareStruct}\n' + `\tfrom '${libraryPath}';`; const named_class = (name, type, content) => `class ${name} extends ${type} {${content}}`; @@ -99,7 +99,7 @@ function resolveContent(objectTail) { content = struct_content(entryContent); break; default: - throw Error("Unknown content type '" + type + "'"); + throw Error('Unknown content type \'' + type + '\''); } return content; } diff --git a/converter/templates.test.js b/converter/templates.test.js new file mode 100644 index 0000000..7b49a09 --- /dev/null +++ b/converter/templates.test.js @@ -0,0 +1,112 @@ +import {strict as assert} from 'assert'; + +import templates from './templates.js'; + +const libraryPath = './bare.mjs'; +let importStatement = 'import {mapEnum,mapUnion,BareUInt,BareInt,' + + 'BareU8,BareU16,BareU32,BareU64,BareI8,BareI16,BareI32,BareI64,BareF32,BareF64,' + + 'BareBool,BareEnum,BareString,BareDataFixed,BareData,BareVoid,BareOptional,' + + 'BareArrayFixed,BareArray,BareMap,BareUnion,BareStruct}\n' + + `\tfrom '${libraryPath}';\n` + +function assertTokens(typeDefinition, code) { + try { + assert.deepEqual(templates.generateClasses(typeDefinition, libraryPath), importStatement + code); + } catch (e) { + console.log(e.stack); + } +} + +(() => { + assertTokens( [], '' + + 'export {};\n' + + 'export default {};\n'); + + assertTokens([{ + name: 'Name', type: 'BareString'}], '' + + 'class Name extends BareString {}\n' + + 'export {Name};\n' + + 'export default {Name:Name};\n'); + + assertTokens([{ + name: 'Name', type: {type: 'BareEnum', keys: [ + [0, 'NAME'], [5, 'NAME'], [6, 'NAME']]}}], '' + + 'class Name extends BareEnum {' + + 'static keys = mapEnum(this, {' + + "0:'NAME',5:'NAME',6:'NAME'});}\n" + + 'export {Name};\n' + + 'export default {Name:Name};\n'); + + assertTokens([{ + name: 'Name', type: {type: 'BareOptional', subtype: 'BareUInt8'}}], '' + + 'class Name extends BareOptional {' + + 'static type = BareUInt8;}\n' + + 'export {Name};\n' + + 'export default {Name:Name};\n'); + + assertTokens([{ + name: 'Name', type: {type: 'BareDataFixed', len: 24}}], '' + + 'class Name extends BareDataFixed {' + + 'static length = 24;}\n' + + 'export {Name};\n' + + 'export default {Name:Name};\n'); + + assertTokens([{ + name: 'Name', type: {type: 'BareStruct', entries: []}}], '' + + 'class Name extends BareStruct {' + + 'static entries = [];}\n' + + 'export {Name};\n' + + 'export default {Name:Name};\n'); + + assertTokens([{ + name: 'Name', type: {type: 'BareStruct', entries: [ + ['tname', 'BareBool'], + ['tname', 'Vname'], + ['tname', {type: 'BareMap', keyType: 'BareUInt', valueType: 'Type'}], + ['tname', {type: 'BareStruct', entries: []}] + ]}}], '' + + 'class Name extends BareStruct {' + + 'static entries = [' + + "['tname',BareBool]," + + "['tname',Vname]," + + "['tname',class extends BareMap {" + + "static keyType = BareUInt;" + + "static valueType = Type;}]," + + "['tname',class extends BareStruct {" + + "static entries = [];}]" + + '];}\n' + + 'export {Name};\n' + + 'export default {Name:Name};\n'); + + assertTokens([{ + name: 'Name', type: {type: 'BareArray', subtype: 'BareBool'}}], '' + + 'class Name extends BareArray {' + + 'static type = BareBool;}\n' + + 'export {Name};\n' + + 'export default {Name:Name};\n'); + + assertTokens([{ + name: 'Name', type: {type: 'BareArrayFixed', subtype: 'BareBool', len: 4}}], '' + + 'class Name extends BareArrayFixed {' + + 'static length = 4;' + + 'static type = BareBool;}\n' + + 'export {Name};\n' + + 'export default {Name:Name};\n'); + + assertTokens([{ + name: 'Name', type: {type: 'BareMap', keyType: 'BareU64', valueType: 'Type'}}], '' + + 'class Name extends BareMap {' + + 'static keyType = BareU64;' + + 'static valueType = Type;}\n' + + 'export {Name};\n' + + 'export default {Name:Name};\n'); + + assertTokens([{ + name: 'Name', type: {type: 'BareUnion', subtypes: [ + ['BareI32', 0], ['Type', 4], ['BareBool', 5]]}}], '' + + 'class Name extends BareUnion {' + + 'static indices = mapUnion(this, {' + + '0:BareI32,4:Type,5:BareBool});}\n' + + 'export {Name};\n' + + 'export default {Name:Name};\n'); +})(); \ No newline at end of file diff --git a/converter/tokenizer.test.js b/converter/tokenizer.test.js new file mode 100644 index 0000000..58360ff --- /dev/null +++ b/converter/tokenizer.test.js @@ -0,0 +1,58 @@ +import {strict as assert} from 'assert'; + +import tokenizer from './tokenizer.js'; + +function assertTokens(schema, tokenList) { + try { + assert.deepEqual(tokenizer.tokenizeSchema(schema), tokenList); + } catch (e) { + console.log(e.stack); + } +} + +(() => { + assertTokens('', []); + + assertTokens('42', [['TINTEGER', '42', 0, 0]]); + assertTokens('Name', [['TNAME', 'Name', 0, 0]]); + assertTokens('NAME_42', [['TNAME', 'NAME_42', 0, 0]]); + + assertTokens('type', [['TTYPE', '', 0, 0]]); + assertTokens('enum', [['TENUM', '', 0, 0]]); + assertTokens('uint', [['TUINT', '', 0, 0]]); + assertTokens('u8', [['TU8', '', 0, 0]]); + assertTokens('u16', [['TU16', '', 0, 0]]); + assertTokens('u32', [['TU32', '', 0, 0]]); + assertTokens('u64', [['TU64', '', 0, 0]]); + assertTokens('int', [['TINT', '', 0, 0]]); + assertTokens('i8', [['TI8', '', 0, 0]]); + assertTokens('i16', [['TI16', '', 0, 0]]); + assertTokens('i32', [['TI32', '', 0, 0]]); + assertTokens('i64', [['TI64', '', 0, 0]]); + assertTokens('f32', [['TF32', '', 0, 0]]); + assertTokens('bool', [['TBOOL', '', 0, 0]]); + assertTokens('string', [['TSTRING', '', 0, 0]]); + assertTokens('data', [['TDATA', '', 0, 0]]); + assertTokens('void', [['TVOID', '', 0, 0]]); + assertTokens('optional', [['TOPTIONAL', '', 0, 0]]); + assertTokens('map', [['TMAP', '', 0, 0]]); + + assertTokens('<', [['TLANGLE', '', 0, 0]]); + assertTokens('>', [['TRANGLE', '', 0, 0]]); + assertTokens('{', [['TLBRACE', '', 0, 0]]); + assertTokens('}', [['TRBRACE', '', 0, 0]]); + assertTokens('[', [['TLBRACKET', '', 0, 0]]); + assertTokens(']', [['TRBRACKET', '', 0, 0]]); + assertTokens('(', [['TLPAREN', '', 0, 0]]); + assertTokens(')', [['TRPAREN', '', 0, 0]]); + assertTokens(',', [['TCOMMA', '', 0, 0]]); + assertTokens('|', [['TPIPE', '', 0, 0]]); + assertTokens('=', [['TEQUAL', '', 0, 0]]); + assertTokens(':', [['TCOLON', '', 0, 0]]); + + try { + assert.throws(() => tokenizer.tokenizeSchema('?'), {name: 'SyntaxError'}); + } catch (e) { + console.log(e.stack); + } +})(); \ No newline at end of file -- 2.45.2