~exec64/freekdl

0e4253cdbd3eed97e7d8daa6656c234750f0a7e0 — Harry Jeffery 5 months ago aedb144
Fix -pedantic build

Until now, freekdl has been building without the -pedantic flag which
caused gcc and clang not to respect some of the rules of ANSI C. I was
mistakenly using features not available yet. This change fixes that.
4 files changed, 300 insertions(+), 299 deletions(-)

M CMakeLists.txt
M freekdl.c
M freekdl.h
M test.c
M CMakeLists.txt => CMakeLists.txt +6 -0
@@ 5,6 5,12 @@ enable_testing()
add_library(freekdl freekdl.c)
set_property(TARGET freekdl PROPERTY C_STANDARD 90)

if (MSVC)
  target_compile_options(freekdl PRIVATE /W4 /WX)
else()
  target_compile_options(freekdl PRIVATE -Wall -Wextra -pedantic -Werror)
endif()

add_executable(freekdl_test test.c)
set_property(TARGET freekdl_test PROPERTY C_STANDARD 90)


M freekdl.c => freekdl.c +286 -296
@@ 45,14 45,26 @@ static void fkdl_free(void *context, void *ptr)
  free(ptr);
}

static struct fkdl_allocator fkdl_defaultAllocator = {
  .malloc = fkdl_malloc,
  .realloc = fkdl_realloc,
  .free = fkdl_free,
};

static inline void fkdl_defaultAllocator(struct fkdl_allocator **alloc)
{
  static struct fkdl_allocator defaultAllocator = {0};
  if (*alloc && (*alloc)->malloc && (*alloc)->realloc && (*alloc)->free) {
    return;
  }
  if (!defaultAllocator.malloc) {
    defaultAllocator.malloc = fkdl_malloc;
    defaultAllocator.realloc = fkdl_realloc;
    defaultAllocator.free = fkdl_free;
  }
  *alloc = &defaultAllocator;
}

#else
static struct fkdl_allocator fkdl_defaultAllocator = {0};
static void fkdl_defaultAllocator(struct fkdl_allocator **alloc)
{
  (void)alloc;
}
#endif

static void fkdl_memset(void *dst, char value, size_t len)


@@ 90,10 102,10 @@ static int fkdl_memcmp(const void *s1, const void *s2, size_t len)

static size_t fkdl_strlen(const char *str)
{
  size_t len = 0;
  if (!str) {
    return 0;
  }
  size_t len = 0;
  while (str[len] != '\0') {
    ++len;
  }


@@ 180,8 192,7 @@ static fkdl_bool arrayPush(struct fkdl_array *array, void *item, struct fkdl_all
    array->data = newData;
  }

  char *dst = array->data;
  fkdl_memcpy(dst + (array->len * array->itemSize), item, array->itemSize);
  fkdl_memcpy(((char*)array->data) + (array->len * array->itemSize), item, array->itemSize);
  array->len += 1;
  return FKDL_TRUE;
}


@@ 268,10 279,11 @@ static fkdl_bool writeChar(struct fkdl_writeCursor *cursor, char value)
/* Writes a null-terminated string to a cursor, minus the null */
static fkdl_bool writeCString(struct fkdl_writeCursor *cursor, const char *str)
{
  size_t len, i;
  if (!cursor || !str) {
    return FKDL_FALSE;
  }
  size_t len = fkdl_strlen(str);
  len = fkdl_strlen(str);
  if (!cursor->output) {
    /* Don't write, just count bytes written */
    cursor->index += len;


@@ 280,7 292,6 @@ static fkdl_bool writeCString(struct fkdl_writeCursor *cursor, const char *str)
  if (cursor->index + len >= cursor->outputLen) {
    return FKDL_FALSE;
  }
  size_t i;
  for (i = 0; i < len; ++i) {
    cursor->output[cursor->index++] = str[i];
  }


@@ 290,6 301,7 @@ static fkdl_bool writeCString(struct fkdl_writeCursor *cursor, const char *str)
/* Writes a fkdl_string to a cursor */
static fkdl_bool writeString(struct fkdl_writeCursor *cursor, const struct fkdl_string *str)
{
  size_t i;
  if (!cursor || !str) {
    return FKDL_FALSE;
  }


@@ 301,7 313,6 @@ static fkdl_bool writeString(struct fkdl_writeCursor *cursor, const struct fkdl_
  if (cursor->index + str->len >= cursor->outputLen) {
    return FKDL_FALSE;
  }
  size_t i;
  for (i = 0; i < str->len; ++i) {
    cursor->output[cursor->index++] = str->data[i];
  }


@@ 310,6 321,7 @@ static fkdl_bool writeString(struct fkdl_writeCursor *cursor, const struct fkdl_

static fkdl_bool cursorMemset(struct fkdl_writeCursor *cursor, char value, size_t len)
{
  size_t i;
  if (!cursor) {
    return FKDL_FALSE;
  }


@@ 321,7 333,6 @@ static fkdl_bool cursorMemset(struct fkdl_writeCursor *cursor, char value, size_
  if (cursor->index + len >= cursor->outputLen) {
    return FKDL_FALSE;
  }
  size_t i;
  for (i = 0; i < len; ++i) {
    cursor->output[cursor->index++] = value;
  }


@@ 341,43 352,43 @@ static fkdl_bool atEof(const struct fkdl_cursor *cursor)
/* of bytes read in outSize. */
static fkdl_bool readUtf8Char(const char *input, size_t inputLen, uint32_t *out, size_t *outSize)
{
  uint32_t value = 0;
  size_t size = 0;
  if (!input) {
    return FKDL_FALSE;
  }
  uint32_t value = 0;
  size_t size = 0;
  if (inputLen >= 1 && (input[0] & 0b10000000) == 0) {
    value = input[0] & 0b01111111;
  if (inputLen >= 1 && (input[0] & 0x80) == 0) {
    value = input[0];
    size = 1;
  } else if (inputLen >= 2 &&
             (input[0] & 0b11100000) == 0b11000000 &&
             (input[1] & 0b11000000) == 0b10000000) {
    value = input[0] & 0b00011111;
             (input[0] & 0xE0) == 0xC0 &&
             (input[1] & 0xC0) == 0x80) {
    value = input[0] & 0x1F;
    value <<= 6;
    value |= input[1] & 0b00111111;
    value |= input[1] & 0x3F;
    size = 2;
  } else if (inputLen >= 3 &&
             (input[0] & 0b11110000) == 0b11100000 &&
             (input[1] & 0b11000000) == 0b10000000 &&
             (input[2] & 0b11000000) == 0b10000000) {
    value = (input[0] & 0b00001111);
             (input[0] & 0xF0) == 0xE0 &&
             (input[1] & 0xC0) == 0x80 &&
             (input[2] & 0xC0) == 0x80) {
    value = (input[0] & 0x0F);
    value <<= 6;
    value |= input[1] & 0b00111111;
    value |= input[1] & 0x3F;
    value <<= 6;
    value |= input[2] & 0b00111111;
    value |= input[2] & 0x3F;
    size = 3;
  } else if (inputLen >= 4 &&
             (input[0] & 0b11111000) == 0b11110000 &&
             (input[1] & 0b11000000) == 0b10000000 &&
             (input[2] & 0b11000000) == 0b10000000 &&
             (input[3] & 0b11000000) == 0b10000000) {
    value = (input[0] & 0b00000111);
             (input[0] & 0xF8) == 0xF0 &&
             (input[1] & 0xC0) == 0x80 &&
             (input[2] & 0xC0) == 0x80 &&
             (input[3] & 0xC0) == 0x80) {
    value = (input[0] & 0x07);
    value <<= 6;
    value |= input[1] & 0b00111111;
    value |= input[1] & 0x3F;
    value <<= 6;
    value |= input[2] & 0b00111111;
    value |= input[2] & 0x3F;
    value <<= 6;
    value |= input[3] & 0b00111111;
    value |= input[3] & 0x3F;
    size = 4;
  } else {
    return FKDL_FALSE;


@@ 415,28 426,29 @@ static fkdl_bool writeUtf8Char(
  uint32_t codePoint,
  size_t *outSize)
{
  size_t size = 0;

  if (!output) {
    return FKDL_FALSE;
  }
  size_t size = 0;

  if (outputLen >= 1 && codePoint < 0x80) {
    output[0] = 0b01111111 & codePoint;
    output[0] = 0x7F & codePoint;
    size = 1;
  } else if (outputLen >= 2 && codePoint < 0x800) {
    output[0] = 0b11000000 | ((codePoint >> 6) & 0b00011111);
    output[1] = 0b10000000 | ((codePoint >> 0) & 0b00111111);
    output[0] = 0xC0 | ((codePoint >> 6) & 0x1F);
    output[1] = 0x80 | ((codePoint >> 0) & 0x3F);
    size = 2;
  } else if (outputLen >= 3 && codePoint < 0x10000) {
    output[0] = 0b11100000 | ((codePoint >> 12) & 0b00001111);
    output[1] = 0b10000000 | ((codePoint >> 6) &  0b00111111);
    output[2] = 0b10000000 | ((codePoint >> 0) &  0b00111111);
    output[0] = 0xE0 | ((codePoint >> 12) & 0x0F);
    output[1] = 0x80 | ((codePoint >> 6) &  0x3F);
    output[2] = 0x80 | ((codePoint >> 0) &  0x3F);
    size = 3;
  } else if (outputLen >= 4 && codePoint < 0x110000) {
    output[0] = 0b11110000 | ((codePoint >> 18) & 0b00000111);
    output[1] = 0b10000000 | ((codePoint >> 12) & 0b00111111);
    output[2] = 0b10000000 | ((codePoint >> 6) &  0b00111111);
    output[3] = 0b10000000 | ((codePoint >> 0) &  0b00111111);
    output[0] = 0xF0 | ((codePoint >> 18) & 0x07);
    output[1] = 0x80 | ((codePoint >> 12) & 0x3F);
    output[2] = 0x80 | ((codePoint >> 6) &  0x3F);
    output[3] = 0x80 | ((codePoint >> 0) &  0x3F);
    size = 4;
  } else {
    return FKDL_FALSE;


@@ 469,17 481,18 @@ static const char *toEscapeCode(uint32_t codePoint)
/* Write a fkdl_string to a cursor, escaping anything required to be a valid quoted string */
static fkdl_bool writeEscapedString(struct fkdl_writeCursor *cursor, const struct fkdl_string *str)
{
  size_t i;
  if (!cursor || !str) {
    return FKDL_FALSE;
  }
  size_t i;
  for (i = 0; i < str->len; ) {
    uint32_t codePoint;
    size_t step;
    const char *escape;
    if (!readUtf8Char(str->data + i, str->len - i, &codePoint, &step)) {
      return FKDL_FALSE;
    }
    const char *escape = toEscapeCode(codePoint);
    escape = toEscapeCode(codePoint);
    if (escape) {
      if (!writeCString(cursor, escape)) {
        return FKDL_FALSE;


@@ 507,23 520,23 @@ static fkdl_bool peekChar(
  size_t *outSize,
  struct fkdl_error *error)
{
  fkdl_bool result;
  if (!cursor) {
    return FKDL_FALSE;
  }
  if (atEof(cursor)) {
    errorFromCursor(error, cursor, "Unexpected EOF");
    return 0;
    return FKDL_FALSE;
  }
  fkdl_bool result = readUtf8Char(
  result = readUtf8Char(
    cursor->input + cursor->index,
    cursor->inputLen - cursor->index,
    out,
    outSize);
  if (result) {
    return FKDL_TRUE;
  if (!result) {
    errorFromCursor(error, cursor, "Invalid UTF-8 character");
  }
  errorFromCursor(error, cursor, "Invalid UTF-8 character");
  return FKDL_FALSE;
  return result;
}

static fkdl_bool isWhitespace(uint32_t codePoint)


@@ 559,11 572,11 @@ static fkdl_bool peekStr(
    const struct fkdl_cursor *cursor,
    const char *str)
{
  size_t i, len;
  if (!cursor || !str) {
    return FKDL_FALSE;
  }
  size_t i;
  size_t len = fkdl_strlen(str);
  len = fkdl_strlen(str);
  for (i = 0; i < len; ++i) {
    if (cursor->index + i >= cursor->inputLen) {
      return FKDL_FALSE;


@@ 629,11 642,14 @@ static fkdl_bool peekWord(
    const struct fkdl_cursor *cursor,
    const char *str)
{
  size_t i, len;
  struct fkdl_cursor end;
  uint32_t codePoint;

  if (!cursor || !str) {
    return FKDL_FALSE;
  }
  size_t len = fkdl_strlen(str);
  size_t i;
  len = fkdl_strlen(str);
  for (i = 0; i < len; ++i) {
    if (cursor->index + i >= cursor->inputLen) {
      return FKDL_FALSE;


@@ 642,9 658,8 @@ static fkdl_bool peekWord(
      return FKDL_FALSE;
    }
  }
  struct fkdl_cursor end = *cursor;
  end = *cursor;
  end.index += len;
  uint32_t codePoint;
  if (!peekChar(&end, &codePoint, NULL, NULL)) {
    /* EOF can count as end of a word */
    return FKDL_TRUE;


@@ 659,11 674,11 @@ static fkdl_bool consumeChar(
  size_t *outSize,
  struct fkdl_error *error)
{
  uint32_t localOut;
  size_t localOutSize;
  if (!cursor) {
    return FKDL_FALSE;
  }
  uint32_t localOut;
  size_t localOutSize;
  if (!peekChar(cursor, &localOut, &localOutSize, error)) {
    return FKDL_FALSE;
  }


@@ 695,11 710,11 @@ static fkdl_bool consumeHexCode(
  size_t *outSize,
  struct fkdl_error *error)
{
  uint32_t value = 0;
  size_t len = 0;
  if (!cursor) {
    return FKDL_FALSE;
  }
  uint32_t value = 0;
  size_t len = 0;
  while (1) {
    uint32_t codePoint;
    size_t step;


@@ 752,11 767,12 @@ static fkdl_bool consumeNewline(
  struct fkdl_cursor *cursor,
  struct fkdl_error *error)
{
  struct fkdl_cursor start;
  uint32_t codePoint;
  if (!cursor) {
    return FKDL_FALSE;
  }
  struct fkdl_cursor start = *cursor;
  uint32_t codePoint;
  start = *cursor;
  if (!consumeChar(cursor, &codePoint, NULL, error)) {
    return FKDL_FALSE;
  }


@@ 800,20 816,22 @@ static fkdl_bool consumeWhitespace(
  struct fkdl_cursor *cursor,
  struct fkdl_error *error)
{
  size_t commentDepth = 0;
  struct fkdl_cursor start;
  if (!cursor) {
    return FKDL_FALSE;
  }
  size_t commentDepth = 0;
  struct fkdl_cursor start = *cursor;
  start = *cursor;

  while (1) {
    uint32_t codePoint;
    size_t step;

    if (atEof(cursor)) {
      /* Reached EOF */
      break;
    }

    uint32_t codePoint;
    size_t step;
    if (!peekChar(cursor, &codePoint, &step, error)) {
      return FKDL_FALSE;
    }


@@ 867,11 885,11 @@ static fkdl_bool consumeWhitespace(
 * used for finding the end of a raw string
 */
static size_t countHashes(const char *ptr, size_t maxLen) {
  size_t num = 0;
  size_t i;
  if (!ptr) {
    return 0;
  }
  size_t num = 0;
  size_t i;
  for (i = 0; i < maxLen; ++i) {
    if (ptr[i] == '#') {
      num += 1;


@@ 893,17 911,18 @@ static fkdl_bool measureAndConsumeString(
  fkdl_bool *outRaw,
  struct fkdl_error *error)
{
  uint32_t codePoint;
  fkdl_bool isRaw, isEscape;
  size_t hashLen, start, len;

  if (!cursor) {
    return FKDL_FALSE;
  }
  uint32_t codePoint;

  if (!consumeChar(cursor, &codePoint, NULL, error)) {
    return FKDL_FALSE;
  }

  fkdl_bool isRaw;

  if (codePoint == '"') {
    isRaw = FKDL_FALSE;
  } else if (codePoint == 'r') {


@@ 913,7 932,7 @@ static fkdl_bool measureAndConsumeString(
    return FKDL_FALSE;
  }

  size_t hashLen = 0; /* How many #s are in the raw start/end quote */
  hashLen = 0; /* How many #s are in the raw start/end quote */
  if (isRaw) {
    while (1) {
      if (!consumeChar(cursor, &codePoint, NULL, error)) {


@@ 931,10 950,9 @@ static fkdl_bool measureAndConsumeString(
  }

  /* We're now at the first byte of the string's body */
  size_t start = cursor->index;

  size_t len = 0;
  fkdl_bool isEscape = FKDL_FALSE;
  start = cursor->index;
  len = 0;
  isEscape = FKDL_FALSE;
  while (1) {
    size_t step;
    if (!consumeChar(cursor, &codePoint, &step, error)) {


@@ 983,14 1001,16 @@ static fkdl_bool consumeString(
  struct fkdl_token *token,
  struct fkdl_error *error)
{
  struct fkdl_cursor startCursor;
  size_t start;
  size_t len;
  fkdl_bool isRaw;

  if (!cursor || !token) {
    return FKDL_FALSE;
  }
  struct fkdl_cursor startCursor = *cursor;
  startCursor = *cursor;

  size_t start;
  size_t len;
  fkdl_bool isRaw;
  if (!measureAndConsumeString(cursor, &start, &len, &isRaw, error)) {
    return FKDL_FALSE;
  }


@@ 1033,11 1053,12 @@ static fkdl_bool consumeNumber(
  struct fkdl_token *token,
  struct fkdl_error *error)
{
  struct fkdl_cursor start;
  uint32_t codePoint;
  if (!cursor || !token) {
    return FKDL_FALSE;
  }
  struct fkdl_cursor start = *cursor;
  uint32_t codePoint;
  start = *cursor;
  while (1) {
    if (atEof(cursor)) {
      break;


@@ 1063,12 1084,14 @@ static fkdl_bool consumeNumber(
/* True if the cursor points to the start of a raw string */
static fkdl_bool isRawStringStart(const struct fkdl_cursor *startCursor)
{
  struct fkdl_cursor cursor;
  uint32_t codePoint;

  if (!startCursor) {
    return FKDL_FALSE;
  }
  struct fkdl_cursor cursor = *startCursor;
  cursor = *startCursor;
  
  uint32_t codePoint;
  if (!consumeChar(&cursor, &codePoint, NULL, NULL)) {
    return FKDL_FALSE;
  }


@@ 1102,10 1125,10 @@ static fkdl_bool isStartOfIdentifier(uint32_t codePoint)
/* Consumes an identifier from cursor */
static fkdl_bool consumeIdentifier(struct fkdl_cursor *cursor, struct fkdl_error *error)
{
  uint32_t codePoint;
  if (!cursor) {
    return FKDL_FALSE;
  }
  uint32_t codePoint;

  if (!consumeChar(cursor, &codePoint, NULL, error)) {
    return FKDL_FALSE;


@@ 1139,10 1162,10 @@ static fkdl_bool consumeIdentifier(struct fkdl_cursor *cursor, struct fkdl_error
/* consumes a type annotation from cursor */
static fkdl_bool consumeTypeAnnotation(struct fkdl_cursor *cursor, struct fkdl_error *error)
{
  uint32_t codePoint;
  if (!cursor) {
    return FKDL_FALSE;
  }
  uint32_t codePoint;
  if (!consumeChar(cursor, &codePoint, NULL, error)) {
    return FKDL_FALSE;
  }


@@ 1190,14 1213,13 @@ static fkdl_bool tokenise(
  struct fkdl_error *error, /* Optional parse error info */
  struct fkdl_allocator *allocator) /* the allocator to use */
{
  struct fkdl_cursor cursor = {
    .input = input,
    .inputLen = inputLen,
    .index = 0,
    .line = 1,
    .column = 0,
    .prevWhitespace = FKDL_FALSE,
  };
  struct fkdl_cursor cursor = {0};
  cursor.input = input;
  cursor.inputLen = inputLen;
  cursor.index = 0;
  cursor.line = 1;
  cursor.column = 0;
  cursor.prevWhitespace = FKDL_FALSE;

  if (!input) {
    errorFromCursor(error, &cursor, "NULL input");


@@ 1210,13 1232,12 @@ static fkdl_bool tokenise(

    if (atEof(&cursor)) {
      /* Reached EOF, throw down a node terminator */
      struct fkdl_token token = {
        .type = FKDL_TOK_NODE_TERMINATOR,
        .index = cursor.index,
        .length = 0,
        .line = cursor.line,
        .column = cursor.column,
      };
      struct fkdl_token token = {0};
      token.type = FKDL_TOK_NODE_TERMINATOR;
      token.index = cursor.index;
      token.length = 0;
      token.line = cursor.line;
      token.column = cursor.column;
      if (!arrayPush(tokens, &token, allocator)) {
        return FKDL_FALSE;
      }


@@ 1236,18 1257,17 @@ static fkdl_bool tokenise(

    if (isNewline(codePoint)) {
      struct fkdl_cursor start = cursor;
      struct fkdl_token token = {0};

      if (!consumeNewline(&cursor, error)) {
        return FKDL_FALSE;
      }

      struct fkdl_token token = {
        .type = FKDL_TOK_NODE_TERMINATOR,
        .index = start.index,
        .length = cursor.index - start.index,
        .line = start.line,
        .column = start.column,
      };
      token.type = FKDL_TOK_NODE_TERMINATOR;
      token.index = start.index;
      token.length = cursor.index - start.index;
      token.line = start.line;
      token.column = start.column;
      if (!arrayPush(tokens, &token, allocator)) {
        return FKDL_FALSE;
      }


@@ 1278,16 1298,16 @@ static fkdl_bool tokenise(

    if (peekStr(&cursor, "/-")) {
      struct fkdl_cursor start = cursor;
      struct fkdl_token token = {0};

      consumeChar(&cursor, NULL, NULL, NULL);
      consumeChar(&cursor, NULL, NULL, NULL);

      struct fkdl_token token = {
        .type = FKDL_TOK_SLASHDASH,
        .index = start.index,
        .length = cursor.index - start.index,
        .line = start.line,
        .column = start.column,
      };
      token.type = FKDL_TOK_SLASHDASH;
      token.index = start.index;
      token.length = cursor.index - start.index;
      token.line = start.line;
      token.column = start.column;
      if (!arrayPush(tokens, &token, allocator)) {
        return FKDL_FALSE;
      }


@@ 1296,17 1316,16 @@ static fkdl_bool tokenise(

    if (codePoint == '(') {
      struct fkdl_cursor start = cursor;
      struct fkdl_token token = {0};
      if (!consumeTypeAnnotation(&cursor, error)) {
        return FKDL_FALSE;
      }

      struct fkdl_token token = {
        .type = FKDL_TOK_TYPE_ANNOTATION,
        .index = start.index + 1, /* +1 to skip the '(' */
        .length = cursor.index - start.index - 2, /* same reason as above */
        .line = start.line,
        .column = start.column,
      };
      token.type = FKDL_TOK_TYPE_ANNOTATION;
      token.index = start.index + 1; /* +1 to skip the '(' */
      token.length = cursor.index - start.index - 2; /* same reason as above */
      token.line = start.line;
      token.column = start.column;
      if (!arrayPush(tokens, &token, allocator)) {
        return FKDL_FALSE;
      }


@@ 1338,17 1357,16 @@ static fkdl_bool tokenise(

    if (peekWord(&cursor, "null")) {
      struct fkdl_cursor start = cursor;
      struct fkdl_token token = {0};
      size_t i;
      for (i = 0; i < 4; ++i) {
        consumeChar(&cursor, NULL, NULL, NULL);
      }
      struct fkdl_token token = {
        .type = FKDL_TOK_NULL,
        .index = start.index,
        .length = cursor.index - start.index,
        .line = start.line,
        .column = start.column,
      };
      token.type = FKDL_TOK_NULL;
      token.index = start.index;
      token.length = cursor.index - start.index;
      token.line = start.line;
      token.column = start.column;
      if (!arrayPush(tokens, &token, allocator)) {
        return FKDL_FALSE;
      }


@@ 1357,17 1375,16 @@ static fkdl_bool tokenise(

    if (peekWord(&cursor, "true")) {
      struct fkdl_cursor start = cursor;
      struct fkdl_token token = {0};
      size_t i;
      for (i = 0; i < 4; ++i) {
        consumeChar(&cursor, NULL, NULL, NULL);
      }
      struct fkdl_token token = {
        .type = FKDL_TOK_BOOL,
        .index = start.index,
        .length = cursor.index - start.index,
        .line = start.line,
        .column = start.column,
      };
      token.type = FKDL_TOK_BOOL;
      token.index = start.index;
      token.length = cursor.index - start.index;
      token.line = start.line;
      token.column = start.column;
      if (!arrayPush(tokens, &token, allocator)) {
        return FKDL_FALSE;
      }


@@ 1376,17 1393,16 @@ static fkdl_bool tokenise(

    if (peekWord(&cursor, "false")) {
      struct fkdl_cursor start = cursor;
      struct fkdl_token token = {0};
      size_t i;
      for (i = 0; i < 5; ++i) {
        consumeChar(&cursor, NULL, NULL, NULL);
      }
      struct fkdl_token token = {
        .type = FKDL_TOK_BOOL,
        .index = start.index,
        .length = cursor.index - start.index,
        .line = start.line,
        .column = start.column,
      };
      token.type = FKDL_TOK_BOOL;
      token.index = start.index;
      token.length = cursor.index - start.index;
      token.line = start.line;
      token.column = start.column;
      if (!arrayPush(tokens, &token, allocator)) {
        return FKDL_FALSE;
      }


@@ 1395,16 1411,15 @@ static fkdl_bool tokenise(

    if (isStartOfIdentifier(codePoint)) {
      struct fkdl_cursor start = cursor;
      struct fkdl_token token = {0};
      if (!consumeIdentifier(&cursor, error)) {
        return FKDL_FALSE;
      }
      struct fkdl_token token = {
        .type = FKDL_TOK_IDENTIFIER,
        .index = start.index,
        .length = cursor.index - start.index,
        .line = start.line,
        .column = start.column,
      };
      token.type = FKDL_TOK_IDENTIFIER;
      token.index = start.index;
      token.length = cursor.index - start.index;
      token.line = start.line;
      token.column = start.column;
      if (!arrayPush(tokens, &token, allocator)) {
        return FKDL_FALSE;
      }


@@ 1413,14 1428,13 @@ static fkdl_bool tokenise(

    if (codePoint == '{') {
      struct fkdl_cursor start = cursor;
      struct fkdl_token token = {0};
      consumeChar(&cursor, NULL, NULL, NULL);
      struct fkdl_token token = {
        .type = FKDL_TOK_OPEN_BRACE,
        .index = start.index,
        .length = cursor.index - start.index,
        .line = start.line,
        .column = start.column,
      };
      token.type = FKDL_TOK_OPEN_BRACE;
      token.index = start.index;
      token.length = cursor.index - start.index;
      token.line = start.line;
      token.column = start.column;
      if (!arrayPush(tokens, &token, allocator)) {
        return FKDL_FALSE;
      }


@@ 1429,24 1443,22 @@ static fkdl_bool tokenise(

    if (codePoint == '}') {
      struct fkdl_cursor start = cursor;
      struct fkdl_token termToken = {0};
      struct fkdl_token braceToken = {0};
      consumeChar(&cursor, NULL, NULL, NULL);
      struct fkdl_token termToken = {
        .type = FKDL_TOK_NODE_TERMINATOR,
        .index = start.index,
        .length = cursor.index - start.index,
        .line = start.line,
        .column = start.column,
      };
      termToken.type = FKDL_TOK_NODE_TERMINATOR;
      termToken.index = start.index;
      termToken.length = cursor.index - start.index;
      termToken.line = start.line;
      termToken.column = start.column;
      if (!arrayPush(tokens, &termToken, allocator)) {
        return FKDL_FALSE;
      }
      struct fkdl_token braceToken = {
        .type = FKDL_TOK_CLOSE_BRACE,
        .index = start.index,
        .length = cursor.index - start.index,
        .line = start.line,
        .column = start.column,
      };
      braceToken.type = FKDL_TOK_CLOSE_BRACE;
      braceToken.index = start.index;
      braceToken.length = cursor.index - start.index;
      braceToken.line = start.line;
      braceToken.column = start.column;
      if (!arrayPush(tokens, &braceToken, allocator)) {
        return FKDL_FALSE;
      }


@@ 1455,14 1467,13 @@ static fkdl_bool tokenise(

    if (codePoint == ';') {
      struct fkdl_cursor start = cursor;
      struct fkdl_token token = {0};
      consumeChar(&cursor, NULL, NULL, NULL);
      struct fkdl_token token = {
        .type = FKDL_TOK_NODE_TERMINATOR,
        .index = start.index,
        .length = cursor.index - start.index,
        .line = start.line,
        .column = start.column,
      };
      token.type = FKDL_TOK_NODE_TERMINATOR;
      token.index = start.index;
      token.length = cursor.index - start.index;
      token.line = start.line;
      token.column = start.column;
      if (!arrayPush(tokens, &token, allocator)) {
        return FKDL_FALSE;
      }


@@ 1471,14 1482,13 @@ static fkdl_bool tokenise(

    if (codePoint == '=' && !cursor.prevWhitespace) {
      struct fkdl_cursor start = cursor;
      struct fkdl_token token = {0};
      consumeChar(&cursor, NULL, NULL, NULL);
      struct fkdl_token token = {
        .type = FKDL_TOK_PROPERTY_EQUALS,
        .index = start.index,
        .length = cursor.index - start.index,
        .line = start.line,
        .column = start.column,
      };
      token.type = FKDL_TOK_PROPERTY_EQUALS;
      token.index = start.index;
      token.length = cursor.index - start.index;
      token.line = start.line;
      token.column = start.column;
      if (!arrayPush(tokens, &token, allocator)) {
        return FKDL_FALSE;
      }


@@ 1544,12 1554,16 @@ static struct fkdl_string copyString(struct fkdl_allocator *allocator, const cha
/* Return how much space a string will need after unescaping is applied */
static fkdl_bool unescapedStringLength(const char *str, size_t strLen, size_t *newLen)
{
  size_t i, len;
  if (!str) {
    return FKDL_FALSE;
  }
  size_t i;
  size_t len = 0;
  len = 0;
  for (i = 0; i < strLen; ++i) {
    struct fkdl_cursor cursor = {0};
    uint32_t codePoint;
    size_t bytesRead;

    if (str[i] != '\\') {
      len += 1;
      continue;


@@ 1593,13 1607,9 @@ static fkdl_bool unescapedStringLength(const char *str, size_t strLen, size_t *n
      return FKDL_FALSE;
    }

    struct fkdl_cursor cursor = {
      .input = str + i + 3,
      .inputLen = strLen - i - 3,
      .line = 1,
    };
    uint32_t codePoint;
    size_t bytesRead;
    cursor.input = str + i + 3;
    cursor.inputLen = strLen - i - 3;
    cursor.line = 1;
    if (!consumeHexCode(&cursor, &codePoint, &bytesRead, NULL)) {
      return FKDL_FALSE;
    }


@@ 1615,11 1625,16 @@ static fkdl_bool unescapedStringLength(const char *str, size_t strLen, size_t *n
/* copy a string from in to out, applying any escape patterns found */
static fkdl_bool unescapeString(char *out, const char *in, size_t inLen)
{
  size_t i;
  if (!in || !out) {
    return FKDL_FALSE;
  }
  size_t i;
  for (i = 0; i < inLen; ++i) {
    struct fkdl_cursor cursor = {0};
    uint32_t codePoint;
    size_t bytesRead;
    size_t bytesWritten;

    if (in[i] != '\\') {
      *out = in[i];
      out += 1;


@@ 1691,14 1706,9 @@ static fkdl_bool unescapeString(char *out, const char *in, size_t inLen)
      return FKDL_FALSE;
    }

    struct fkdl_cursor cursor = {
      .input = in + i + 3,
      .inputLen = inLen - i - 3,
      .line = 1,
    };
    uint32_t codePoint;
    size_t bytesRead;
    size_t bytesWritten;
    cursor.input = in + i + 3;
    cursor.inputLen = inLen - i - 3;
    cursor.line = 1;
    if (!consumeHexCode(&cursor, &codePoint, &bytesRead, NULL)) {
      return FKDL_FALSE;
    }


@@ 1715,10 1725,10 @@ static fkdl_bool unescapeString(char *out, const char *in, size_t inLen)
static struct fkdl_string unescapeAndCopyString(struct fkdl_allocator *allocator, const char *str, size_t strLen)
{
  struct fkdl_string out = {0};
  size_t rawLen;
  if (!allocator || !allocator->malloc || !str) {
    return out;
  }
  size_t rawLen;
  if (!unescapedStringLength(str, strLen, &rawLen)) {
    return out;
  }


@@ 1819,9 1829,7 @@ static fkdl_bool tokenTypeIsValue(enum fkdl_token_type type)

void fkdl_cleanupString(struct fkdl_string *string, struct fkdl_allocator *allocator)
{
  if (!allocator) {
    allocator = &fkdl_defaultAllocator;
  }
  fkdl_defaultAllocator(&allocator);
  if (!string || !allocator->free) {
    return;
  }


@@ 1852,9 1860,7 @@ void fkdl_cleanupProperty(struct fkdl_property *property, struct fkdl_allocator 

void fkdl_cleanupNode(struct fkdl_node *node, struct fkdl_allocator *allocator)
{
  if (!allocator) {
    allocator = &fkdl_defaultAllocator;
  }
  fkdl_defaultAllocator(&allocator);
  if (!node || !allocator->free) {
    return;
  }


@@ 1895,9 1901,7 @@ void fkdl_cleanupNode(struct fkdl_node *node, struct fkdl_allocator *allocator)

void fkdl_cleanupDocument(struct fkdl_document *document, struct fkdl_allocator *allocator)
{
  if (!allocator) {
    allocator = &fkdl_defaultAllocator;
  }
  fkdl_defaultAllocator(&allocator);
  if (!document || !allocator->free) {
    return;
  }


@@ 1916,9 1920,7 @@ fkdl_bool fkdl_nodeAddArgument(
  const struct fkdl_value *value,
  struct fkdl_allocator *allocator)
{
  if (!allocator) {
    allocator = &fkdl_defaultAllocator;
  }
  fkdl_defaultAllocator(&allocator);
  if (!node || !value || !allocator->malloc || !allocator->realloc) {
    return FKDL_FALSE;
  }


@@ 1953,9 1955,9 @@ fkdl_bool fkdl_nodeRemoveArgument(
  size_t argumentIndex,
  struct fkdl_allocator *allocator)
{
  if (!allocator) {
    allocator = &fkdl_defaultAllocator;
  }
  size_t i;

  fkdl_defaultAllocator(&allocator);
  if (!node || !allocator->free || !node->arguments) {
    return FKDL_FALSE;
  }


@@ 1965,7 1967,6 @@ fkdl_bool fkdl_nodeRemoveArgument(

  fkdl_cleanupValue(&node->arguments[argumentIndex], allocator);

  size_t i;
  for (i = argumentIndex; i + 1 < node->argumentsLen; ++i) {
    node->arguments[i] = node->arguments[i+1];
  }


@@ 2009,22 2010,23 @@ static struct fkdl_property *nodeAllocProperty(
/* Takes the last property of a node and shifts it left until sorted */
static void nodeSortProperty(struct fkdl_node *node)
{
  size_t i;
  if (!node) {
    return;
  }
  size_t i;
  for (i = node->propertiesLen; i >= 2; --i) {
    struct fkdl_property *rhs = &node->properties[i-1];
    struct fkdl_property *lhs = &node->properties[i-2];
    size_t len = lhs->key.len < rhs->key.len ? lhs->key.len : rhs->key.len;
    int cmp = fkdl_memcmp(lhs->key.data, rhs->key.data, len);
    struct fkdl_property tmp;
    if (cmp < 0) {
      break;
    }
    if (cmp == 0 && lhs->key.len <= rhs->key.len) {
      break;
    }
    struct fkdl_property tmp = *lhs;
    tmp = *lhs;
    *lhs = *rhs;
    *rhs = tmp;
  }


@@ 2035,18 2037,16 @@ fkdl_bool fkdl_nodeAddProperty(
  const struct fkdl_property *property,
  struct fkdl_allocator *allocator)
{
  if (!allocator) {
    allocator = &fkdl_defaultAllocator;
  }
  size_t i;
  size_t propertyIndex;
  fkdl_bool foundProperty = FKDL_FALSE;

  fkdl_defaultAllocator(&allocator);
  if (!node || !property) {
    return FKDL_FALSE;
  }

  /* See if we're overwriting an existing property */
  fkdl_bool foundProperty = FKDL_FALSE;
  size_t propertyIndex;

  size_t i;
  for (i = 0; i < node->propertiesLen; ++i) {
    if (node->properties[i].key.len != property->key.len)
      continue;


@@ 2084,9 2084,10 @@ fkdl_bool fkdl_nodeRemoveProperty(
  const struct fkdl_string *key,
  struct fkdl_allocator *allocator)
{
  if (!allocator) {
    allocator = &fkdl_defaultAllocator;
  }
  fkdl_bool foundProperty = FKDL_FALSE;
  size_t propertyIndex, i;

  fkdl_defaultAllocator(&allocator);
  if (!node || !key) {
    return FKDL_FALSE;
  }


@@ 2094,10 2095,7 @@ fkdl_bool fkdl_nodeRemoveProperty(
  if (!node->properties) {
    return FKDL_FALSE;
  }
  fkdl_bool foundProperty = FKDL_FALSE;
  size_t propertyIndex;

  size_t i;
  for (i = 0; i < node->propertiesLen; ++i) {
    if (node->properties[i].key.len != key->len) {
      continue;


@@ 2132,12 2130,13 @@ static fkdl_bool buildProperty(
  const char *key,
  const char *value)
{
  struct fkdl_string valueStr = {0};
  struct fkdl_string keyStr;
  fkdl_cleanupProperty(property, allocator);
  struct fkdl_string keyStr = copyString(allocator, key, fkdl_strlen(key));
  keyStr = copyString(allocator, key, fkdl_strlen(key));
  if (!keyStr.data) {
    return FKDL_FALSE;
  }
  struct fkdl_string valueStr = {0};
  if (type != FKDL_BOOL) {
    valueStr = copyString(allocator, value, fkdl_strlen(value));
    if (!valueStr.data) {


@@ 2157,14 2156,13 @@ fkdl_bool fkdl_nodeSetStringProperty(
  const char *value,
  struct fkdl_allocator *allocator)
{
  if (!allocator) {
    allocator = &fkdl_defaultAllocator;
  }
  struct fkdl_property property;

  fkdl_defaultAllocator(&allocator);
  if (!node || !key || !value) {
    return FKDL_FALSE;
  }

  struct fkdl_property property;
  if (!buildProperty(&property, allocator, FKDL_STRING, key, value)) {
    return FKDL_FALSE;
  }


@@ 2183,15 2181,14 @@ fkdl_bool fkdl_nodeSetBoolProperty(
  fkdl_bool value,
  struct fkdl_allocator *allocator)
{
  if (!allocator) {
    allocator = &fkdl_defaultAllocator;
  }
  struct fkdl_property property;
  const char *valueStr = value ? "true" : "false";

  fkdl_defaultAllocator(&allocator);
  if (!node || !key) {
    return FKDL_FALSE;
  }

  const char *valueStr = value ? "true" : "false";
  struct fkdl_property property;
  if (!buildProperty(&property, allocator, FKDL_BOOL, key, valueStr)) {
    return FKDL_FALSE;
  }


@@ 2209,14 2206,13 @@ fkdl_bool fkdl_nodeSetNullProperty(
  const char *key,
  struct fkdl_allocator *allocator)
{
  if (!allocator) {
    allocator = &fkdl_defaultAllocator;
  }
  struct fkdl_property property;

  fkdl_defaultAllocator(&allocator);
  if (!node || !key) {
    return FKDL_FALSE;
  }

  struct fkdl_property property;
  if (!buildProperty(&property, allocator, FKDL_NULL, key, NULL)) {
    return FKDL_FALSE;
  }


@@ 2235,14 2231,13 @@ fkdl_bool fkdl_nodeSetNumberProperty(
  const char *value,
  struct fkdl_allocator *allocator)
{
  if (!allocator) {
    allocator = &fkdl_defaultAllocator;
  }
  struct fkdl_property property;

  fkdl_defaultAllocator(&allocator);
  if (!node || !key || !value) {
    return FKDL_FALSE;
  }

  struct fkdl_property property;
  if (!buildProperty(&property, allocator, FKDL_NUMBER, key, value)) {
    return FKDL_FALSE;
  }


@@ 2260,9 2255,7 @@ fkdl_bool fkdl_nodeAddChild(
  const struct fkdl_node *child,
  struct fkdl_allocator *allocator)
{
  if (!allocator) {
    allocator = &fkdl_defaultAllocator;
  }
  fkdl_defaultAllocator(&allocator);
  if (!node || !child || !allocator->malloc || !allocator->realloc) {
    return FKDL_FALSE;
  }


@@ 2297,9 2290,7 @@ fkdl_bool fkdl_documentAddNode(
  const struct fkdl_node *node,
  struct fkdl_allocator *allocator)
{
  if (!allocator) {
    allocator = &fkdl_defaultAllocator;
  }
  fkdl_defaultAllocator(&allocator);
  if (!document || !node || !allocator->malloc || !allocator->realloc) {
    return FKDL_FALSE;
  }


@@ 2378,6 2369,7 @@ static fkdl_bool processTokens(
  struct fkdl_allocator *allocator,
  struct fkdl_error *error)
{
  size_t i;
  fkdl_bool failed = FKDL_FALSE;

  /* The current node we're building, i.e. we're going through its properties */


@@ 2388,7 2380,7 @@ static fkdl_bool processTokens(
  struct fkdl_node *parentNode = NULL;
  /* Stack of parental nodes excluding the current parentNode. For tracking */
  /* recursion through children. */
  struct fkdl_array parents = { .itemSize = sizeof(struct fkdl_node*) };
  struct fkdl_array parents = {0};

  /* slashdash support */
  fkdl_bool skipNextNode = FKDL_FALSE;


@@ 2399,7 2391,8 @@ static fkdl_bool processTokens(
  /* increment or decrement skipDepth respectively. */
  size_t skipDepth = 0;

  size_t i;
  parents.itemSize = sizeof(struct fkdl_node*);

  for (i = 0; i < tokensLen; ++i) {

    /* If skipDepth is >0, we need to fast forward through everything except braces */


@@ 2561,6 2554,7 @@ static fkdl_bool processTokens(
    if (haveCurNode && 
         (tokens[i].type == FKDL_TOK_TYPE_ANNOTATION ||
          tokenTypeIsValue(tokens[i].type))) {
      fkdl_bool propertyHasType, argumentHasType;
      /* Already in a node, so this is a property or argument */
      fkdl_bool isProperty = i + 2 < tokensLen
        && tokens[i+1].type == FKDL_TOK_PROPERTY_EQUALS;


@@ 2571,10 2565,10 @@ static fkdl_bool processTokens(
        break;
      }

      fkdl_bool propertyHasType = i + 3 < tokensLen
      propertyHasType = i + 3 < tokensLen
        && tokens[i+2].type == FKDL_TOK_TYPE_ANNOTATION;

      fkdl_bool argumentHasType = !isProperty && i + 1 < tokensLen
      argumentHasType = !isProperty && i + 1 < tokensLen
        && tokens[i].type == FKDL_TOK_TYPE_ANNOTATION
        && tokenTypeIsValue(tokens[i+1].type);



@@ 2591,17 2585,19 @@ static fkdl_bool processTokens(
      }

      if (isProperty) {
        struct fkdl_string key = copyIdentifierOrStringToString(
        size_t valueIndex;
        size_t typeIndex;
        struct fkdl_property property = {0};

        property.key = copyIdentifierOrStringToString(
          allocator,
          input,
          &tokens[i],
          error);
        if (!key.data) {
        if (!property.key.data) {
          failed = FKDL_TRUE;
          break;
        }
        size_t valueIndex;
        size_t typeIndex;
        if (propertyHasType) {
          typeIndex = i + 2;
          valueIndex = i + 3;


@@ 2611,26 2607,22 @@ static fkdl_bool processTokens(
        }
        if (!tokenTypeIsValue(tokens[valueIndex].type)) {
          errorFromToken(error, &tokens[valueIndex], "Expected value after '='");
          fkdl_cleanupString(&key, allocator);
          fkdl_cleanupString(&property.key, allocator);
          failed = FKDL_TRUE;
          break;
        }
        struct fkdl_value value = copyTokenToValue(
        property.value = copyTokenToValue(
          allocator,
          input,
          propertyHasType ? &tokens[typeIndex] : NULL,
          &tokens[valueIndex],
          error);
        if (value.type == FKDL_INVALID_VALUE) {
          fkdl_cleanupString(&key, allocator);
        if (property.value.type == FKDL_INVALID_VALUE) {
          fkdl_cleanupString(&property.key, allocator);
          failed = FKDL_TRUE;
          break;
        }

        struct fkdl_property property = {
          .key = key,
          .value = value,
        };
        if (!fkdl_nodeAddProperty(&curNode, &property, allocator)) {
          errorFromToken(error, &tokens[i], "Out of memory");
          fkdl_cleanupProperty(&property, allocator);


@@ 2645,13 2637,14 @@ static fkdl_bool processTokens(
        /* Just a plain argument */
        size_t typeIndex;
        size_t valueIndex;
        struct fkdl_value value;
        if (argumentHasType) {
          typeIndex = i;
          valueIndex = i + 1;
        } else {
          valueIndex = i;
        }
        struct fkdl_value value = copyTokenToValue(
        value = copyTokenToValue(
          allocator,
          input,
          argumentHasType ? &tokens[typeIndex] : NULL,


@@ 2728,17 2721,16 @@ fkdl_bool fkdl_readDocument(
  struct fkdl_error *error
)
{
  if (!allocator) {
    allocator = &fkdl_defaultAllocator;
  }
  struct fkdl_array tokens = {0};
  struct fkdl_document document = {0};

  fkdl_defaultAllocator(&allocator);

  if (inputLen == 0) {
    inputLen = fkdl_strlen(input);
  }

  struct fkdl_array tokens = {
    .itemSize = sizeof(struct fkdl_token),
  };
  tokens.itemSize = sizeof(struct fkdl_token);

  if (!tokenise(
        input,


@@ 2749,7 2741,6 @@ fkdl_bool fkdl_readDocument(
    return FKDL_FALSE;
  }

  struct fkdl_document document = {0};
  if (!processTokens(input, (void*)tokens.data, tokens.len, &document, allocator, error)) {
    return FKDL_FALSE;
  }


@@ 2781,13 2772,13 @@ static fkdl_bool isEscaped(uint32_t codePoint)
/* True if the string would need quotes to be used as an identifier */
static fkdl_bool identifierNeedsQuotes(const struct fkdl_string *str)
{
  size_t i;
  if (!str) {
    return FKDL_FALSE;
  }
  if (str->len == 0) {
    return FKDL_TRUE;
  }
  size_t i;
  for (i = 0; i < str->len;) {
    uint32_t codePoint;
    size_t step;


@@ 2971,18 2962,17 @@ fkdl_bool fkdl_writeDocument(
  size_t *bytesWritten,
  const struct fkdl_document *document)
{
  struct fkdl_writeCursor cursor = {0};
  size_t i;

  /* NULL output doesn't write anything, just determine space required */
  if (!document) {
    return FKDL_FALSE;
  }

  struct fkdl_writeCursor cursor = {
    .output = output,
    .outputLen = outputLen,
    .index = 0,
  };

  size_t i;
  cursor.output = output;
  cursor.outputLen = outputLen;
  cursor.index = 0;
  for (i = 0; i < document->nodesLen; ++i) {
    if (!writeNode(&cursor, 0, &document->nodes[i])) {
      return FKDL_FALSE;

M freekdl.h => freekdl.h +1 -1
@@ 35,7 35,7 @@ enum fkdl_value_type
  FKDL_NULL,
  FKDL_STRING,
  FKDL_NUMBER,
  FKDL_BOOL,
  FKDL_BOOL
};

/** A UTF-8 string. Intended as a means for passing strings, not a working area

M test.c => test.c +7 -2
@@ 525,8 525,10 @@ int testTokeniseSuccess(
    .itemSize = sizeof(struct fkdl_token),
  };
  struct fkdl_error error;
  struct fkdl_allocator *alloc = NULL;
  fkdl_defaultAllocator(&alloc);

  if (!tokenise(input, inputLen, &tokens, &error, &fkdl_defaultAllocator)) {
  if (!tokenise(input, inputLen, &tokens, &error, alloc)) {
    printf("fkdl_tokenise failed for: \"%s\"\n", input);
    printf("error: %s index=%zd line=%zd column=%zd\n", error.message, error.index, error.line, error.column);
    return 0;


@@ 758,10 760,13 @@ int testBasicTokenise(void)

int testEscapedUnicode(void)
{
  struct fkdl_allocator *alloc = NULL;
  fkdl_defaultAllocator(&alloc);

  const char input[] = "🙈 emoji 😁";
  const char expected[] = "🙈 emoji 😁";

  struct fkdl_string str = copyString(&fkdl_defaultAllocator, input, sizeof input);
  struct fkdl_string str = copyString(alloc, input, sizeof input);
  if (!str.data) {
    printf("copyString failed for unicode\n");
    return 0;