Merge parserstate.c and .h files into respective parser files + some cleanups (#26)

Part of Shopify/team-ruby-dx#1436

In addition to merging the files, I also did some cleanups like:
- Remove unused function declaration
- Make some helper functions that aren't referenced outside of C library
AND doesn't look like public API as static
- A leftover in #25 

I recommend reviewing by commits.

Author: st0012

include/rbs/encoding.h

@@ -4,6 +4,6 @@
 #include "rbs_string.h"
 
 unsigned int rbs_utf8_to_codepoint(const rbs_string_t string);
-int utf8_codelen(unsigned int c);
+int rbs_utf8_codelen(unsigned int c);
 
 #endif // RBS_ENCODING_H

include/rbs/parser.h

@@ -1,14 +1,144 @@
 #ifndef RBS__PARSER_H
 #define RBS__PARSER_H
 
-#include "parserstate.h"
+#include "rbs/defines.h"
+#include "rbs/util/rbs_allocator.h"
+#include "rbs/util/rbs_constant_pool.h"
+#include "rbs/lexer.h"
+#include "rbs/ast.h"
+
+#include <stdbool.h>
+#include <stddef.h>
+
+/**
+ * id_table represents a set of RBS constant IDs.
+ * This is used to manage the set of bound variables.
+ * */
+typedef struct id_table {
+  size_t size;
+  size_t count;
+  rbs_constant_id_t *ids;
+  struct id_table *next;
+} id_table;
+
+/**
+ * comment represents a sequence of comment lines.
+ *
+ *     # Comment for the method.
+ *     #
+ *     # ```rb
+ *     # object.foo()  # Do something
+ *     # ```
+ *     #
+ *     def foo: () -> void
+ *
+ * A comment object represents the six lines of comments.
+ * */
+typedef struct comment {
+  position start;
+  position end;
+
+  size_t line_size;
+  size_t line_count;
+  token *tokens;
+
+  struct comment *next_comment;
+} comment;
+
+typedef struct error {
+  char *message;
+  token token;
+  bool syntax_error;
+} error;
+
+/**
+ * An RBS parser is a LL(3) parser.
+ * */
+typedef struct {
+  lexstate *lexstate;
+
+  token current_token;
+  token next_token;       /* The first lookahead token */
+  token next_token2;      /* The second lookahead token */
+  token next_token3;      /* The third lookahead token */
+
+  id_table *vars;         /* Known type variables */
+  comment *last_comment;  /* Last read comment */
+
+  rbs_constant_pool_t constant_pool;
+  rbs_allocator_t allocator;
+  error *error;
+} parserstate;
+
+/**
+ * Insert new table entry.
+ * Setting `reset` inserts a _reset_ entry, which stops searching.
+ *
+ * ```
+ * class Foo[A]
+ *          ^^^                      <= push new table with reset
+ *   def foo: [B] () -> [A, B]
+ *            ^^^                    <= push new table without reset
+ *
+ *   class Baz[C]
+ *            ^^^                    <= push new table with reset
+ *   end
+ * end
+ * ```
+ * */
+id_table *parser_push_typevar_table(parserstate *state, bool reset);
+
+/**
+ * Insert new type variable into the latest table.
+ * */
+NODISCARD bool parser_insert_typevar(parserstate *state, rbs_constant_id_t id);
+
+/**
+ * Allocate new lexstate object.
+ *
+ * ```
+ * VALUE string = rb_funcall(buffer, rb_intern("content"), 0);
+ * alloc_lexer(string, 0, 31)    // New lexstate with buffer content
+ * ```
+ * */
+lexstate *alloc_lexer(rbs_allocator_t *, rbs_string_t string, const rbs_encoding_t *encoding, int start_pos, int end_pos);
+
+/**
+ * Allocate new parserstate object.
+ *
+ * ```
+ * alloc_parser(buffer, string, encoding, 0, 1);
+ * ```
+ * */
+parserstate *alloc_parser(rbs_string_t string, const rbs_encoding_t *encoding, int start_pos, int end_pos);
+void free_parser(parserstate *parser);
+
+/**
+ * Advance one token.
+ * */
+void parser_advance(parserstate *state);
+
+void print_parser(parserstate *state);
+
+/**
+ * Returns a RBS::Comment object associated with an subject at `subject_line`.
+ *
+ * ```rbs
+ * # Comment1
+ * class Foo           # This is the subject line for Comment1
+ *
+ *   # Comment2
+ *   %a{annotation}    # This is the subject line for Comment2
+ *   def foo: () -> void
+ * end
+ * ```
+ * */
+rbs_ast_comment_t *get_comment(parserstate *state, int subject_line);
 
 void set_error(parserstate *state, token tok, bool syntax_error, const char *fmt, ...) RBS_ATTRIBUTE_FORMAT(4, 5);
 
 bool parse_type(parserstate *state, rbs_node_t **type);
 bool parse_method_type(parserstate *state, rbs_methodtype_t **method_type);
 bool parse_signature(parserstate *state, rbs_signature_t **signature);
 
-void rbs__init_parser();
-
 #endif

include/rbs/parserstate.h

@@ -48,7 +48,7 @@ unsigned int rbs_utf8_to_codepoint(const rbs_string_t string) {
     return codepoint;
 }
 
-int utf8_codelen(unsigned int c) {
+int rbs_utf8_codelen(unsigned int c) {
     if (c <= 0x7F) return 1;
     if (c <= 0x7FF) return 2;
     if (c <= 0xFFFF) return 3;