Add some basic lexer test cases

lexer/array_summation.mj

+public class Summator {
+
+    public int sum(int[] array, int len) {
+        int sum = 0;
+        int i = 0;
+        while(i < len) {
+            sum += array[i];
+            i += 1;
+        }
+        return sum;
+    }
+
+    public static void main(String[] args) {
+
+        acc = new Summator();
+
+        int arrayLen = System.in.read();
+        int[] array = new int[arrayLen];
+
+        int i = 0;
+        while(i < arrayLen) {
+            array[i] = System.in.read();
+            i += 1;
+        }
+
+        int arraySum = acc.sum(array, arrayLen);
+        System.out.write(arraySum);
+        System.out.flush();
+    }
+}

lexer/array_summation.mj.out

+public
+class
+identifier Summator
+{
+public
+int
+identifier sum
+(
+int
+[
+]
+identifier array
+,
+int
+identifier len
+)
+{
+int
+identifier sum
+=
+integer literal 0
+;
+int
+identifier i
+=
+integer literal 0
+;
+while
+(
+identifier i
+<
+identifier len
+)
+{
+identifier sum
++=
+identifier array
+[
+identifier i
+]
+;
+identifier i
++=
+integer literal 1
+;
+}
+return
+identifier sum
+;
+}
+public
+static
+void
+identifier main
+(
+identifier String
+[
+]
+identifier args
+)
+{
+identifier acc
+=
+new
+identifier Summator
+(
+)
+;
+int
+identifier arrayLen
+=
+identifier System
+.
+identifier in
+.
+identifier read
+(
+)
+;
+int
+[
+]
+identifier array
+=
+new
+int
+[
+identifier arrayLen
+]
+;
+int
+identifier i
+=
+integer literal 0
+;
+while
+(
+identifier i
+<
+identifier arrayLen
+)
+{
+identifier array
+[
+identifier i
+]
+=
+identifier System
+.
+identifier in
+.
+identifier read
+(
+)
+;
+identifier i
++=
+integer literal 1
+;
+}
+int
+identifier arraySum
+=
+identifier acc
+.
+identifier sum
+(
+identifier array
+,
+identifier arrayLen
+)
+;
+identifier System
+.
+identifier out
+.
+identifier write
+(
+identifier arraySum
+)
+;
+identifier System
+.
+identifier out
+.
+identifier flush
+(
+)
+;
+}
+}
+EOF

lexer/commented_out_program.mj

+/*
+class Test {
+    public static void main(String[] args) {
+        return;
+    }
+
+    public boolean test() {
+        return true;
+    }
+}
+*/

lexer/commented_out_program.mj.out

+EOF

lexer/illegal_character.invalid.mj

+public static void illegalIdentifier$() {}

lexer/illegal_string_quotes.invalid.mj

+System.out.println("Hello world!");

lexer/invalid_leading_zero.invalid.mj

+0123

lexer/invalid_numerical_token.invalid.mj

+123a

lexer/keyword_collection.mj

+abstract assert boolean break byte case catch char class const continue default double do else enum extends false finally final float for goto if implements import instanceof interface int long native new null package private protected public return short static strictfp super switch synchronized this throws throw transient true try void volatile while

lexer/keyword_collection.mj.out

+abstract
+assert
+boolean
+break
+byte
+case
+catch
+char
+class
+const
+continue
+default
+double
+do
+else
+enum
+extends
+false
+finally
+final
+float
+for
+goto
+if
+implements
+import
+instanceof
+interface
+int
+long
+native
+new
+null
+package
+private
+protected
+public
+return
+short
+static
+strictfp
+super
+switch
+synchronized
+this
+throws
+throw
+transient
+true
+try
+void
+volatile
+while
+EOF

lexer/minimal_program.mj

+class Prog1 { public static void main(String[] args) {} }

lexer/minimal_program.mj.out

+class
+identifier Prog1
+{
+public
+static
+void
+identifier main
+(
+identifier String
+[
+]
+identifier args
+)
+{
+}
+}
+EOF

lexer/nested_comment.mj

+/* Comment (outer) /* Comment (still outer, nested comments not supported) */
+public void method() {}
+/* Code above should be included in lexed input */

lexer/nested_comment.mj.out

+public
+void
+identifier method
+(
+)
+{
+}
+EOF

lexer/operator_collection.mj

+= == != < <= > >= + += - -= * *= / /= % %= ++ -- ! || && ? : ~ & &= | |= ^ ^= << <<= >> >>= >>> >>>= ( ) { } [ ] ; , .

lexer/operator_collection.mj.out

+=
+==
+!=
+<
+<=
+>
+>=
++
++=
+-
+-=
+*
+*=
+/
+/=
+%
+%=
+++
+--
+!
+||
+&&
+?
+:
+~
+&
+&=
+|
+|=
+^
+^=
+<<
+<<=
+>>
+>>=
+>>>
+>>>=
+(
+)
+{
+}
+[
+]
+;
+,
+.
+EOF

lexer/operator_comment.mj

+/=/* A comment with operators at either side */=

lexer/operator_comment.mj.out

+/=
+=
+EOF

lexer/parens_matched.mj

+()(())(()())
+[{[[][]]}]
+(())
+

lexer/parens_matched.mj.out

+(
+)
+(
+(
+)
+)
+(
+(
+)
+(
+)
+)
+[
+{
+[
+[
+]
+[
+]
+]
+}
+]
+(
+(
+)
+)
+EOF

lexer/parens_unmatched.mj

+(()
+}(}
+[(])

lexer/parens_unmatched.mj.out

+(
+(
+)
+}
+(
+}
+[
+(
+]
+)
+EOF

lexer/whitespace_only.mj

+     
+    
+    
+         

lexer/whitespace_only.mj.out

+EOF