restore prolog_parser test files (#782)

diff --git a/crates/prolog_parser/Cargo.lock b/crates/prolog_parser/Cargo.lock
new file mode 100644 (file)
index 0000000..f90f0d6
--- /dev/null
+++ b/crates/prolog_parser/Cargo.lock
@@ -0,0 +1,265 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+[[package]]
+name = "arrayvec"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd9fd44efafa8690358b7408d253adf110036b88f55672a933f01d616ad9b1b9"
+dependencies = [
+ "nodrop",
+]
+
+[[package]]
+name = "autocfg"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
+
+[[package]]
+name = "az"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d84e1d907bfc5795a6addb95ef8666141ee73c8f2f5250ff2a46bf4e4f4aec8a"
+
+[[package]]
+name = "cfg-if"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
+
+[[package]]
+name = "gmp-mpfr-sys"
+version = "1.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a57fdb339d49833021b1fded600ed240ae907e33909d5511a61dff884df7f16e"
+dependencies = [
+ "libc",
+ "winapi",
+]
+
+[[package]]
+name = "lexical"
+version = "2.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e0d09e60c187a6d0a3fa418aec8587c6a4ae9de872f6126f2134f319b5ed10d"
+dependencies = [
+ "cfg-if",
+ "lexical-core",
+ "rustc_version",
+]
+
+[[package]]
+name = "lexical-core"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304bccb228c4b020f3a4835d247df0a02a7c4686098d4167762cfbbe4c5cb14"
+dependencies = [
+ "arrayvec",
+ "cfg-if",
+ "rustc_version",
+ "ryu",
+ "static_assertions",
+]
+
+[[package]]
+name = "libc"
+version = "0.2.85"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ccac4b00700875e6a07c6cde370d44d32fa01c5a65cdd2fca6858c479d28bb3"
+
+[[package]]
+name = "nodrop"
+version = "0.1.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
+
+[[package]]
+name = "num-bigint"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304"
+dependencies = [
+ "autocfg",
+ "num-integer",
+ "num-traits 0.2.14",
+]
+
+[[package]]
+name = "num-integer"
+version = "0.1.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db"
+dependencies = [
+ "autocfg",
+ "num-traits 0.2.14",
+]
+
+[[package]]
+name = "num-rational"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef"
+dependencies = [
+ "autocfg",
+ "num-bigint",
+ "num-integer",
+ "num-traits 0.2.14",
+]
+
+[[package]]
+name = "num-rug-adapter"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7470b6acf85abce0771203112db4181d03f7b8a6be49f0e842a78030192f8a58"
+dependencies = [
+ "libc",
+ "num-bigint",
+ "num-integer",
+ "num-rational",
+ "num-traits 0.2.14",
+]
+
+[[package]]
+name = "num-traits"
+version = "0.1.43"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31"
+dependencies = [
+ "num-traits 0.2.14",
+]
+
+[[package]]
+name = "num-traits"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "ordered-float"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7eb5259643245d3f292c7a146b2df53bba24d7eab159410e648eb73dc164669d"
+dependencies = [
+ "num-traits 0.1.43",
+ "unreachable",
+]
+
+[[package]]
+name = "prolog_parser_rebis"
+version = "0.8.68"
+dependencies = [
+ "lexical",
+ "num-rug-adapter",
+ "ordered-float",
+ "rug",
+ "unicode_reader",
+]
+
+[[package]]
+name = "rug"
+version = "1.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e538d00da450a8e48aac7e6322e67b2dc86ec71a1feeac0e3954c4f07f01bc45"
+dependencies = [
+ "az",
+ "gmp-mpfr-sys",
+ "libc",
+]
+
+[[package]]
+name = "rustc_version"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
+dependencies = [
+ "semver",
+]
+
+[[package]]
+name = "ryu"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
+
+[[package]]
+name = "semver"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
+dependencies = [
+ "semver-parser",
+]
+
+[[package]]
+name = "semver-parser"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
+
+[[package]]
+name = "smallvec"
+version = "1.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"
+
+[[package]]
+name = "static_assertions"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f3eb36b47e512f8f1c9e3d10c2c1965bc992bd9cdb024fa581e2194501c83d3"
+
+[[package]]
+name = "unicode-segmentation"
+version = "1.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796"
+
+[[package]]
+name = "unicode_reader"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b639121690b27acd92c97ed2b52c5e5e8d3d39482e943b4559695cef62f771a"
+dependencies = [
+ "smallvec",
+ "unicode-segmentation",
+]
+
+[[package]]
+name = "unreachable"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56"
+dependencies = [
+ "void",
+]
+
+[[package]]
+name = "void"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
+
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

diff --git a/crates/prolog_parser/tests/bom.rs b/crates/prolog_parser/tests/bom.rs
new file mode 100644 (file)
index 0000000..25b5710
--- /dev/null
+++ b/crates/prolog_parser/tests/bom.rs
@@ -0,0 +1,43 @@
+extern crate prolog_parser_rebis;
+
+use prolog_parser_rebis::ast::*;
+use prolog_parser_rebis::lexer::{Lexer, Token};
+use prolog_parser_rebis::tabled_rc::TabledData;
+
+use std::rc::Rc;
+
+#[test]
+fn valid_token() {
+    let stream = parsing_stream("valid text".as_bytes());
+    assert!(stream.is_ok());
+}
+
+#[test]
+fn empty_stream() {
+    let bytes: &[u8] = &[];
+    assert!(parsing_stream(bytes).is_ok());
+}
+
+#[test]
+fn skip_utf8_bom() {
+    let atom_tbl = TabledData::new(Rc::new("my_module".to_string()));
+    let flags = MachineFlags::default();
+    let bytes: &[u8] = &[0xEF, 0xBB, 0xBF, '4' as u8, '\n' as u8];
+    let mut stream = parsing_stream(bytes).expect("valid stream");
+    let mut lexer = Lexer::new(atom_tbl, flags, &mut stream);
+    match lexer.next_token() {
+        Ok(Token::Constant(Constant::Fixnum(4))) => (),
+        _ => assert!(false)
+    }
+}
+
+#[test]
+fn invalid_utf16_bom() {
+    let bytes: &[u8] = &[0xFF, 0xFE, 'a' as u8, '\n' as u8];
+    let stream = parsing_stream(bytes);
+    match stream {
+        Err(ParserError::Utf8Error(0, 0)) => (),
+        _ => assert!(false)
+    }
+}
+

diff --git a/crates/prolog_parser/tests/parse_tokens.rs b/crates/prolog_parser/tests/parse_tokens.rs
new file mode 100644 (file)
index 0000000..3afb916
--- /dev/null
+++ b/crates/prolog_parser/tests/parse_tokens.rs
@@ -0,0 +1,114 @@
+extern crate prolog_parser_rebis;
+
+use prolog_parser_rebis::ast::*;
+use prolog_parser_rebis::lexer::{Lexer, Token};
+use prolog_parser_rebis::tabled_rc::TabledData;
+
+use std::rc::Rc;
+
+fn read_all_tokens(text: &str) -> Result<Vec<Token>, ParserError> {
+    let atom_tbl = TabledData::new(Rc::new("my_module".to_string()));
+    let flags = MachineFlags::default();
+    let mut stream = parsing_stream(text.as_bytes())?;
+    let mut lexer = Lexer::new(atom_tbl, flags, &mut stream);
+
+    let mut tokens = Vec::new();
+    while !lexer.eof()? {
+        let token = lexer.next_token()?;
+        tokens.push(token);
+    }
+    Ok(tokens)
+}
+
+#[test]
+fn empty_multiline_comment() -> Result<(), ParserError> {
+    let tokens = read_all_tokens("/**/ 4\n")?;
+    assert_eq!(tokens, [Token::Constant(Constant::Fixnum(4))]);
+    Ok(())
+}
+
+#[test]
+fn any_char_multiline_comment() -> Result<(), ParserError> {
+    let tokens = read_all_tokens("/* █╗╚═══╝ © */ 4\n")?;
+    assert_eq!(tokens, [Token::Constant(Constant::Fixnum(4))]);
+    Ok(())
+}
+
+#[test]
+fn simple_char() -> Result<(), ParserError> {
+    let tokens = read_all_tokens("'a'\n")?;
+    assert_eq!(tokens, [Token::Constant(Constant::Char('a'))]);
+    Ok(())
+}
+
+#[test]
+fn char_with_meta_seq() -> Result<(), ParserError> {
+    let tokens = read_all_tokens(r#"'\\' '\'' '\"' '\`' "#)?; // use literal string so \ are escaped
+    assert_eq!(
+        tokens,
+        [
+            Token::Constant(Constant::Char('\\')),
+            Token::Constant(Constant::Char('\'')),
+            Token::Constant(Constant::Char('"')),
+            Token::Constant(Constant::Char('`'))
+        ]
+    );
+    Ok(())
+}
+
+#[test]
+fn char_with_control_seq() -> Result<(), ParserError> {
+    let tokens = read_all_tokens(r"'\a' '\b' '\r' '\f' '\t' '\n' '\v' ")?;
+    assert_eq!(
+        tokens,
+        [
+            Token::Constant(Constant::Char('\u{07}')),
+            Token::Constant(Constant::Char('\u{08}')),
+            Token::Constant(Constant::Char('\r')),
+            Token::Constant(Constant::Char('\u{0c}')),
+            Token::Constant(Constant::Char('\t')),
+            Token::Constant(Constant::Char('\n')),
+            Token::Constant(Constant::Char('\u{0b}')),
+        ]
+    );
+    Ok(())
+}
+
+#[test]
+fn char_with_octseq() -> Result<(), ParserError> {
+    let tokens = read_all_tokens(r"'\60433\' ")?;
+    assert_eq!(tokens, [Token::Constant(Constant::Char('愛'))]); // Japanese character
+    Ok(())
+}
+
+#[test]
+fn char_with_octseq_0() -> Result<(), ParserError> {
+    let tokens = read_all_tokens(r"'\0\' ")?;
+    assert_eq!(tokens, [Token::Constant(Constant::Char('\u{0000}'))]);
+    Ok(())
+}
+
+#[test]
+fn char_with_hexseq() -> Result<(), ParserError> {
+    let tokens = read_all_tokens(r"'\x2124\' ")?;
+    assert_eq!(tokens, [Token::Constant(Constant::Char('ℤ'))]); // Z math symbol
+    Ok(())
+}
+
+#[test]
+fn char_with_hexseq_invalid() {
+    assert!(read_all_tokens(r"'\x\' ").is_err());
+}
+
+#[test]
+fn empty() -> Result<(), ParserError> {
+    let tokens = read_all_tokens("")?;
+    assert!(tokens.is_empty());
+    Ok(())
+}
+
+#[test]
+fn comment_then_eof() -> Result<(), ParserError> {
+    assert!(read_all_tokens("% only a comment").is_err());
+    Ok(())
+}