From 5749d43fb6b575f47a73b03bc8cacc86d39c8a8f Mon Sep 17 00:00:00 2001 From: Mark Thom Date: Wed, 3 Feb 2021 11:19:33 -0700 Subject: [PATCH] restore prolog_parser test files (#782) --- crates/prolog_parser/Cargo.lock | 265 +++++++++++++++++++++ crates/prolog_parser/tests/bom.rs | 43 ++++ crates/prolog_parser/tests/parse_tokens.rs | 114 +++++++++ 3 files changed, 422 insertions(+) create mode 100644 crates/prolog_parser/Cargo.lock create mode 100644 crates/prolog_parser/tests/bom.rs create mode 100644 crates/prolog_parser/tests/parse_tokens.rs diff --git a/crates/prolog_parser/Cargo.lock b/crates/prolog_parser/Cargo.lock new file mode 100644 index 00000000..f90f0d69 --- /dev/null +++ b/crates/prolog_parser/Cargo.lock @@ -0,0 +1,265 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "arrayvec" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd9fd44efafa8690358b7408d253adf110036b88f55672a933f01d616ad9b1b9" +dependencies = [ + "nodrop", +] + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "az" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d84e1d907bfc5795a6addb95ef8666141ee73c8f2f5250ff2a46bf4e4f4aec8a" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "gmp-mpfr-sys" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a57fdb339d49833021b1fded600ed240ae907e33909d5511a61dff884df7f16e" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "lexical" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e0d09e60c187a6d0a3fa418aec8587c6a4ae9de872f6126f2134f319b5ed10d" +dependencies = [ + "cfg-if", + "lexical-core", + "rustc_version", +] + +[[package]] +name = "lexical-core" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304bccb228c4b020f3a4835d247df0a02a7c4686098d4167762cfbbe4c5cb14" +dependencies = [ + "arrayvec", + "cfg-if", + "rustc_version", + "ryu", + "static_assertions", +] + +[[package]] +name = "libc" +version = "0.2.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ccac4b00700875e6a07c6cde370d44d32fa01c5a65cdd2fca6858c479d28bb3" + +[[package]] +name = "nodrop" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" + +[[package]] +name = "num-bigint" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" +dependencies = [ + "autocfg", + "num-integer", + "num-traits 0.2.14", +] + +[[package]] +name = "num-integer" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +dependencies = [ + "autocfg", + "num-traits 0.2.14", +] + +[[package]] +name = "num-rational" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef" +dependencies = [ + "autocfg", + "num-bigint", + "num-integer", + "num-traits 0.2.14", +] + +[[package]] +name = "num-rug-adapter" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7470b6acf85abce0771203112db4181d03f7b8a6be49f0e842a78030192f8a58" +dependencies = [ + "libc", + "num-bigint", + "num-integer", + "num-rational", + "num-traits 0.2.14", +] + +[[package]] +name = "num-traits" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" +dependencies = [ + "num-traits 0.2.14", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg", +] + +[[package]] +name = "ordered-float" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eb5259643245d3f292c7a146b2df53bba24d7eab159410e648eb73dc164669d" +dependencies = [ + "num-traits 0.1.43", + "unreachable", +] + +[[package]] +name = "prolog_parser_rebis" +version = "0.8.68" +dependencies = [ + "lexical", + "num-rug-adapter", + "ordered-float", + "rug", + "unicode_reader", +] + +[[package]] +name = "rug" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e538d00da450a8e48aac7e6322e67b2dc86ec71a1feeac0e3954c4f07f01bc45" +dependencies = [ + "az", + "gmp-mpfr-sys", + "libc", +] + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "smallvec" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" + +[[package]] +name = "static_assertions" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f3eb36b47e512f8f1c9e3d10c2c1965bc992bd9cdb024fa581e2194501c83d3" + +[[package]] +name = "unicode-segmentation" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796" + +[[package]] +name = "unicode_reader" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b639121690b27acd92c97ed2b52c5e5e8d3d39482e943b4559695cef62f771a" +dependencies = [ + "smallvec", + "unicode-segmentation", +] + +[[package]] +name = "unreachable" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" +dependencies = [ + "void", +] + +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/crates/prolog_parser/tests/bom.rs b/crates/prolog_parser/tests/bom.rs new file mode 100644 index 00000000..25b57104 --- /dev/null +++ b/crates/prolog_parser/tests/bom.rs @@ -0,0 +1,43 @@ +extern crate prolog_parser_rebis; + +use prolog_parser_rebis::ast::*; +use prolog_parser_rebis::lexer::{Lexer, Token}; +use prolog_parser_rebis::tabled_rc::TabledData; + +use std::rc::Rc; + +#[test] +fn valid_token() { + let stream = parsing_stream("valid text".as_bytes()); + assert!(stream.is_ok()); +} + +#[test] +fn empty_stream() { + let bytes: &[u8] = &[]; + assert!(parsing_stream(bytes).is_ok()); +} + +#[test] +fn skip_utf8_bom() { + let atom_tbl = TabledData::new(Rc::new("my_module".to_string())); + let flags = MachineFlags::default(); + let bytes: &[u8] = &[0xEF, 0xBB, 0xBF, '4' as u8, '\n' as u8]; + let mut stream = parsing_stream(bytes).expect("valid stream"); + let mut lexer = Lexer::new(atom_tbl, flags, &mut stream); + match lexer.next_token() { + Ok(Token::Constant(Constant::Fixnum(4))) => (), + _ => assert!(false) + } +} + +#[test] +fn invalid_utf16_bom() { + let bytes: &[u8] = &[0xFF, 0xFE, 'a' as u8, '\n' as u8]; + let stream = parsing_stream(bytes); + match stream { + Err(ParserError::Utf8Error(0, 0)) => (), + _ => assert!(false) + } +} + diff --git a/crates/prolog_parser/tests/parse_tokens.rs b/crates/prolog_parser/tests/parse_tokens.rs new file mode 100644 index 00000000..3afb9164 --- /dev/null +++ b/crates/prolog_parser/tests/parse_tokens.rs @@ -0,0 +1,114 @@ +extern crate prolog_parser_rebis; + +use prolog_parser_rebis::ast::*; +use prolog_parser_rebis::lexer::{Lexer, Token}; +use prolog_parser_rebis::tabled_rc::TabledData; + +use std::rc::Rc; + +fn read_all_tokens(text: &str) -> Result, ParserError> { + let atom_tbl = TabledData::new(Rc::new("my_module".to_string())); + let flags = MachineFlags::default(); + let mut stream = parsing_stream(text.as_bytes())?; + let mut lexer = Lexer::new(atom_tbl, flags, &mut stream); + + let mut tokens = Vec::new(); + while !lexer.eof()? { + let token = lexer.next_token()?; + tokens.push(token); + } + Ok(tokens) +} + +#[test] +fn empty_multiline_comment() -> Result<(), ParserError> { + let tokens = read_all_tokens("/**/ 4\n")?; + assert_eq!(tokens, [Token::Constant(Constant::Fixnum(4))]); + Ok(()) +} + +#[test] +fn any_char_multiline_comment() -> Result<(), ParserError> { + let tokens = read_all_tokens("/* █╗╚═══╝ © */ 4\n")?; + assert_eq!(tokens, [Token::Constant(Constant::Fixnum(4))]); + Ok(()) +} + +#[test] +fn simple_char() -> Result<(), ParserError> { + let tokens = read_all_tokens("'a'\n")?; + assert_eq!(tokens, [Token::Constant(Constant::Char('a'))]); + Ok(()) +} + +#[test] +fn char_with_meta_seq() -> Result<(), ParserError> { + let tokens = read_all_tokens(r#"'\\' '\'' '\"' '\`' "#)?; // use literal string so \ are escaped + assert_eq!( + tokens, + [ + Token::Constant(Constant::Char('\\')), + Token::Constant(Constant::Char('\'')), + Token::Constant(Constant::Char('"')), + Token::Constant(Constant::Char('`')) + ] + ); + Ok(()) +} + +#[test] +fn char_with_control_seq() -> Result<(), ParserError> { + let tokens = read_all_tokens(r"'\a' '\b' '\r' '\f' '\t' '\n' '\v' ")?; + assert_eq!( + tokens, + [ + Token::Constant(Constant::Char('\u{07}')), + Token::Constant(Constant::Char('\u{08}')), + Token::Constant(Constant::Char('\r')), + Token::Constant(Constant::Char('\u{0c}')), + Token::Constant(Constant::Char('\t')), + Token::Constant(Constant::Char('\n')), + Token::Constant(Constant::Char('\u{0b}')), + ] + ); + Ok(()) +} + +#[test] +fn char_with_octseq() -> Result<(), ParserError> { + let tokens = read_all_tokens(r"'\60433\' ")?; + assert_eq!(tokens, [Token::Constant(Constant::Char('愛'))]); // Japanese character + Ok(()) +} + +#[test] +fn char_with_octseq_0() -> Result<(), ParserError> { + let tokens = read_all_tokens(r"'\0\' ")?; + assert_eq!(tokens, [Token::Constant(Constant::Char('\u{0000}'))]); + Ok(()) +} + +#[test] +fn char_with_hexseq() -> Result<(), ParserError> { + let tokens = read_all_tokens(r"'\x2124\' ")?; + assert_eq!(tokens, [Token::Constant(Constant::Char('ℤ'))]); // Z math symbol + Ok(()) +} + +#[test] +fn char_with_hexseq_invalid() { + assert!(read_all_tokens(r"'\x\' ").is_err()); +} + +#[test] +fn empty() -> Result<(), ParserError> { + let tokens = read_all_tokens("")?; + assert!(tokens.is_empty()); + Ok(()) +} + +#[test] +fn comment_then_eof() -> Result<(), ParserError> { + assert!(read_all_tokens("% only a comment").is_err()); + Ok(()) +} -- 2.54.0