From ee01900dcffc46d617572927553a89b6c181885e Mon Sep 17 00:00:00 2001 From: delta Date: Tue, 22 Nov 2022 00:24:23 +0100 Subject: [PATCH] Update --- Cargo.lock | 900 +++++++++++++++++++++++++++++++++++++ Cargo.toml | 9 + src/highlighter.rs | 1 + src/lib.rs | 2 + src/syntax/loader.rs | 2 + src/syntax/mod.rs | 2 + src/syntax/types.rs | 29 ++ src/tokenizer.rs | 93 ---- src/tokenizer/mod.rs | 3 + src/tokenizer/state.rs | 39 ++ src/tokenizer/tokenizer.rs | 101 +++++ src/tokenizer/types.rs | 41 ++ 12 files changed, 1129 insertions(+), 93 deletions(-) create mode 100644 src/highlighter.rs create mode 100644 src/syntax/loader.rs create mode 100644 src/syntax/mod.rs create mode 100644 src/syntax/types.rs delete mode 100644 src/tokenizer.rs create mode 100644 src/tokenizer/mod.rs create mode 100644 src/tokenizer/state.rs create mode 100644 src/tokenizer/tokenizer.rs create mode 100644 src/tokenizer/types.rs diff --git a/Cargo.lock b/Cargo.lock index 5f45e1c..7955504 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,906 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "aho-corasick" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e" +dependencies = [ + "memchr", +] + +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + +[[package]] +name = "arrayref" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" + +[[package]] +name = "arrayvec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" + +[[package]] +name = "ascii-canvas" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff8eb72df928aafb99fe5d37b383f2fe25bd2a765e3e5f7c365916b6f2463a29" +dependencies = [ + "term", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dde43e75fd43e8a1bf86103336bc699aa8d17ad1be60c76c0bdfd4828e19b78" +dependencies = [ + "autocfg 1.1.0", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "blake2b_simd" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afa748e348ad3be8263be728124b24a24f268266f6f5d58af9d75f6a40b5c587" +dependencies = [ + "arrayref", + "arrayvec", + "constant_time_eq", +] + +[[package]] +name = "block-buffer" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" +dependencies = [ + "block-padding", + "byte-tools", + "byteorder", + "generic-array", +] + +[[package]] +name = "block-padding" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" +dependencies = [ + "byte-tools", +] + +[[package]] +name = "bstr" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" +dependencies = [ + "memchr", +] + +[[package]] +name = "byte-tools" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "cc" +version = "1.0.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f73505338f7d905b19d18738976aae232eb46b8efc15554ffc56deb5d9ebe4" +dependencies = [ + "jobserver", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags", +] + +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + +[[package]] +name = "crossbeam-utils" +version = "0.8.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "422f23e724af1240ec469ea1e834d87a4b59ce2efe2c6a96256b0c47e2fd86aa" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + +[[package]] +name = "digest" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" +dependencies = [ + "generic-array", +] + +[[package]] +name = "dirs" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "docopt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f3f119846c823f9eafcf953a8f6ffb6ed69bf6240883261a7f13b634579a51f" +dependencies = [ + "lazy_static", + "regex", + "serde", + "strsim", +] + +[[package]] +name = "either" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" + +[[package]] +name = "ena" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8944dc8fa28ce4a38f778bd46bf7d923fe73eed5a439398507246c8e017e6f36" +dependencies = [ + "log", +] + +[[package]] +name = "enum-as-inner" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9720bba047d567ffc8a3cba48bf19126600e249ab7f128e9233e6376976a116" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "fake-simd" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" + +[[package]] +name = "fixedbitset" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86d4de0081402f5e88cdac65c8dcdcc73118c1a7a465e2a05f0da05843a8ea33" + +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + +[[package]] +name = "generic-array" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd" +dependencies = [ + "typenum", +] + +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "heck" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "itertools" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484" +dependencies = [ + "either", +] + +[[package]] +name = "jobserver" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b" +dependencies = [ + "libc", +] + +[[package]] +name = "lalrpop" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64dc3698e75d452867d9bd86f4a723f452ce9d01fe1d55990b79f0c790aa67db" +dependencies = [ + "ascii-canvas", + "atty", + "bit-set", + "diff", + "docopt", + "ena", + "itertools", + "lalrpop-util", + "petgraph", + "regex", + "regex-syntax", + "serde", + "serde_derive", + "sha2", + "string_cache", + "term", + "unicode-xid", +] + +[[package]] +name = "lalrpop-util" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c277d18683b36349ab5cd030158b54856fca6bb2d5dc5263b06288f486958b7c" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.137" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89" + [[package]] name = "litelighter" version = "0.1.0" +dependencies = [ + "enum-as-inner", + "lua-patterns", + "mlua", + "pcre2", + "typed-html-macros", +] + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "lua-patterns" +version = "0.3.0" +source = "git+https://git.karx.xyz/delta/lua-patterns#82b68b283f577efec9a68a9b575041655c4a4bc7" + +[[package]] +name = "lua-src" +version = "544.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "708ba3c844d5e9d38def4a09dd871c17c370f519b3c4b7261fbabe4a613a814c" +dependencies = [ + "cc", +] + +[[package]] +name = "luajit-src" +version = "210.4.3+resty8384278" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ee5d5afddf1ec76ffa55ca7c3001f2f8a703834beba53c56a38ea6641cef44" +dependencies = [ + "cc", +] + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "mlua" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4351dbcc863fb6249c81b3bd0c8001214e9d4d44d22cabda17026353a77fe612" +dependencies = [ + "bstr", + "cc", + "lua-src", + "luajit-src", + "mlua_derive", + "num-traits", + "once_cell", + "pkg-config", + "rustc-hash", +] + +[[package]] +name = "mlua_derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9214e60d3cf1643013b107330fcd374ccec1e4ba1eef76e7e5da5e8202e71c0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "new_debug_unreachable" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54" + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg 1.1.0", +] + +[[package]] +name = "once_cell" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" + +[[package]] +name = "opaque-debug" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" + +[[package]] +name = "ordermap" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a86ed3f5f244b372d6b1a00b72ef7f8876d0bc6a78a4c9985c53614041512063" + +[[package]] +name = "pcre2" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85b30f2f69903b439dd9dc9e824119b82a55bf113b29af8d70948a03c1b11ab1" +dependencies = [ + "libc", + "log", + "pcre2-sys", + "thread_local", +] + +[[package]] +name = "pcre2-sys" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec30e5e9ec37eb8fbf1dea5989bc957fd3df56fbee5061aa7b7a99dbb37b722" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "petgraph" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3659d1ee90221741f65dd128d9998311b0e40c5d3c23a62445938214abce4f" +dependencies = [ + "fixedbitset", + "ordermap", +] + +[[package]] +name = "phf_generator" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09364cc93c159b8b06b1f4dd8a4398984503483891b0c26b867cf431fb132662" +dependencies = [ + "phf_shared", + "rand", +] + +[[package]] +name = "phf_shared" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234f71a15de2288bcb7e3b6515828d22af7ec8598ee6d24c3b526fa0a80b67a0" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pkg-config" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" + +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + +[[package]] +name = "proc-macro2" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" +dependencies = [ + "autocfg 0.1.8", + "libc", + "rand_chacha", + "rand_core 0.4.2", + "rand_hc", + "rand_isaac", + "rand_jitter", + "rand_os", + "rand_pcg", + "rand_xorshift", + "winapi", +] + +[[package]] +name = "rand_chacha" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +dependencies = [ + "autocfg 0.1.8", + "rand_core 0.3.1", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + +[[package]] +name = "rand_hc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_isaac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_jitter" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" +dependencies = [ + "libc", + "rand_core 0.4.2", + "winapi", +] + +[[package]] +name = "rand_os" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" +dependencies = [ + "cloudabi", + "fuchsia-cprng", + "libc", + "rand_core 0.4.2", + "rdrand", + "winapi", +] + +[[package]] +name = "rand_pcg" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" +dependencies = [ + "autocfg 0.1.8", + "rand_core 0.4.2", +] + +[[package]] +name = "rand_xorshift" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + +[[package]] +name = "redox_users" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d" +dependencies = [ + "getrandom", + "redox_syscall", + "rust-argon2", +] + +[[package]] +name = "regex" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" + +[[package]] +name = "rust-argon2" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b18820d944b33caa75a71378964ac46f58517c92b6ae5f762636247c09e78fb" +dependencies = [ + "base64", + "blake2b_simd", + "constant_time_eq", + "crossbeam-utils", +] + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "serde" +version = "1.0.147" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d193d69bae983fc11a79df82342761dfbf28a99fc8d203dca4c3c1b590948965" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.147" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f1d362ca8fc9c3e3a7484440752472d68a6caa98f1ab81d99b5dfe517cec852" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sha2" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a256f46ea78a0c0d9ff00077504903ac881a1dafdc20da66545699e7776b3e69" +dependencies = [ + "block-buffer", + "digest", + "fake-simd", + "opaque-debug", +] + +[[package]] +name = "siphasher" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" + +[[package]] +name = "string_cache" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89c058a82f9fd69b1becf8c274f412281038877c553182f1d02eb027045a2d67" +dependencies = [ + "lazy_static", + "new_debug_unreachable", + "phf_shared", + "precomputed-hash", + "serde", + "string_cache_codegen", + "string_cache_shared", +] + +[[package]] +name = "string_cache_codegen" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f45ed1b65bf9a4bf2f7b7dc59212d1926e9eaf00fa998988e420fd124467c6" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "string_cache_shared", +] + +[[package]] +name = "string_cache_shared" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1884d1bc09741d466d9b14e6d37ac89d6909cbcac41dd9ae982d4d063bbedfc" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a864042229133ada95abf3b54fdc62ef5ccabe9515b64717bcb9a1919e59445d" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "term" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd106a334b7657c10b7c540a0106114feadeb4dc314513e97df481d5d966f42" +dependencies = [ + "byteorder", + "dirs", + "winapi", +] + +[[package]] +name = "thread_local" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +dependencies = [ + "once_cell", +] + +[[package]] +name = "typed-html-macros" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44a4dba17ed65147f4780560f1078de857f3f4d48f5aeb2197bace8e103a7356" +dependencies = [ + "ansi_term", + "lalrpop", + "lalrpop-util", + "proc-macro-hack", + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "typenum" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" + +[[package]] +name = "unicode-ident" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" + +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/Cargo.toml b/Cargo.toml index e8d51dc..f2b00f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,3 +6,12 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +lua-patterns = { git = "https://git.karx.xyz/delta/lua-patterns" } +pcre2 = "0.2.3" +typed-html-macros = "0.2.2" +enum-as-inner = "0.5.1" +mlua = { version = "0.8.5", features = [ "vendored", "module", "lua54" ] } + + +[lib] +crate-type = ["cdylib"] diff --git a/src/highlighter.rs b/src/highlighter.rs new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/highlighter.rs @@ -0,0 +1 @@ + diff --git a/src/lib.rs b/src/lib.rs index 4b64d0d..44efabb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,7 @@ #![allow(dead_code)] mod tokenizer; +mod highlighter; +mod syntax; fn main() { println!("Hello, world!"); diff --git a/src/syntax/loader.rs b/src/syntax/loader.rs new file mode 100644 index 0000000..139597f --- /dev/null +++ b/src/syntax/loader.rs @@ -0,0 +1,2 @@ + + diff --git a/src/syntax/mod.rs b/src/syntax/mod.rs new file mode 100644 index 0000000..c195316 --- /dev/null +++ b/src/syntax/mod.rs @@ -0,0 +1,2 @@ +pub mod types; +pub mod loader; diff --git a/src/syntax/types.rs b/src/syntax/types.rs new file mode 100644 index 0000000..ba7786d --- /dev/null +++ b/src/syntax/types.rs @@ -0,0 +1,29 @@ +use lua_patterns::LuaPattern; +use pcre2::bytes::Regex; +use enum_as_inner::EnumAsInner; +use crate::tokenizer::types::TokenKind; + +pub type Syntax<'r> = Vec>; + +pub enum RegexOrPattern<'r> { + Regex(Regex), + Pattern(LuaPattern<'r>) +} + +pub enum PatternTypes<'r> { + Type1(RegexOrPattern<'r>), + Type2([RegexOrPattern<'r>; 2]), + Type3([RegexOrPattern<'r>; 3]) +} + +#[derive(EnumAsInner)] +pub enum KindOrSubsyntax<'r> { + Kind(TokenKind<'r>), + Subsyntax(Syntax<'r>) +} + +pub struct Rule<'r> { + pub pattern: PatternTypes<'r>, + pub group: KindOrSubsyntax<'r>, + pub whole_line: bool +} diff --git a/src/tokenizer.rs b/src/tokenizer.rs deleted file mode 100644 index 045c47c..0000000 --- a/src/tokenizer.rs +++ /dev/null @@ -1,93 +0,0 @@ -pub enum TokenizerError { - ExceededDepthRange, -} - -mod state { - use crate::tokenizer::TokenizerError; - - /// State is divided into four 8 byte long ints, each int represents the rule id - /// Should never be modified directly. Instead use the push_id and pop_id methods. - pub struct State { - depth: u8, // Valid range is 0-3 - id_stack: [u8; 4] - } - - impl State { - pub fn new() -> Self { - State { - depth: 0, - id_stack: [0, 0, 0, 0] - } - } - - fn get_state() { - - } - - pub fn push_id(&mut self, id: u8) -> Result<(), TokenizerError> { - if self.depth + 1 > 3 { - return Err(TokenizerError::ExceededDepthRange); - } - - self.depth += 1; - self.id_stack[self.depth as usize] = id; - - Ok(()) - } - - pub fn pop_id(&mut self) -> Result { - if self.depth.checked_sub(1) == None { - return Err(TokenizerError::ExceededDepthRange); - } - - let id = self.id_stack[self.depth as usize]; - self.id_stack[self.depth as usize] = 0; - self.depth -= 1; - - Ok(id) - } - } -} - -use crate::tokenizer::state::State; - -enum TokenTypes<'t> { - Normal, - Comment, - String, - Number, - Operator, - Symbol, - Literal, - Whitespace, - Function, - Keyword, - KeywordAlt, - Custom(&'t str) -} - -pub struct Tokenizer<'t> { - pub syntax: &'t str, - syntax_tree: Vec>, - state: State -} - -impl <'t>Tokenizer<'t> { - pub fn new(syntax: &'t str) -> Self { - Tokenizer { - syntax, - syntax_tree: Vec::new(), - state: State::new() - } - } - - fn push_token() { - - } - - pub fn tokenize() { - - } -} - - diff --git a/src/tokenizer/mod.rs b/src/tokenizer/mod.rs new file mode 100644 index 0000000..5b012d7 --- /dev/null +++ b/src/tokenizer/mod.rs @@ -0,0 +1,3 @@ +pub mod state; +pub mod types; +pub mod tokenizer; diff --git a/src/tokenizer/state.rs b/src/tokenizer/state.rs new file mode 100644 index 0000000..f2f6e96 --- /dev/null +++ b/src/tokenizer/state.rs @@ -0,0 +1,39 @@ +use crate::tokenizer::types::Error; + +/// Should never be modified directly. Instead use the `push_id` and `pop_id` methods. +pub struct State { + depth: u8, + id_stack: Vec +} + +impl State { + pub fn new() -> Self { + State { + depth: 0, + id_stack: Vec::new() + } + } + + pub fn get_id(&self) -> u8 { + self.id_stack[self.depth as usize] + } + + pub fn push_id(&mut self, id: u8) -> Result<(), Error> { + self.depth += 1; + self.id_stack[self.depth as usize] = id; + + Ok(()) + } + + pub fn pop_id(&mut self) -> Result { + if self.depth.checked_sub(1) == None { + return Err(Error::ExceededDepthRange); + } + + let id = self.get_id(); + self.id_stack[self.depth as usize] = 0; + self.depth -= 1; + + Ok(id) + } +} diff --git a/src/tokenizer/tokenizer.rs b/src/tokenizer/tokenizer.rs new file mode 100644 index 0000000..b98ab80 --- /dev/null +++ b/src/tokenizer/tokenizer.rs @@ -0,0 +1,101 @@ +use lua_patterns::LuaPattern; +use pcre2::bytes::Regex; + +use crate::syntax::types::{ + Rule, + RegexOrPattern, + PatternTypes, + Syntax +}; + +use crate::tokenizer::{ + state::State, + types::{ + TokenKind, + Token + } +}; + +pub struct Tokenizer<'t> { + pub syntax: Syntax<'t>, + syntax_tree: Vec>, + state: State +} + +impl <'t>Tokenizer<'t> { + pub fn new(syntax: Syntax<'t>) -> Self { + Tokenizer { + syntax, + syntax_tree: Vec::new(), + state: State::new() + } + } + + fn push_token(&mut self, token_type: TokenKind<'t>, value: String) { + match &self.syntax_tree.last() { + Some(last_token) => { + let mut pattern = LuaPattern::new("^%s*$"); + + if last_token.token_type == token_type || pattern.matches(&last_token.value) { + self.syntax_tree.last_mut().unwrap().value.push_str(&value); + } + }, + + None => { + self.syntax_tree.push(Token { + token_type, + value + }); + } + } + } + + /// 'offset' - Where to start matching + /// 'at_start' - Forces the tokenizer to start matching at the start of text + /// 'close' - Whether the rule is a closing one + fn find_text(text: &str, rule: &mut Rule<'t>, offset: u8, at_start: bool, close: bool) { + let rule_index = if close { 2 } else { 1 }; + let target_pattern: &RegexOrPattern = match rule.pattern { + PatternTypes::Type1(ref pattern) => &pattern, + PatternTypes::Type2(ref pattern) => &pattern[rule_index], + PatternTypes::Type3(ref pattern) => &pattern[rule_index] + }; + + let target_as_string; + + match target_pattern { + RegexOrPattern::Regex(regex) => { + target_as_string = regex.as_str().to_string() + }, + RegexOrPattern::Pattern(pattern) => { + target_as_string = pattern.as_string(); + } + } + + // Check if the pattern contains a "^" + let anchor_check = LuaPattern::new("^%^").match_maybe(&target_as_string); + + rule.whole_line = match anchor_check { + Some(_) => true, + None => false + }; + + if rule.whole_line { + match &mut rule.pattern { + PatternTypes::Type1(RegexOrPattern::Regex(regex)) => { + *regex = Regex::new(&target_as_string).unwrap(); + }, + PatternTypes::Type1(RegexOrPattern::Pattern(pattern)) => { + *pattern = LuaPattern::new(&target_as_string); + }, + _ => {} + } + } + } + + pub fn tokenize() { + + } +} + + diff --git a/src/tokenizer/types.rs b/src/tokenizer/types.rs new file mode 100644 index 0000000..710ca41 --- /dev/null +++ b/src/tokenizer/types.rs @@ -0,0 +1,41 @@ +pub enum Error { + ExceededDepthRange, +} + +#[derive(PartialEq)] +pub enum TokenKind<'t> { + Normal, + Comment, + String, + Number, + Operator, + Symbol, + Literal, + Function, + Keyword, + KeywordAlt, + Custom(&'t str) +} + +impl <'t> Into<&'t str> for TokenKind<'t> { + fn into(self) -> &'t str { + match self { + TokenKind::Normal => "normal", + TokenKind::Comment => "comment", + TokenKind::String => "string", + TokenKind::Number => "number", + TokenKind::Operator => "operator", + TokenKind::Symbol => "symbol", + TokenKind::Literal => "literal", + TokenKind::Function => "function", + TokenKind::Keyword => "keyword", + TokenKind::KeywordAlt => "keyword2", + TokenKind::Custom(custom_kind) => custom_kind + } + } +} + +pub struct Token<'t> { + pub token_type: TokenKind<'t>, + pub value: String +}