94 lines
1.8 KiB
Rust
94 lines
1.8 KiB
Rust
pub enum TokenizerError {
|
|
ExceededDepthRange,
|
|
}
|
|
|
|
mod state {
|
|
use crate::tokenizer::TokenizerError;
|
|
|
|
/// State is divided into four 8 byte long ints, each int represents the rule id
|
|
/// Should never be modified directly. Instead use the push_id and pop_id methods.
|
|
pub struct State {
|
|
depth: u8, // Valid range is 0-3
|
|
id_stack: [u8; 4]
|
|
}
|
|
|
|
impl State {
|
|
pub fn new() -> Self {
|
|
State {
|
|
depth: 0,
|
|
id_stack: [0, 0, 0, 0]
|
|
}
|
|
}
|
|
|
|
fn get_state() {
|
|
|
|
}
|
|
|
|
pub fn push_id(&mut self, id: u8) -> Result<(), TokenizerError> {
|
|
if self.depth + 1 > 3 {
|
|
return Err(TokenizerError::ExceededDepthRange);
|
|
}
|
|
|
|
self.depth += 1;
|
|
self.id_stack[self.depth as usize] = id;
|
|
|
|
Ok(())
|
|
}
|
|
|
|
pub fn pop_id(&mut self) -> Result<u8, TokenizerError> {
|
|
if self.depth.checked_sub(1) == None {
|
|
return Err(TokenizerError::ExceededDepthRange);
|
|
}
|
|
|
|
let id = self.id_stack[self.depth as usize];
|
|
self.id_stack[self.depth as usize] = 0;
|
|
self.depth -= 1;
|
|
|
|
Ok(id)
|
|
}
|
|
}
|
|
}
|
|
|
|
use crate::tokenizer::state::State;
|
|
|
|
enum TokenTypes<'t> {
|
|
Normal,
|
|
Comment,
|
|
String,
|
|
Number,
|
|
Operator,
|
|
Symbol,
|
|
Literal,
|
|
Whitespace,
|
|
Function,
|
|
Keyword,
|
|
KeywordAlt,
|
|
Custom(&'t str)
|
|
}
|
|
|
|
pub struct Tokenizer<'t> {
|
|
pub syntax: &'t str,
|
|
syntax_tree: Vec<TokenTypes<'t>>,
|
|
state: State
|
|
}
|
|
|
|
impl <'t>Tokenizer<'t> {
|
|
pub fn new(syntax: &'t str) -> Self {
|
|
Tokenizer {
|
|
syntax,
|
|
syntax_tree: Vec::new(),
|
|
state: State::new()
|
|
}
|
|
}
|
|
|
|
fn push_token() {
|
|
|
|
}
|
|
|
|
pub fn tokenize() {
|
|
|
|
}
|
|
}
|
|
|
|
|