This is a follow up question to a question see:
Is there any way of accessing the tokens from a token tree or expression in rust (without stringifing and having to parse)
for context.
I am currently trying to Tokenise the input using macros so if I put in an input:
let func: Vec<Symbolic> = symbolic!{fn a * sin(a * x) + x^2}
should output
use Symbolic::*; // Obviously shouldn't output this but just to simplify
vec! [ Identifier("a") , Operator("*") , Function("sin(") , Identifier("a") , Operator("*") ,
Identifier("x") , Seperator(')') , Operator("+") , Identifier("x") , Operator("^") ,
Rational(2,1)
my idea is that when I convert from infix to postfix/rpn any function requires a closing delimiter ) and acts as a unary expression on its child:
*
/ \
sin a
|
*
/ \
a x
so here is my issue I'm using push-down accumulators and for some reason I am getting the error
symbolic!(#munch ( $($rest)* ) -> [ $($accum)* Identifier(stringify!($param)), ])
^ no rules expected this token in macro call
Here is my current code
use Symbolic::*;
#[macro_export]
macro_rules! symbolic{
(fn $($expression:tt)*) => {
symbolic!(#munch ( $($expression)* ) -> [])
};
( #munch ($num:literal $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic!(#munch ( $($rest)* ) -> [ $($accum)* Rational($num,1), ])
};
( #munch ($param:ident $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic!(#munch ( $($rest)* ) -> [ $($accum)* Identifier(stringify!($param)), ])
};
( #munch ($func:ident ($($expression:tt)*) $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic!(#munch ( $($rest)* ) -> [
$($accum)*
Function(stringify!(func)),
symbolic!(#munch ( $($expression)* ) -> []),
Seperator(")")
])
};
( #munch (($($expression:tt)*) $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic!(#munch ( $($rest)* ) -> [
$($accum)*
Operator("("),
symbolic!(#munch ( $($expression)* ) -> []),
Seperator(")")
])
};
( #munch (+ $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic!(#munch ( $($rest)* ) -> [ $($accum)* Operator('+'), ])
};
( #munch (- $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic!(#munch ( $($rest)* ) -> [ $($accum)* Operator('-'), ])
};
( #munch (/ $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic!(#munch ( $($rest)* ) -> [ $($accum)* Operator('/'), ])
};
( #munch (. $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic!(#munch ( $($rest)* ) -> [ $($accum)* Operator('*'), ])
};
( #munch (^ $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic!(#munch ( $($rest)* ) -> [ $($accum)* Operator('^'), ])
};
( #munch () -> [$($accum:tt)*] ) => {
vec![ $($accum)* ]
};
}
this is the new data model
pub enum Symbolic<'a>{
Identifier(&'a str),
Function(&'a str),
Separator(char),
Rational((isize, usize)),
Operator(char)
}
pub enum Node<'a>{
Symbolic(Symbolic<'a>),
Expression{
op: Symbolic<'a>,
buf: Vec<&'a Self>
}
}
Related
So my question is two-fold, I'm trying to make a computer algebra system similar to SymPy or Math.NET symbolics
my idea is to use some sort of macro to allow this syntax:
let symbolic!(fn function(x, a) -> 2/4*x^2 + a*x + 4)
function.derive(x) // x + 1
What I'm looking for is a way to access the tokens from the token tree so that x, a becomes Symbolic::Symbol("x"), Symbolic::Symbol("a") and 2/4, 4 become Symbolic::Rational(2,4) Symbolic::Rational(4,1), i can then use operator overloading to construct the abstract syntax tree, : eg :
impl Add<Self> for Symbolic {
type Output = Node;
fn add(self, rhs: Self) -> Self::Output {
use Node::*;
BinaryExpr { op: '+', lhs: Box::new(Symb(self)), rhs: Box::new(Symb(rhs)) }
}
}
where the enumsSymbolic and Node are:
pub enum Symbolic{
Symbol(String),
Rational(isize, usize),
Operator(char)
}
pub enum Node{
Symb(Symbolic),
UnaryExpr{
op: char,
child: Box<Node>
},
BinaryExpr{
op: char,
lhs: Box<Node>,
rhs: Box<Node>
}
}
There is certainly a way to accomplish this using a declarative macro_rules! macro.
Here is what I came up with:
/// Macro for declaring a symbolic `Function`
///
/// # Example
///
/// ```
/// let function = symbolic!(fn (a, b) -> a + b^2);
/// let functione = Function {
/// parameters: vec![String::from("a"), String::from("b")],
/// expression: Node::Symb(Symbolic::Symbol(String::from("a"))) + (Node::Symb(Symbolic::Symbol(String::from("b"))) ^ Node::Symb(Symbolic::Rational(2, 1))),
/// };
/// assert_eq!(function, functione);
/// ```
macro_rules! symbolic {
// Main entry point
( fn ($($params:ident),* $(,)*) -> $($expression:tt)* ) => {
Function {
// Extract parameters
parameters: vec![
$( String::from(stringify!($params)), )*
],
// Pass expression to tt muncher
// Starting with an empty accumulator
expression: symbolic!(#munch ( $($expression)* ) -> ()),
}
};
// Handle exponentiation with ^ as highest priority
//
// Capture the left (base) and right (exponent) sides as raw token trees,
// which helpfully captures parenthesized expressions
( #munch ($base:tt^$exponent:tt $($rest:tt)*) -> ($($accum:tt)*) ) => {
// Pass the rest of the expression to continue parsing recursively
symbolic!(#munch ( $($rest)* ) -> (
// Append the exponentiation wrapped in parens to the accumulator
$($accum)*
// Parse the base and exponent as expressions
(symbolic!(#munch ($base) -> ()) ^ symbolic!(#munch ($exponent) -> ()))
))
};
// Handle parenthesized expressions directly
//
// Unwrap the parenthesis
( #munch (($($expression:tt)*) $($rest:tt)*) -> ($($accum:tt)*) ) => {
// Pass the rest of the expression to continue parsing recursively
symbolic!(#munch ( $($rest)* ) -> (
// Append the expression parse invocation to the accumulator
$($accum)*
// Parse the inner expression
// This is wrapped in parens by the final munch case
symbolic!(#munch ( $($expression)* ) -> ())
))
};
// Handle division of two literal integers as a single rational
//
// Capture the left (numerator) and right (denominator) sides,
// and pass them through as a rational
( #munch ($numerator:literal/$denominator:literal $($rest:tt)*) -> ($($accum:tt)*) ) => {
// Pass the rest of the expression to continue parsing recursively
symbolic!(#munch ( $($rest)* ) -> (
// Append the rational to the accumulator
$($accum)*
Node::Symb(Symbolic::Rational($numerator, $denominator))
))
};
// Handle a single literal number as a rational with denominator of 1
( #munch ($num:literal $($rest:tt)*) -> ($($accum:tt)*) ) => {
// Pass the rest of the expression to continue parsing recursively
symbolic!(#munch ( $($rest)* ) -> (
// Append the rational to the accumulator
$($accum)*
Node::Symb(Symbolic::Rational($num, 1))
))
};
// Handle a parameter name
( #munch ($param:ident $($rest:tt)*) -> ($($accum:tt)*) ) => {
// Pass the rest of the expression to continue parsing recursively
symbolic!(#munch ( $($rest)* ) -> (
// Append the parameter symbol to the accumulator
$($accum)*
Node::Symb(Symbolic::Symbol(String::from(stringify!($param))))
))
};
// Pass through operators directly
//
// For better ergonomics, you may want to handle each operator separately,
// as this will allow literally any token through
( #munch ($op:tt $($rest:tt)*) -> ($($accum:tt)*) ) => {
symbolic!(#munch ( $($rest)* ) -> ( $($accum)* $op ))
};
// Handle the final output when all tokens have been handled
( #munch () -> ($($accum:tt)*) ) => {
// Just return the final accumulated Rust expression
( $($accum)* )
};
}
This macro is a lot. The first thing you might pick out is a lot of patterns that look like this:
( #munch ($first:thing $($rest:tt)*) -> ($($accum:tt)*) )
This is a combination of two patterns:
Incremental TT munchers
Push-down Accumulation
I highly recommend reading through that macro book, especially those pages. Here is a quick explanation.
Incremental TT munchers allow the declarative macro to handle a token stream bit by bit. You give a pattern for the bit you want matched, and then the $($rest:tt)* pattern matches any sequence of tokens after it. By passing through $($rest)* to a subsequent invocation, the whole token sequence can be handled.
Push-down Accumulation is way of working around the fact that a macro must always produce a valid item. Essentially, macros can't return something like 1, 2 because that's not a valid expression. Instead, we have to pass through an accumulated value and append to it with each subsequent macro invocation.
However, I don't think that macro with overloading operators is really what would serve you best. I think a better way would be to have a simpler macro that just converts tokens as a kind of lexer, and then parse the output of that yourself.
That way, you could completely control operator precedence, etc.
Here's a macro that will do the simpler lexing instead:
fn parse_symbolic_expression(symbols: Vec<Symbolic>) -> Node {
todo!()
}
macro_rules! symbolic2 {
( fn ($($params:ident),* $(,)?) -> $($expression:tt)* ) => {
Function {
// Extract parameters
parameters: vec![
$( String::from(stringify!($params)), )*
],
expression: parse_symbolic_expression(
symbolic2!(#munch ( $($expression)* ) -> [])
),
}
};
( #munch ($num:literal $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic2!(#munch ( $($rest)* ) -> [ $($accum)* Symbolic::Integer($num), ])
};
( #munch ($param:ident $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic2!(#munch ( $($rest)* ) -> [ $($accum)* Symbolic::Symbol(String::from(stringify!($param))), ])
};
( #munch (($($expression:tt)*) $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic2!(#munch ( $($rest)* ) -> [ $($accum)* Symbolic::Parenthesized(symbolic2!(#munch ( $($expression)* ) -> [])), ])
};
( #munch (+ $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic2!(#munch ( $($rest)* ) -> [ $($accum)* Symbolic::BinaryOperator('+'), ])
};
( #munch (- $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic2!(#munch ( $($rest)* ) -> [ $($accum)* Symbolic::BinaryOperator('-'), ])
};
( #munch (/ $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic2!(#munch ( $($rest)* ) -> [ $($accum)* Symbolic::BinaryOperator('/'), ])
};
( #munch (* $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic2!(#munch ( $($rest)* ) -> [ $($accum)* Symbolic::BinaryOperator('*'), ])
};
( #munch (^ $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic2!(#munch ( $($rest)* ) -> [ $($accum)* Symbolic::BinaryOperator('^'), ])
};
( #munch () -> [$($accum:tt)*] ) => {
vec![ $($accum)* ]
};
}
My goal is to write a macro expand! such that:
struct A;
struct B;
struct Mut<T>;
expand!() => ()
expand!(A) => (A,)
expand!(mut A) => (Mut<A>,)
expand!(A, mut B) => (A, Mut<B>,)
// etc
[Edit] added trailing comma for consistent tuple syntax.
I wrote this macro so far:
macro_rules! to_type {
( $ty:ty ) => { $ty };
( mut $ty:ty ) => { Mut<$ty> };
}
macro_rules! expand {
( $( $(mut)? $ty:ty ),* ) => {
(
$( to_type!($ty) ),*
,)
};
}
What I'm struggling with, is capturing the mut token. How can I assign it to a variable and reuse it in the macro body? Is it possible to work on more than 1 token at a time?
Something like this?
macro_rules! expand {
(#phase2($($ty_final:ty),*),) => {
($($ty_final,)*)
};
(#phase2($($ty_final:ty),*), mut $ty:ty, $($rest:tt)*) => {
expand!(#phase2($($ty_final,)* Mut::<$ty>), $($rest)*)
};
(#phase2($($ty_final:ty),*), $ty:ty, $($rest:tt)*) => {
expand!(#phase2($($ty_final,)* $ty), $($rest)*)
};
($($t:tt)*) => {
expand!(#phase2(), $($t)*)
};
}
struct A;
struct B;
struct Mut<T>(std::marker::PhantomData<T>);
fn main() {
#[allow(unused_parens)]
let _: expand!() = ();
#[allow(unused_parens)]
let _: expand!(A,) = (A,);
#[allow(unused_parens)]
let _: expand!(mut B,) = (Mut::<B>(Default::default()),);
#[allow(unused_parens)]
let _: expand!(A, mut B,) = (A, Mut::<B>(Default::default()));
}
I'm trying to create a generic impl for generating a From/Into based on different field types.
Link to Playground
I'm seeing the following issues:
error[E0425]: cannot find value `item` in this scope
--> src/lib.rs:23:21
|
23 | $param: item.$param,
| ^^^^ not found in this scope
...
65 | create_impl! { TargetStruct, InputStruct, { field1: Option<String>, field2: Option<Uuid> }}
| ------------------------------------------------------------------------------------------- in this macro invocation
Can anyone point me in the right direction on how to get this to work / if it's possible. At this point, I'm uncertain of how to pass the input parameter to the rules.
Thanks!
#[macro_use]
macro_rules! create_impl {
( # $target:ident, $input:ident, { } -> ($($result:tt)*) ) => (
impl From<$input> for $target {
fn from(item: $input) -> Self {
Self {
$($result)*
..Default::default()
}
}
});
( # $target:ident, $input:ident, { $param:ident : Option<Uuid>, $($rest:tt)* } -> ($($result:tt)*) ) => (
create_impl!(# $target, $input, { $($rest)* } -> (
$($result)*
$param: item.$param.map(|v| v.to_string()),
));
);
( # $target:ident, $input:ident, { $param:ident : $type:ty, $($rest:tt)* } -> ($($result:tt)*) ) => (
create_impl!(# $target, $input, { $($rest)* } -> (
$($result)*
$param: item.$param,
));
);
( # $target:ident, $input:ident, { $param:ident : $type:ty, $($rest:tt)* } -> ($($result:tt)*) ) => (
create_impl!(# $target, $input, { $($rest)* } -> (
$($result)*
$param: item.$param,
));
);
( $target:ident, $input:ident, { $( $param:ident : $type:ty ),* $(,)* } ) => (
create_impl!(# $target, $input, { $($param : $type,)* } -> ());
);
}
You could let rust do the heavy lifting:
macro_rules! memberize_result {
($item: ident, $param:ident : Option<Uuid>) => (
$item.$param.map(|v| v.to_string())
);
($item: ident, $param:ident : Option<String>) => (
$item.$param
);
}
#[macro_use]
macro_rules! create_impl {
( $target:ident, $input:ident, { $( $param:ident : ($($type:tt)*) ),* $(,)* } ) => (
impl From<$input> for $target {
fn from(item: $input) -> Self {
Self {
$($param: memberize_result!(item, $param : $($type)*),)*
..Default::default()
}
}
}
);
}
use uuid::Uuid; // 0.8.1
#[derive(Default)]
pub struct InputStruct {
pub field1: Option<String>,
pub field2: Option<Uuid>,
}
#[derive(Default)]
pub struct TargetStruct {
pub field1: Option<String>,
pub field2: Option<String>,
}
// Trying to turn this into a macro
// impl From<ExampleStruct> for TargetStruct {
// fn from(item: ExampleStruct) -> Self {
// let mut res = Self::default();
// res.field1 = item.field1;
// res.field2 = item.field2.map(|v| v.to_string());
// res
// }
// }
create_impl! { TargetStruct, InputStruct, { field1: (Option<String>), field2: (Option<Uuid>) }}
Take a look at this implementation:
impl consensus::Trait for Runtime {
type Log = Log;
type SessionKey = AuthorityId;
// The Aura module handles offline-reports internally
// rather than using an explicit report system.
type InherentOfflineReport = ();
}
How is Log defined? There is no use clause for importing this symbol.
Running
cargo rustc -- -Z unstable-options --pretty=expanded
does not show any Log entry with type clause. It does show other macro declarations after expanding macros at level 0, but I am not sure if this relevant.
I tried using the Atom IDE because it automatically parses the files and lets you find the definition of the symbols, but it did not help.
How can I find how Log is defined?
Log is defined by construct_runtime macro.
Here are some relevant code:
construct_runtime macro:
https://github.com/paritytech/substrate/blob/950e90e75dc7d16dcf99972fcc733945a832dc3e/srml/support/src/runtime.rs#L79
macro_rules! construct_runtime {
(
pub enum $runtime:ident with Log ($log_internal:ident: DigestItem<$( $log_genarg:ty ),+>)
where
Block = $block:ident,
NodeBlock = $node_block:ty,
UncheckedExtrinsic = $uncheckedextrinsic:ident
{
$( $rest:tt )*
}
)
calling __decl_outer_log
https://github.com/paritytech/substrate/blob/950e90e75dc7d16dcf99972fcc733945a832dc3e/srml/support/src/runtime.rs#L267
$crate::__decl_outer_log!(
$runtime;
$log_internal < $( $log_genarg ),* >;
{};
$(
$name: $module:: $( < $module_instance >:: )? { $( $modules $( ( $( $modules_args )* ) )* )* }
)*
);
__decl_outer_log macro
https://github.com/paritytech/substrate/blob/950e90e75dc7d16dcf99972fcc733945a832dc3e/srml/support/src/runtime.rs#L706
macro_rules! __decl_outer_log {
(
$runtime:ident;
$log_internal:ident <$( $log_genarg:ty ),+>;
{ $( $parsed:tt )* };
$name:ident: $module:ident:: $(<$module_instance:ident>::)? {
Log ( $( $args:ident )* ) $( $modules:ident $( ( $( $modules_args:ident )* ) )* )*
}
$( $rest:tt )*
) => {
calling impl_outer_log
https://github.com/paritytech/substrate/blob/950e90e75dc7d16dcf99972fcc733945a832dc3e/srml/support/src/runtime.rs#L763
(
$runtime:ident;
$log_internal:ident <$( $log_genarg:ty ),+>;
{ $(
$parsed_modules:ident $(< $parsed_instance:ident >)? ( $( $parsed_args:ident )* )
)* };
) => {
$crate::paste::item! {
$crate::runtime_primitives::impl_outer_log!(
pub enum Log($log_internal: DigestItem<$( $log_genarg ),*>) for $runtime {
$( [< $parsed_modules $(_ $parsed_instance)? >] $(< $parsed_modules::$parsed_instance >)? ( $( $parsed_args ),* ) ),*
}
);
}
};
impl_outer_log macro:
https://github.com/paritytech/substrate/blob/950e90e75dc7d16dcf99972fcc733945a832dc3e/core/sr-primitives/src/lib.rs#L630
macro_rules! impl_outer_log {
(
$(#[$attr:meta])*
pub enum $name:ident ($internal:ident: DigestItem<$( $genarg:ty ),*>) for $trait:ident {
$( $module:ident $(<$instance:path>)? ( $( $sitem:ident ),* ) ),*
}
)
which actually declare and implement the Log struct
You should be able to see the result when cargo expand the runtime crate.
I'm not really sure how to phrase this, so the question title is pretty rubbish, but here's what I'm trying to do:
I can write this macro:
macro_rules! op(
( $v1:ident && $v2:ident ) => { Op::And($v1, $v2) };
( $v1:ident || $v2:ident ) => { Op::Or($v1, $v2) };
);
Which I can use like this:
let _ = op!(Expr || Expr);
let _ = op!(Expr && Expr);
What I want to do is to write an arbitrary sequence of tokens like this:
let _ = op!(Expr || Expr || Expr && Expr || Expr);
Which resolves into a Vec of tokens, like:
vec!(T::Expr(e1), T::Or, T::Expr(e2), T::Or, ...)
I can write a vec! like macro:
macro_rules! query(
( $( $x:expr ),* ) => {
{
let mut temp_vec = Vec::new();
$(temp_vec.push($x);)*
temp_vec
}
};
);
...but I can't see how to convert the arbitrary symbols (eg. &&) into tokens as the macro runs.
Is this possible somehow?
playpen link: http://is.gd/I9F5YV
It seems that it's impossible to capture arbitrary symbols matches during macroexpand: as the language reference says, "valid designators are item, block, stmt, pat, expr, ty (type), ident, path, tt". So the best I could suggest is to use "ident"-valid tokens, like "and"/"or" instead of "&&"/"||", for example:
macro_rules! query_op(
( and ) => { "T::And" };
( or ) => { "T::Or" };
( $e:ident ) => { concat!("T::Expr(", stringify!($e), ")") };
);
macro_rules! query(
( $( $x:ident )* ) => {
{
let mut temp_vec = Vec::new();
$(temp_vec.push(query_op!($x));)*
temp_vec
}
};
);
fn main() {
let q = query!(Expr1 or Expr2 and Expr3 or Expr4);
println!("{:?}", q);
}
Outputs:
["T::Expr(Expr1)", "T::Or", "T::Expr(Expr2)", "T::And", "T::Expr(Expr3)", "T::Or", "T::Expr(Expr4)"]