Push-down Accumulation and TT muncher does not identify tokens - rust

This is a follow up question to a question see:
Is there any way of accessing the tokens from a token tree or expression in rust (without stringifing and having to parse)
for context.
I am currently trying to Tokenise the input using macros so if I put in an input:
let func: Vec<Symbolic> = symbolic!{fn a * sin(a * x) + x^2}
should output
use Symbolic::*; // Obviously shouldn't output this but just to simplify
vec! [ Identifier("a") , Operator("*") , Function("sin(") , Identifier("a") , Operator("*") ,
Identifier("x") , Seperator(')') , Operator("+") , Identifier("x") , Operator("^") ,
Rational(2,1)
my idea is that when I convert from infix to postfix/rpn any function requires a closing delimiter ) and acts as a unary expression on its child:
*
/ \
sin a
|
*
/ \
a x
so here is my issue I'm using push-down accumulators and for some reason I am getting the error
symbolic!(#munch ( $($rest)* ) -> [ $($accum)* Identifier(stringify!($param)), ])
^ no rules expected this token in macro call
Here is my current code
use Symbolic::*;
#[macro_export]
macro_rules! symbolic{
(fn $($expression:tt)*) => {
symbolic!(#munch ( $($expression)* ) -> [])
};
( #munch ($num:literal $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic!(#munch ( $($rest)* ) -> [ $($accum)* Rational($num,1), ])
};
( #munch ($param:ident $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic!(#munch ( $($rest)* ) -> [ $($accum)* Identifier(stringify!($param)), ])
};
( #munch ($func:ident ($($expression:tt)*) $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic!(#munch ( $($rest)* ) -> [
$($accum)*
Function(stringify!(func)),
symbolic!(#munch ( $($expression)* ) -> []),
Seperator(")")
])
};
( #munch (($($expression:tt)*) $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic!(#munch ( $($rest)* ) -> [
$($accum)*
Operator("("),
symbolic!(#munch ( $($expression)* ) -> []),
Seperator(")")
])
};
( #munch (+ $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic!(#munch ( $($rest)* ) -> [ $($accum)* Operator('+'), ])
};
( #munch (- $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic!(#munch ( $($rest)* ) -> [ $($accum)* Operator('-'), ])
};
( #munch (/ $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic!(#munch ( $($rest)* ) -> [ $($accum)* Operator('/'), ])
};
( #munch (. $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic!(#munch ( $($rest)* ) -> [ $($accum)* Operator('*'), ])
};
( #munch (^ $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic!(#munch ( $($rest)* ) -> [ $($accum)* Operator('^'), ])
};
( #munch () -> [$($accum:tt)*] ) => {
vec![ $($accum)* ]
};
}
this is the new data model
pub enum Symbolic<'a>{
Identifier(&'a str),
Function(&'a str),
Separator(char),
Rational((isize, usize)),
Operator(char)
}
pub enum Node<'a>{
Symbolic(Symbolic<'a>),
Expression{
op: Symbolic<'a>,
buf: Vec<&'a Self>
}
}

Related

Is there any way of accessing the tokens from a token tree or expression in rust (without stringifing and having to parse)

So my question is two-fold, I'm trying to make a computer algebra system similar to SymPy or Math.NET symbolics
my idea is to use some sort of macro to allow this syntax:
let symbolic!(fn function(x, a) -> 2/4*x^2 + a*x + 4)
function.derive(x) // x + 1
What I'm looking for is a way to access the tokens from the token tree so that x, a becomes Symbolic::Symbol("x"), Symbolic::Symbol("a") and 2/4, 4 become Symbolic::Rational(2,4) Symbolic::Rational(4,1), i can then use operator overloading to construct the abstract syntax tree, : eg :
impl Add<Self> for Symbolic {
type Output = Node;
fn add(self, rhs: Self) -> Self::Output {
use Node::*;
BinaryExpr { op: '+', lhs: Box::new(Symb(self)), rhs: Box::new(Symb(rhs)) }
}
}
where the enumsSymbolic and Node are:
pub enum Symbolic{
Symbol(String),
Rational(isize, usize),
Operator(char)
}
pub enum Node{
Symb(Symbolic),
UnaryExpr{
op: char,
child: Box<Node>
},
BinaryExpr{
op: char,
lhs: Box<Node>,
rhs: Box<Node>
}
}
There is certainly a way to accomplish this using a declarative macro_rules! macro.
Here is what I came up with:
/// Macro for declaring a symbolic `Function`
///
/// # Example
///
/// ```
/// let function = symbolic!(fn (a, b) -> a + b^2);
/// let functione = Function {
/// parameters: vec![String::from("a"), String::from("b")],
/// expression: Node::Symb(Symbolic::Symbol(String::from("a"))) + (Node::Symb(Symbolic::Symbol(String::from("b"))) ^ Node::Symb(Symbolic::Rational(2, 1))),
/// };
/// assert_eq!(function, functione);
/// ```
macro_rules! symbolic {
// Main entry point
( fn ($($params:ident),* $(,)*) -> $($expression:tt)* ) => {
Function {
// Extract parameters
parameters: vec![
$( String::from(stringify!($params)), )*
],
// Pass expression to tt muncher
// Starting with an empty accumulator
expression: symbolic!(#munch ( $($expression)* ) -> ()),
}
};
// Handle exponentiation with ^ as highest priority
//
// Capture the left (base) and right (exponent) sides as raw token trees,
// which helpfully captures parenthesized expressions
( #munch ($base:tt^$exponent:tt $($rest:tt)*) -> ($($accum:tt)*) ) => {
// Pass the rest of the expression to continue parsing recursively
symbolic!(#munch ( $($rest)* ) -> (
// Append the exponentiation wrapped in parens to the accumulator
$($accum)*
// Parse the base and exponent as expressions
(symbolic!(#munch ($base) -> ()) ^ symbolic!(#munch ($exponent) -> ()))
))
};
// Handle parenthesized expressions directly
//
// Unwrap the parenthesis
( #munch (($($expression:tt)*) $($rest:tt)*) -> ($($accum:tt)*) ) => {
// Pass the rest of the expression to continue parsing recursively
symbolic!(#munch ( $($rest)* ) -> (
// Append the expression parse invocation to the accumulator
$($accum)*
// Parse the inner expression
// This is wrapped in parens by the final munch case
symbolic!(#munch ( $($expression)* ) -> ())
))
};
// Handle division of two literal integers as a single rational
//
// Capture the left (numerator) and right (denominator) sides,
// and pass them through as a rational
( #munch ($numerator:literal/$denominator:literal $($rest:tt)*) -> ($($accum:tt)*) ) => {
// Pass the rest of the expression to continue parsing recursively
symbolic!(#munch ( $($rest)* ) -> (
// Append the rational to the accumulator
$($accum)*
Node::Symb(Symbolic::Rational($numerator, $denominator))
))
};
// Handle a single literal number as a rational with denominator of 1
( #munch ($num:literal $($rest:tt)*) -> ($($accum:tt)*) ) => {
// Pass the rest of the expression to continue parsing recursively
symbolic!(#munch ( $($rest)* ) -> (
// Append the rational to the accumulator
$($accum)*
Node::Symb(Symbolic::Rational($num, 1))
))
};
// Handle a parameter name
( #munch ($param:ident $($rest:tt)*) -> ($($accum:tt)*) ) => {
// Pass the rest of the expression to continue parsing recursively
symbolic!(#munch ( $($rest)* ) -> (
// Append the parameter symbol to the accumulator
$($accum)*
Node::Symb(Symbolic::Symbol(String::from(stringify!($param))))
))
};
// Pass through operators directly
//
// For better ergonomics, you may want to handle each operator separately,
// as this will allow literally any token through
( #munch ($op:tt $($rest:tt)*) -> ($($accum:tt)*) ) => {
symbolic!(#munch ( $($rest)* ) -> ( $($accum)* $op ))
};
// Handle the final output when all tokens have been handled
( #munch () -> ($($accum:tt)*) ) => {
// Just return the final accumulated Rust expression
( $($accum)* )
};
}
This macro is a lot. The first thing you might pick out is a lot of patterns that look like this:
( #munch ($first:thing $($rest:tt)*) -> ($($accum:tt)*) )
This is a combination of two patterns:
Incremental TT munchers
Push-down Accumulation
I highly recommend reading through that macro book, especially those pages. Here is a quick explanation.
Incremental TT munchers allow the declarative macro to handle a token stream bit by bit. You give a pattern for the bit you want matched, and then the $($rest:tt)* pattern matches any sequence of tokens after it. By passing through $($rest)* to a subsequent invocation, the whole token sequence can be handled.
Push-down Accumulation is way of working around the fact that a macro must always produce a valid item. Essentially, macros can't return something like 1, 2 because that's not a valid expression. Instead, we have to pass through an accumulated value and append to it with each subsequent macro invocation.
However, I don't think that macro with overloading operators is really what would serve you best. I think a better way would be to have a simpler macro that just converts tokens as a kind of lexer, and then parse the output of that yourself.
That way, you could completely control operator precedence, etc.
Here's a macro that will do the simpler lexing instead:
fn parse_symbolic_expression(symbols: Vec<Symbolic>) -> Node {
todo!()
}
macro_rules! symbolic2 {
( fn ($($params:ident),* $(,)?) -> $($expression:tt)* ) => {
Function {
// Extract parameters
parameters: vec![
$( String::from(stringify!($params)), )*
],
expression: parse_symbolic_expression(
symbolic2!(#munch ( $($expression)* ) -> [])
),
}
};
( #munch ($num:literal $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic2!(#munch ( $($rest)* ) -> [ $($accum)* Symbolic::Integer($num), ])
};
( #munch ($param:ident $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic2!(#munch ( $($rest)* ) -> [ $($accum)* Symbolic::Symbol(String::from(stringify!($param))), ])
};
( #munch (($($expression:tt)*) $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic2!(#munch ( $($rest)* ) -> [ $($accum)* Symbolic::Parenthesized(symbolic2!(#munch ( $($expression)* ) -> [])), ])
};
( #munch (+ $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic2!(#munch ( $($rest)* ) -> [ $($accum)* Symbolic::BinaryOperator('+'), ])
};
( #munch (- $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic2!(#munch ( $($rest)* ) -> [ $($accum)* Symbolic::BinaryOperator('-'), ])
};
( #munch (/ $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic2!(#munch ( $($rest)* ) -> [ $($accum)* Symbolic::BinaryOperator('/'), ])
};
( #munch (* $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic2!(#munch ( $($rest)* ) -> [ $($accum)* Symbolic::BinaryOperator('*'), ])
};
( #munch (^ $($rest:tt)*) -> [$($accum:tt)*] ) => {
symbolic2!(#munch ( $($rest)* ) -> [ $($accum)* Symbolic::BinaryOperator('^'), ])
};
( #munch () -> [$($accum:tt)*] ) => {
vec![ $($accum)* ]
};
}

Rust macro: capture exactly matching tokens

My goal is to write a macro expand! such that:
struct A;
struct B;
struct Mut<T>;
expand!() => ()
expand!(A) => (A,)
expand!(mut A) => (Mut<A>,)
expand!(A, mut B) => (A, Mut<B>,)
// etc
[Edit] added trailing comma for consistent tuple syntax.
I wrote this macro so far:
macro_rules! to_type {
( $ty:ty ) => { $ty };
( mut $ty:ty ) => { Mut<$ty> };
}
macro_rules! expand {
( $( $(mut)? $ty:ty ),* ) => {
(
$( to_type!($ty) ),*
,)
};
}
What I'm struggling with, is capturing the mut token. How can I assign it to a variable and reuse it in the macro body? Is it possible to work on more than 1 token at a time?
Something like this?
macro_rules! expand {
(#phase2($($ty_final:ty),*),) => {
($($ty_final,)*)
};
(#phase2($($ty_final:ty),*), mut $ty:ty, $($rest:tt)*) => {
expand!(#phase2($($ty_final,)* Mut::<$ty>), $($rest)*)
};
(#phase2($($ty_final:ty),*), $ty:ty, $($rest:tt)*) => {
expand!(#phase2($($ty_final,)* $ty), $($rest)*)
};
($($t:tt)*) => {
expand!(#phase2(), $($t)*)
};
}
struct A;
struct B;
struct Mut<T>(std::marker::PhantomData<T>);
fn main() {
#[allow(unused_parens)]
let _: expand!() = ();
#[allow(unused_parens)]
let _: expand!(A,) = (A,);
#[allow(unused_parens)]
let _: expand!(mut B,) = (Mut::<B>(Default::default()),);
#[allow(unused_parens)]
let _: expand!(A, mut B,) = (A, Mut::<B>(Default::default()));
}

Conditionally generate a From impl based on field type

I'm trying to create a generic impl for generating a From/Into based on different field types.
Link to Playground
I'm seeing the following issues:
error[E0425]: cannot find value `item` in this scope
--> src/lib.rs:23:21
|
23 | $param: item.$param,
| ^^^^ not found in this scope
...
65 | create_impl! { TargetStruct, InputStruct, { field1: Option<String>, field2: Option<Uuid> }}
| ------------------------------------------------------------------------------------------- in this macro invocation
Can anyone point me in the right direction on how to get this to work / if it's possible. At this point, I'm uncertain of how to pass the input parameter to the rules.
Thanks!
#[macro_use]
macro_rules! create_impl {
( # $target:ident, $input:ident, { } -> ($($result:tt)*) ) => (
impl From<$input> for $target {
fn from(item: $input) -> Self {
Self {
$($result)*
..Default::default()
}
}
});
( # $target:ident, $input:ident, { $param:ident : Option<Uuid>, $($rest:tt)* } -> ($($result:tt)*) ) => (
create_impl!(# $target, $input, { $($rest)* } -> (
$($result)*
$param: item.$param.map(|v| v.to_string()),
));
);
( # $target:ident, $input:ident, { $param:ident : $type:ty, $($rest:tt)* } -> ($($result:tt)*) ) => (
create_impl!(# $target, $input, { $($rest)* } -> (
$($result)*
$param: item.$param,
));
);
( # $target:ident, $input:ident, { $param:ident : $type:ty, $($rest:tt)* } -> ($($result:tt)*) ) => (
create_impl!(# $target, $input, { $($rest)* } -> (
$($result)*
$param: item.$param,
));
);
( $target:ident, $input:ident, { $( $param:ident : $type:ty ),* $(,)* } ) => (
create_impl!(# $target, $input, { $($param : $type,)* } -> ());
);
}
You could let rust do the heavy lifting:
macro_rules! memberize_result {
($item: ident, $param:ident : Option<Uuid>) => (
$item.$param.map(|v| v.to_string())
);
($item: ident, $param:ident : Option<String>) => (
$item.$param
);
}
#[macro_use]
macro_rules! create_impl {
( $target:ident, $input:ident, { $( $param:ident : ($($type:tt)*) ),* $(,)* } ) => (
impl From<$input> for $target {
fn from(item: $input) -> Self {
Self {
$($param: memberize_result!(item, $param : $($type)*),)*
..Default::default()
}
}
}
);
}
use uuid::Uuid; // 0.8.1
#[derive(Default)]
pub struct InputStruct {
pub field1: Option<String>,
pub field2: Option<Uuid>,
}
#[derive(Default)]
pub struct TargetStruct {
pub field1: Option<String>,
pub field2: Option<String>,
}
// Trying to turn this into a macro
// impl From<ExampleStruct> for TargetStruct {
// fn from(item: ExampleStruct) -> Self {
// let mut res = Self::default();
// res.field1 = item.field1;
// res.field2 = item.field2.map(|v| v.to_string());
// res
// }
// }
create_impl! { TargetStruct, InputStruct, { field1: (Option<String>), field2: (Option<Uuid>) }}

Where is the type Log defined inside of the Substrate runtime crate?

Take a look at this implementation:
impl consensus::Trait for Runtime {
type Log = Log;
type SessionKey = AuthorityId;
// The Aura module handles offline-reports internally
// rather than using an explicit report system.
type InherentOfflineReport = ();
}
How is Log defined? There is no use clause for importing this symbol.
Running
cargo rustc -- -Z unstable-options --pretty=expanded
does not show any Log entry with type clause. It does show other macro declarations after expanding macros at level 0, but I am not sure if this relevant.
I tried using the Atom IDE because it automatically parses the files and lets you find the definition of the symbols, but it did not help.
How can I find how Log is defined?
Log is defined by construct_runtime macro.
Here are some relevant code:
construct_runtime macro:
https://github.com/paritytech/substrate/blob/950e90e75dc7d16dcf99972fcc733945a832dc3e/srml/support/src/runtime.rs#L79
macro_rules! construct_runtime {
(
pub enum $runtime:ident with Log ($log_internal:ident: DigestItem<$( $log_genarg:ty ),+>)
where
Block = $block:ident,
NodeBlock = $node_block:ty,
UncheckedExtrinsic = $uncheckedextrinsic:ident
{
$( $rest:tt )*
}
)
calling __decl_outer_log
https://github.com/paritytech/substrate/blob/950e90e75dc7d16dcf99972fcc733945a832dc3e/srml/support/src/runtime.rs#L267
$crate::__decl_outer_log!(
$runtime;
$log_internal < $( $log_genarg ),* >;
{};
$(
$name: $module:: $( < $module_instance >:: )? { $( $modules $( ( $( $modules_args )* ) )* )* }
)*
);
__decl_outer_log macro
https://github.com/paritytech/substrate/blob/950e90e75dc7d16dcf99972fcc733945a832dc3e/srml/support/src/runtime.rs#L706
macro_rules! __decl_outer_log {
(
$runtime:ident;
$log_internal:ident <$( $log_genarg:ty ),+>;
{ $( $parsed:tt )* };
$name:ident: $module:ident:: $(<$module_instance:ident>::)? {
Log ( $( $args:ident )* ) $( $modules:ident $( ( $( $modules_args:ident )* ) )* )*
}
$( $rest:tt )*
) => {
calling impl_outer_log
https://github.com/paritytech/substrate/blob/950e90e75dc7d16dcf99972fcc733945a832dc3e/srml/support/src/runtime.rs#L763
(
$runtime:ident;
$log_internal:ident <$( $log_genarg:ty ),+>;
{ $(
$parsed_modules:ident $(< $parsed_instance:ident >)? ( $( $parsed_args:ident )* )
)* };
) => {
$crate::paste::item! {
$crate::runtime_primitives::impl_outer_log!(
pub enum Log($log_internal: DigestItem<$( $log_genarg ),*>) for $runtime {
$( [< $parsed_modules $(_ $parsed_instance)? >] $(< $parsed_modules::$parsed_instance >)? ( $( $parsed_args ),* ) ),*
}
);
}
};
impl_outer_log macro:
https://github.com/paritytech/substrate/blob/950e90e75dc7d16dcf99972fcc733945a832dc3e/core/sr-primitives/src/lib.rs#L630
macro_rules! impl_outer_log {
(
$(#[$attr:meta])*
pub enum $name:ident ($internal:ident: DigestItem<$( $genarg:ty ),*>) for $trait:ident {
$( $module:ident $(<$instance:path>)? ( $( $sitem:ident ),* ) ),*
}
)
which actually declare and implement the Log struct
You should be able to see the result when cargo expand the runtime crate.

How do you write a macro with chainable tokens?

I'm not really sure how to phrase this, so the question title is pretty rubbish, but here's what I'm trying to do:
I can write this macro:
macro_rules! op(
( $v1:ident && $v2:ident ) => { Op::And($v1, $v2) };
( $v1:ident || $v2:ident ) => { Op::Or($v1, $v2) };
);
Which I can use like this:
let _ = op!(Expr || Expr);
let _ = op!(Expr && Expr);
What I want to do is to write an arbitrary sequence of tokens like this:
let _ = op!(Expr || Expr || Expr && Expr || Expr);
Which resolves into a Vec of tokens, like:
vec!(T::Expr(e1), T::Or, T::Expr(e2), T::Or, ...)
I can write a vec! like macro:
macro_rules! query(
( $( $x:expr ),* ) => {
{
let mut temp_vec = Vec::new();
$(temp_vec.push($x);)*
temp_vec
}
};
);
...but I can't see how to convert the arbitrary symbols (eg. &&) into tokens as the macro runs.
Is this possible somehow?
playpen link: http://is.gd/I9F5YV
It seems that it's impossible to capture arbitrary symbols matches during macroexpand: as the language reference says, "valid designators are item, block, stmt, pat, expr, ty (type), ident, path, tt". So the best I could suggest is to use "ident"-valid tokens, like "and"/"or" instead of "&&"/"||", for example:
macro_rules! query_op(
( and ) => { "T::And" };
( or ) => { "T::Or" };
( $e:ident ) => { concat!("T::Expr(", stringify!($e), ")") };
);
macro_rules! query(
( $( $x:ident )* ) => {
{
let mut temp_vec = Vec::new();
$(temp_vec.push(query_op!($x));)*
temp_vec
}
};
);
fn main() {
let q = query!(Expr1 or Expr2 and Expr3 or Expr4);
println!("{:?}", q);
}
Outputs:
["T::Expr(Expr1)", "T::Or", "T::Expr(Expr2)", "T::And", "T::Expr(Expr3)", "T::Or", "T::Expr(Expr4)"]

Resources