feat: release 2.05.0.3.0

CHANGELOG.md

@@ -37,6 +37,7 @@ and this project adheres to the versioning scheme outlined in the [README.md](RE
   state machine to only report fatal errors (#3228)
 - Spawn the p2p thread before processing number of sortitions. Fixes issue (#3216) where sync from genesis paused (#3236)
 - Drop well-formed "problematic" transactions that result in miner performance degradation (#3212)
+- Ignore blocks that include problematic transactions
 
 
 ## [2.05.0.2.1]

Cargo.toml

@@ -117,7 +117,7 @@ default = ["developer-mode"]
 developer-mode = []
 monitoring_prom = ["prometheus"]
 slog_json = ["slog-json", "stacks_common/slog_json", "clarity/slog_json"]
-
+testing = []
 
 [profile.dev.package.regex]
 opt-level = 2

clarity/src/vm/analysis/mod.rs

@@ -40,12 +40,20 @@ use self::contract_interface_builder::build_contract_interface;
 use self::read_only_checker::ReadOnlyChecker;
 use self::trait_checker::TraitChecker;
 use self::type_checker::TypeChecker;
+use crate::vm::ast::build_ast_with_rules;
+use crate::vm::ast::ASTRules;
 
 /// Used by CLI tools like the docs generator. Not used in production
 pub fn mem_type_check(snippet: &str) -> CheckResult<(Option<TypeSignature>, ContractAnalysis)> {
-    use crate::vm::ast::parse;
     let contract_identifier = QualifiedContractIdentifier::transient();
-    let mut contract = parse(&contract_identifier, snippet).unwrap();
+    let mut contract = build_ast_with_rules(
+        &contract_identifier,
+        snippet,
+        &mut (),
+        ASTRules::PrecheckSize,
+    )
+    .unwrap()
+    .expressions;
     let mut marf = MemoryBackingStore::new();
     let mut analysis_db = marf.as_analysis_db();
     let cost_tracker = LimitedCostTracker::new_free();

clarity/src/vm/analysis/type_checker/natives/mod.rs

@@ -207,7 +207,7 @@ pub fn check_special_tuple_cons(
     })?;
 
     let tuple_signature = TupleTypeSignature::try_from(tuple_type_data)
-        .map_err(|_| CheckErrors::BadTupleConstruction)?;
+        .map_err(|_e| CheckErrors::BadTupleConstruction)?;
 
     Ok(TypeSignature::TupleType(tuple_signature))
 }

clarity/src/vm/ast/errors.rs

@@ -31,6 +31,7 @@ pub enum ParseErrors {
     MemoryBalanceExceeded(u64, u64),
     TooManyExpressions,
     ExpressionStackDepthTooDeep,
+    VaryExpressionStackDepthTooDeep,
     FailedCapturingInput,
     SeparatorExpected(String),
     SeparatorExpectedAfterColon(String),
@@ -235,6 +236,10 @@ impl DiagnosableError for ParseErrors {
                 "AST has too deep of an expression nesting. The maximum stack depth is {}",
                 MAX_CALL_STACK_DEPTH
             ),
+            ParseErrors::VaryExpressionStackDepthTooDeep => format!(
+                "AST has too deep of an expression nesting. The maximum stack depth is {}",
+                MAX_CALL_STACK_DEPTH
+            ),
             ParseErrors::InvalidCharactersDetected => format!("invalid characters detected"),
             ParseErrors::InvalidEscaping => format!("invalid escaping detected in string"),
             ParseErrors::CostComputationFailed(s) => format!("Cost computation failed: {}", s),

clarity/src/vm/ast/mod.rs

@@ -33,13 +33,15 @@ use self::definition_sorter::DefinitionSorter;
 use self::errors::ParseResult;
 use self::expression_identifier::ExpressionIdentifier;
 use self::stack_depth_checker::StackDepthChecker;
+use self::stack_depth_checker::VaryStackDepthChecker;
 use self::sugar_expander::SugarExpander;
 use self::traits_resolver::TraitsResolver;
 use self::types::BuildASTPass;
 pub use self::types::ContractAST;
 use crate::vm::costs::cost_functions::ClarityCostFunction;
 
 /// Legacy function
+#[cfg(test)]
 pub fn parse(
     contract_identifier: &QualifiedContractIdentifier,
     source_code: &str,
@@ -48,7 +50,43 @@ pub fn parse(
     Ok(ast.expressions)
 }
 
-pub fn build_ast<T: CostTracker>(
+// AST parser rulesets to apply.
+define_u8_enum!(ASTRules {
+    Typical = 0,
+    PrecheckSize = 1
+});
+
+/// This is the part of the AST parser that runs without respect to cost analysis, specifically
+/// pertaining to verifying that the AST is reasonably-sized.
+/// Used mainly to filter transactions that might be too costly, as an optimization heuristic.
+pub fn ast_check_size(
+    contract_identifier: &QualifiedContractIdentifier,
+    source_code: &str,
+) -> ParseResult<ContractAST> {
+    let pre_expressions = parser::parse(source_code)?;
+    let mut contract_ast = ContractAST::new(contract_identifier.clone(), pre_expressions);
+    StackDepthChecker::run_pass(&mut contract_ast)?;
+    VaryStackDepthChecker::run_pass(&mut contract_ast)?;
+    Ok(contract_ast)
+}
+
+/// Build an AST according to a ruleset
+pub fn build_ast_with_rules<T: CostTracker>(
+    contract_identifier: &QualifiedContractIdentifier,
+    source_code: &str,
+    cost_track: &mut T,
+    ruleset: ASTRules,
+) -> ParseResult<ContractAST> {
+    match ruleset {
+        ASTRules::Typical => build_ast_typical(contract_identifier, source_code, cost_track),
+        ASTRules::PrecheckSize => {
+            build_ast_precheck_size(contract_identifier, source_code, cost_track)
+        }
+    }
+}
+
+/// Build an AST with the typical rules
+fn build_ast_typical<T: CostTracker>(
     contract_identifier: &QualifiedContractIdentifier,
     source_code: &str,
     cost_track: &mut T,
@@ -58,7 +96,7 @@ pub fn build_ast<T: CostTracker>(
         cost_track,
         source_code.len() as u64,
     )?;
-    let pre_expressions = parser::parse(source_code)?;
+    let pre_expressions = parser::parse_no_stack_limit(source_code)?;
     let mut contract_ast = ContractAST::new(contract_identifier.clone(), pre_expressions);
     StackDepthChecker::run_pass(&mut contract_ast)?;
     ExpressionIdentifier::run_pre_expression_pass(&mut contract_ast)?;
@@ -69,14 +107,49 @@ pub fn build_ast<T: CostTracker>(
     Ok(contract_ast)
 }
 
+/// Built an AST, but pre-check the size of the AST before doing more work
+fn build_ast_precheck_size<T: CostTracker>(
+    contract_identifier: &QualifiedContractIdentifier,
+    source_code: &str,
+    cost_track: &mut T,
+) -> ParseResult<ContractAST> {
+    runtime_cost(
+        ClarityCostFunction::AstParse,
+        cost_track,
+        source_code.len() as u64,
+    )?;
+    let mut contract_ast = ast_check_size(contract_identifier, source_code)?;
+    ExpressionIdentifier::run_pre_expression_pass(&mut contract_ast)?;
+    DefinitionSorter::run_pass(&mut contract_ast, cost_track)?;
+    TraitsResolver::run_pass(&mut contract_ast)?;
+    SugarExpander::run_pass(&mut contract_ast)?;
+    ExpressionIdentifier::run_expression_pass(&mut contract_ast)?;
+    Ok(contract_ast)
+}
+
+/// Test compatibility
+#[cfg(any(test, feature = "testing"))]
+pub fn build_ast<T: CostTracker>(
+    contract_identifier: &QualifiedContractIdentifier,
+    source_code: &str,
+    cost_track: &mut T,
+) -> ParseResult<ContractAST> {
+    build_ast_typical(contract_identifier, source_code, cost_track)
+}
+
 #[cfg(test)]
 mod test {
-    use std::collections::HashMap;
-
-    use crate::vm::ast::build_ast;
+    use crate::vm::ast::errors::ParseErrors;
+    use crate::vm::ast::stack_depth_checker::AST_CALL_STACK_DEPTH_BUFFER;
+    use crate::vm::ast::{build_ast, build_ast_with_rules, ASTRules};
     use crate::vm::costs::LimitedCostTracker;
+    use crate::vm::costs::*;
     use crate::vm::representations::depth_traverse;
     use crate::vm::types::QualifiedContractIdentifier;
+    use crate::vm::ClarityCostFunction;
+    use crate::vm::ClarityName;
+    use crate::vm::MAX_CALL_STACK_DEPTH;
+    use std::collections::HashMap;
 
     #[test]
     fn test_expression_identification_tuples() {

clarity/src/vm/ast/parser/mod.rs

@@ -26,6 +26,9 @@ use stacks_common::util::hash::hex_bytes;
 use std::cmp;
 use std::convert::TryInto;
 
+use crate::vm::ast::stack_depth_checker::AST_CALL_STACK_DEPTH_BUFFER;
+use crate::vm::MAX_CALL_STACK_DEPTH;
+
 pub const CONTRACT_MIN_NAME_LENGTH: usize = 1;
 pub const CONTRACT_MAX_NAME_LENGTH: usize = 40;
 
@@ -124,14 +127,8 @@ lazy_static! {
     );
     pub static ref CLARITY_NAME_REGEX: String =
         format!(r#"([[:word:]]|[-!?+<>=/*]){{1,{}}}"#, MAX_STRING_LEN);
-}
 
-pub fn lex(input: &str) -> ParseResult<Vec<(LexItem, u32, u32)>> {
-    // Aaron: I'd like these to be static, but that'd require using
-    //    lazy_static (or just hand implementing that), and I'm not convinced
-    //    it's worth either (1) an extern macro, or (2) the complexity of hand implementing.
-
-    let lex_matchers: &[LexMatcher] = &[
+    static ref lex_matchers: Vec<LexMatcher> = vec![
         LexMatcher::new(
             r##"u"(?P<value>((\\")|([[ -~]&&[^"]]))*)""##,
             TokenType::StringUTF8Literal,
@@ -187,7 +184,10 @@ pub fn lex(input: &str) -> ParseResult<Vec<(LexItem, u32, u32)>> {
             TokenType::Variable,
         ),
     ];
+}
 
+/// Lex the contract, permitting nesting of lists and tuples up to `max_nesting`.
+fn inner_lex(input: &str, max_nesting: u64) -> ParseResult<Vec<(LexItem, u32, u32)>> {
     let mut context = LexContext::ExpectNothing;
 
     let mut line_indices = get_lines_at(input);
@@ -198,6 +198,9 @@ pub fn lex(input: &str) -> ParseResult<Vec<(LexItem, u32, u32)>> {
     let mut munch_index = 0;
     let mut column_pos: u32 = 1;
     let mut did_match = true;
+
+    let mut nesting_depth = 0;
+
     while did_match && munch_index < input.len() {
         if let Some(next_line_ix) = next_line_break {
             if munch_index > next_line_ix {
@@ -255,9 +258,19 @@ pub fn lex(input: &str) -> ParseResult<Vec<(LexItem, u32, u32)>> {
                 let token = match matcher.handler {
                     TokenType::LParens => {
                         context = LexContext::ExpectNothing;
+                        nesting_depth += 1;
+                        if nesting_depth > max_nesting {
+                            return Err(ParseError::new(
+                                ParseErrors::VaryExpressionStackDepthTooDeep,
+                            ));
+                        }
                         Ok(LexItem::LeftParen)
                     }
-                    TokenType::RParens => Ok(LexItem::RightParen),
+                    TokenType::RParens => {
+                        // if this underflows, the contract is invalid anyway
+                        nesting_depth = nesting_depth.saturating_sub(1);
+                        Ok(LexItem::RightParen)
+                    }
                     TokenType::Whitespace => {
                         context = LexContext::ExpectNothing;
                         Ok(LexItem::Whitespace)
@@ -274,9 +287,19 @@ pub fn lex(input: &str) -> ParseResult<Vec<(LexItem, u32, u32)>> {
                     }
                     TokenType::LCurly => {
                         context = LexContext::ExpectNothing;
+                        nesting_depth += 1;
+                        if nesting_depth > max_nesting {
+                            return Err(ParseError::new(
+                                ParseErrors::VaryExpressionStackDepthTooDeep,
+                            ));
+                        }
                         Ok(LexItem::LeftCurly)
                     }
-                    TokenType::RCurly => Ok(LexItem::RightCurly),
+                    TokenType::RCurly => {
+                        // if this underflows, the contract is invalid anyway
+                        nesting_depth = nesting_depth.saturating_sub(1);
+                        Ok(LexItem::RightCurly)
+                    }
                     TokenType::Variable => {
                         let value = get_value_or_err(current_slice, captures)?;
                         if value.contains("#") {
@@ -427,6 +450,13 @@ pub fn lex(input: &str) -> ParseResult<Vec<(LexItem, u32, u32)>> {
     }
 }
 
+pub fn lex(input: &str) -> ParseResult<Vec<(LexItem, u32, u32)>> {
+    inner_lex(
+        input,
+        AST_CALL_STACK_DEPTH_BUFFER + (MAX_CALL_STACK_DEPTH as u64) + 1,
+    )
+}
+
 fn unescape_ascii_chars(escaped_str: String, allow_unicode_escape: bool) -> ParseResult<String> {
     let mut unescaped_str = String::new();
     let mut chars = escaped_str.chars().into_iter();
@@ -689,19 +719,29 @@ pub fn parse_lexed(mut input: Vec<(LexItem, u32, u32)>) -> ParseResult<Vec<PreSy
 }
 
 pub fn parse(input: &str) -> ParseResult<Vec<PreSymbolicExpression>> {
-    let lexed = lex(input)?;
+    let lexed = inner_lex(
+        input,
+        AST_CALL_STACK_DEPTH_BUFFER + (MAX_CALL_STACK_DEPTH as u64) + 1,
+    )?;
+    parse_lexed(lexed)
+}
+
+pub fn parse_no_stack_limit(input: &str) -> ParseResult<Vec<PreSymbolicExpression>> {
+    let lexed = inner_lex(input, u64::MAX)?;
     parse_lexed(lexed)
 }
 
 #[cfg(test)]
 mod test {
     use crate::vm::ast;
     use crate::vm::ast::errors::{ParseError, ParseErrors};
+    use crate::vm::ast::stack_depth_checker::AST_CALL_STACK_DEPTH_BUFFER;
     use crate::vm::representations::{PreSymbolicExpression, PreSymbolicExpressionType};
     use crate::vm::types::TraitIdentifier;
     use crate::vm::types::{
         CharType, PrincipalData, QualifiedContractIdentifier, SequenceData, Value,
     };
+    use crate::vm::MAX_CALL_STACK_DEPTH;
 
     fn make_atom(
         x: &str,

clarity/src/vm/ast/stack_depth_checker.rs

@@ -18,6 +18,7 @@ use crate::vm::ast::errors::{ParseError, ParseErrors, ParseResult};
 use crate::vm::ast::types::{BuildASTPass, ContractAST};
 use crate::vm::representations::PreSymbolicExpression;
 use crate::vm::representations::PreSymbolicExpressionType::List;
+use crate::vm::representations::PreSymbolicExpressionType::Tuple;
 
 use crate::vm::MAX_CALL_STACK_DEPTH;
 
@@ -50,3 +51,29 @@ impl BuildASTPass for StackDepthChecker {
         check(&contract_ast.pre_expressions, 0)
     }
 }
+
+fn check_vary(args: &[PreSymbolicExpression], depth: u64) -> ParseResult<()> {
+    if depth >= (AST_CALL_STACK_DEPTH_BUFFER + MAX_CALL_STACK_DEPTH as u64) {
+        return Err(ParseErrors::VaryExpressionStackDepthTooDeep.into());
+    }
+    for expression in args.iter() {
+        match expression.pre_expr {
+            List(ref exprs) => check_vary(exprs, depth + 1),
+            Tuple(ref exprs) => check_vary(exprs, depth + 1),
+            _ => {
+                // Other symbolic expressions don't have depth
+                //  impacts.
+                Ok(())
+            }
+        }?;
+    }
+    Ok(())
+}
+
+pub struct VaryStackDepthChecker;
+
+impl BuildASTPass for VaryStackDepthChecker {
+    fn run_pass(contract_ast: &mut ContractAST) -> ParseResult<()> {
+        check_vary(&contract_ast.pre_expressions, 0)
+    }
+}

clarity/src/vm/clarity.rs

@@ -2,6 +2,7 @@ use crate::vm::analysis;
 use crate::vm::analysis::ContractAnalysis;
 use crate::vm::analysis::{AnalysisDatabase, CheckError, CheckErrors};
 use crate::vm::ast::errors::{ParseError, ParseErrors};
+use crate::vm::ast::ASTRules;
 use crate::vm::ast::ContractAST;
 use crate::vm::contexts::Environment;
 use crate::vm::contexts::{AssetMap, OwnedEnvironment};
@@ -164,9 +165,11 @@ pub trait TransactionConnection: ClarityConnection {
         &mut self,
         identifier: &QualifiedContractIdentifier,
         contract_content: &str,
+        ast_rules: ASTRules,
     ) -> Result<(ContractAST, ContractAnalysis), Error> {
         self.with_analysis_db(|db, mut cost_track| {
-            let ast_result = ast::build_ast(identifier, contract_content, &mut cost_track);
+            let ast_result =
+                ast::build_ast_with_rules(identifier, contract_content, &mut cost_track, ast_rules);
 
             let mut contract_ast = match ast_result {
                 Ok(x) => x,