Skip to content

Commit fdb77d3

Browse files
committed
feat: expose owned AST transforms and context-aware parameterization
Add iterative preorder traversal with parent/depth/child context, subtree skipping, early termination and explicit node/depth limits. Add arena-backed constructors and deep cloning that copy value and source text, reject cyclic or shared-node trees, and report allocation failures. Validate subtree replacement before mutation, preserve the replaced node's sibling position, and reject attached or overlapping replacement trees. Parameterize a copied AST and return ordered lexical literal mappings plus existing-bind positions. Preserve PostgreSQL bind numbers and allocate new numbers after the largest existing index; track MySQL anonymous positions in emitted order. Preserve output-column ordinals, boolean syntax constants, cast type subtrees and dialect-specific datetime precision arguments. Keep quoted user functions bindable rather than mistaking them for builtins. Reject incomplete ParseResults and unsupported or opaque contexts explicitly, including CTEs, function-shaped CAST, utility roots, SELECT INTO, PostgreSQL string-shaped aliases and MySQL comma-LIMIT containing existing anonymous binds. Literal-only comma-LIMIT follows normalized count/offset order. The API returns lexical values rather than decoded values or inferred types. Preserve quoted DML targets, assignment columns, conflict identifiers and RETURNING aliases so rewriting does not silently change identifier meaning. Register 26 focused tests covering lifetime independence, traversal bounds, arena exhaustion, invalid replacements, quoting, ordinals and both dialects' parameter mappings. All 26 pass under AddressSanitizer and UndefinedBehavior- Sanitizer without diagnostics; the complete tree passes 1,354 active tests with 37 backend-dependent skips. No AST-layout change or runtime dependency on PostgreSQL is introduced.
1 parent 0b73e2c commit fdb77d3

8 files changed

Lines changed: 1078 additions & 15 deletions

File tree

‎Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ TEST_SRCS = $(TEST_DIR)/test_main.cpp \
5454
$(TEST_DIR)/test_pg_query_features.cpp \
5555
$(TEST_DIR)/test_pg_utilities.cpp \
5656
$(TEST_DIR)/test_parse_all.cpp \
57+
$(TEST_DIR)/test_ast_utilities.cpp \
5758
$(TEST_DIR)/test_digest.cpp \
5859
$(TEST_DIR)/test_misc_stmts.cpp \
5960
$(TEST_DIR)/test_value.cpp \

‎include/sql_parser/ast_transform.h‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
#ifndef SQL_PARSER_AST_TRANSFORM_H
2+
#define SQL_PARSER_AST_TRANSFORM_H
3+
4+
#include "sql_parser/ast_walk.h"
5+
#include <unordered_set>
6+
7+
namespace sql_parser {
8+
9+
enum class AstError {
10+
None, LimitExceeded, AllocationFailure, InvalidTree, InvalidReplacement,
11+
NotFound, UnsupportedRoot, UnsupportedContext, InvalidPlaceholder, ParameterOverflow
12+
};
13+
14+
struct AstCloneResult {
15+
AstNode* ast = nullptr;
16+
AstError error = AstError::None;
17+
bool ok() const { return error == AstError::None; }
18+
};
19+
20+
// Unlike make_node(), copies value and source into the destination arena.
21+
inline AstNode* make_owned_node(Arena& arena, NodeType type, StringRef value = {},
22+
uint16_t flags = 0, StringRef source = {}) {
23+
if ((value.len && !value.ptr) || (source.len && !source.ptr)) return nullptr;
24+
StringRef owned_value;
25+
StringRef owned_source;
26+
if (value.len) {
27+
owned_value = arena.allocate_string(value.ptr, value.len);
28+
if (!owned_value.ptr) return nullptr;
29+
}
30+
if (source.len) {
31+
if (source.ptr == value.ptr && source.len == value.len) owned_source = owned_value;
32+
else owned_source = arena.allocate_string(source.ptr, source.len);
33+
if (!owned_source.ptr) return nullptr;
34+
}
35+
AstNode* node = make_node(arena, type, owned_value, flags);
36+
if (node) node->set_source(owned_source);
37+
return node;
38+
}
39+
40+
// Copies exactly one subtree, including all text. On failure ast is null;
41+
// partial allocations remain in the destination arena until its next reset.
42+
inline AstCloneResult clone_ast(const AstNode* root, Arena& arena,
43+
AstWalkLimits limits = {}) {
44+
AstCloneResult result;
45+
std::unordered_set<const AstNode*> seen;
46+
std::vector<AstNode*> ancestors;
47+
std::vector<AstNode*> last_child;
48+
auto walked = walk_ast(root, [&](const AstNode& node, const AstVisitContext& ctx) {
49+
if (!seen.insert(&node).second || (node.value_len && !node.value_ptr) ||
50+
(node.source_len && !node.source_ptr)) {
51+
result.error = AstError::InvalidTree;
52+
return AstVisitAction::Stop;
53+
}
54+
AstNode* copy = make_owned_node(arena, node.type, node.value(), node.flags, node.source());
55+
if (!copy) {
56+
result.error = AstError::AllocationFailure;
57+
return AstVisitAction::Stop;
58+
}
59+
ancestors.resize(ctx.depth + 1);
60+
last_child.resize(ctx.depth + 1);
61+
ancestors[ctx.depth] = copy;
62+
last_child[ctx.depth] = nullptr;
63+
if (!ctx.parent) result.ast = copy;
64+
else {
65+
if (last_child[ctx.depth - 1]) last_child[ctx.depth - 1]->next_sibling = copy;
66+
else ancestors[ctx.depth - 1]->first_child = copy;
67+
last_child[ctx.depth - 1] = copy;
68+
}
69+
return AstVisitAction::Continue;
70+
}, limits);
71+
if (walked.status == AstWalkStatus::LimitExceeded) result.error = AstError::LimitExceeded;
72+
if (!result.ok()) result.ast = nullptr;
73+
return result;
74+
}
75+
76+
// Replaces/removes target within root, preserving its sibling position. A
77+
// replacement must be a detached subtree, disjoint from the whole input tree.
78+
// root must be a standalone tree (no next_sibling). Validation completes
79+
// before any mutation; target is detached on success.
80+
inline AstError replace_ast_subtree(AstNode*& root, AstNode* target,
81+
AstNode* replacement, AstWalkLimits limits = {}) {
82+
if (!target) return AstError::NotFound;
83+
if (root && root->next_sibling) return AstError::InvalidTree;
84+
if (replacement && replacement->next_sibling) return AstError::InvalidReplacement;
85+
std::unordered_set<const AstNode*> original;
86+
AstNode* parent = nullptr;
87+
bool found = false;
88+
AstError error = AstError::None;
89+
auto walked = walk_ast(root, [&](const AstNode& node, const AstVisitContext& ctx) {
90+
if (!original.insert(&node).second) {
91+
error = AstError::InvalidTree;
92+
return AstVisitAction::Stop;
93+
}
94+
if (&node == target) {
95+
parent = const_cast<AstNode*>(ctx.parent);
96+
found = true;
97+
}
98+
return AstVisitAction::Continue;
99+
}, limits);
100+
if (walked.status == AstWalkStatus::LimitExceeded) return AstError::LimitExceeded;
101+
if (error != AstError::None) return error;
102+
if (!found) return AstError::NotFound;
103+
std::unordered_set<const AstNode*> incoming;
104+
walked = walk_ast(replacement, [&](const AstNode& node, const AstVisitContext&) {
105+
if (original.count(&node) || !incoming.insert(&node).second) {
106+
error = AstError::InvalidReplacement;
107+
return AstVisitAction::Stop;
108+
}
109+
return AstVisitAction::Continue;
110+
}, limits);
111+
if (walked.status == AstWalkStatus::LimitExceeded) return AstError::LimitExceeded;
112+
if (error != AstError::None) return error;
113+
AstNode** slot = parent ? &parent->first_child : &root;
114+
while (*slot != target) slot = &(*slot)->next_sibling;
115+
if (replacement) replacement->next_sibling = target->next_sibling;
116+
*slot = replacement ? replacement : target->next_sibling;
117+
target->next_sibling = nullptr;
118+
return AstError::None;
119+
}
120+
121+
} // namespace sql_parser
122+
123+
#endif // SQL_PARSER_AST_TRANSFORM_H

‎include/sql_parser/ast_walk.h‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
#ifndef SQL_PARSER_AST_WALK_H
2+
#define SQL_PARSER_AST_WALK_H
3+
4+
#include "sql_parser/ast.h"
5+
#include <cstddef>
6+
#include <vector>
7+
8+
namespace sql_parser {
9+
10+
enum class AstVisitAction { Continue, SkipChildren, Stop };
11+
enum class AstWalkStatus { Completed, Stopped, LimitExceeded };
12+
13+
struct AstWalkLimits {
14+
size_t max_nodes = 1000000;
15+
size_t max_depth = 4096;
16+
};
17+
18+
struct AstVisitContext {
19+
const AstNode* parent;
20+
size_t depth; // Root is depth zero.
21+
size_t child_index; // Zero-based index within the parent's children.
22+
};
23+
24+
struct AstWalkResult {
25+
AstWalkStatus status = AstWalkStatus::Completed;
26+
size_t visited = 0;
27+
};
28+
29+
// Preorder over one subtree, excluding root->next_sibling. The visitor must
30+
// not change links during traversal. Bounds also terminate cyclic input.
31+
// Uses O(depth) temporary storage and never recurses on the C++ stack.
32+
template <typename Visitor>
33+
AstWalkResult walk_ast(const AstNode* root, Visitor&& visitor,
34+
AstWalkLimits limits = {}) {
35+
struct Pending { const AstNode* node; AstVisitContext context; };
36+
std::vector<Pending> pending;
37+
if (root) pending.push_back({root, {nullptr, 0, 0}});
38+
AstWalkResult result;
39+
while (!pending.empty()) {
40+
const Pending current = pending.back();
41+
pending.pop_back();
42+
if (result.visited >= limits.max_nodes || current.context.depth > limits.max_depth) {
43+
result.status = AstWalkStatus::LimitExceeded;
44+
return result;
45+
}
46+
++result.visited;
47+
const AstVisitAction action = visitor(*current.node, current.context);
48+
if (action == AstVisitAction::Stop) {
49+
result.status = AstWalkStatus::Stopped;
50+
return result;
51+
}
52+
if (current.context.parent && current.node->next_sibling) {
53+
pending.push_back({current.node->next_sibling,
54+
{current.context.parent, current.context.depth, current.context.child_index + 1}});
55+
}
56+
if (action != AstVisitAction::SkipChildren && current.node->first_child) {
57+
pending.push_back({current.node->first_child,
58+
{current.node, current.context.depth + 1, 0}});
59+
}
60+
}
61+
return result;
62+
}
63+
64+
} // namespace sql_parser
65+
66+
#endif // SQL_PARSER_AST_WALK_H

‎include/sql_parser/delete_parser.h‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,13 @@ class DeleteParser {
3939
}
4040

4141
private:
42+
AstNode* make_identifier(const Token& token, NodeType type = NodeType::NODE_IDENTIFIER) {
43+
AstNode* node = make_node_from_token(arena_, type, token);
44+
if (node && token.type == TokenType::TK_IDENTIFIER && token.source.ptr != token.text.ptr)
45+
node->flags |= FLAG_IDENT_DELIMITED;
46+
return node;
47+
}
48+
4249
Tokenizer<D>& tok_;
4350
Arena& arena_;
4451
ExpressionParser<D> expr_parser_;
@@ -241,11 +248,11 @@ class DeleteParser {
241248
tok_.skip();
242249
Token table_name = tok_.next_token();
243250
AstNode* qname = make_node(arena_, NodeType::NODE_QUALIFIED_NAME);
244-
qname->add_child(make_node(arena_, NodeType::NODE_IDENTIFIER, name.text));
245-
qname->add_child(make_node(arena_, NodeType::NODE_IDENTIFIER, table_name.text));
251+
qname->add_child(make_identifier(name));
252+
qname->add_child(make_identifier(table_name));
246253
ref->add_child(qname);
247254
} else {
248-
ref->add_child(make_node(arena_, NodeType::NODE_IDENTIFIER, name.text));
255+
ref->add_child(make_identifier(name));
249256
}
250257

251258
return ref;
@@ -337,7 +344,8 @@ class DeleteParser {
337344
if (next.type == TokenType::TK_AS) {
338345
tok_.skip();
339346
Token alias_name = tok_.next_token();
340-
ret->add_child(make_node(arena_, NodeType::NODE_ALIAS, alias_name.text));
347+
ret->add_child(make_node(arena_, NodeType::NODE_ALIAS,
348+
alias_name.source.empty() ? alias_name.text : alias_name.source));
341349
}
342350

343351
if (tok_.peek().type == TokenType::TK_COMMA) {

‎include/sql_parser/insert_parser.h‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,13 @@ class InsertParser {
116116
}
117117

118118
private:
119+
AstNode* make_identifier(const Token& token, NodeType type = NodeType::NODE_IDENTIFIER) {
120+
AstNode* node = make_node_from_token(arena_, type, token);
121+
if (node && token.type == TokenType::TK_IDENTIFIER && token.source.ptr != token.text.ptr)
122+
node->flags |= FLAG_IDENT_DELIMITED;
123+
return node;
124+
}
125+
119126
Tokenizer<D>& tok_;
120127
Arena& arena_;
121128
ExpressionParser<D> expr_parser_;
@@ -161,7 +168,7 @@ class InsertParser {
161168
tok_.skip(); // consume (
162169
while (true) {
163170
Token col = tok_.next_token();
164-
cols->add_child(make_node(arena_, NodeType::NODE_IDENTIFIER, col.text));
171+
cols->add_child(make_identifier(col));
165172
if (tok_.peek().type == TokenType::TK_COMMA) {
166173
tok_.skip();
167174
} else {
@@ -243,11 +250,11 @@ class InsertParser {
243250
tok_.skip();
244251
Token actual_col = tok_.next_token();
245252
AstNode* qname = make_node(arena_, NodeType::NODE_QUALIFIED_NAME);
246-
qname->add_child(make_node(arena_, NodeType::NODE_IDENTIFIER, col.text));
247-
qname->add_child(make_node(arena_, NodeType::NODE_IDENTIFIER, actual_col.text));
253+
qname->add_child(make_identifier(col));
254+
qname->add_child(make_identifier(actual_col));
248255
item->add_child(qname);
249256
} else {
250-
item->add_child(make_node(arena_, NodeType::NODE_COLUMN_REF, col.text));
257+
item->add_child(make_identifier(col, NodeType::NODE_COLUMN_REF));
251258
}
252259

253260
// = sign
@@ -333,7 +340,7 @@ class InsertParser {
333340
tok_.skip(); // consume (
334341
while (true) {
335342
Token col = tok_.next_token();
336-
target->add_child(make_node(arena_, NodeType::NODE_IDENTIFIER, col.text));
343+
target->add_child(make_identifier(col));
337344
if (tok_.peek().type == TokenType::TK_COMMA) {
338345
tok_.skip();
339346
} else {
@@ -356,7 +363,7 @@ class InsertParser {
356363
tok_.skip(); // CONSTRAINT
357364
}
358365
Token name = tok_.next_token();
359-
target->add_child(make_node(arena_, NodeType::NODE_IDENTIFIER, name.text));
366+
target->add_child(make_identifier(name));
360367

361368
return target;
362369
}
@@ -422,7 +429,8 @@ class InsertParser {
422429
if (next.type == TokenType::TK_AS) {
423430
tok_.skip();
424431
Token alias_name = tok_.next_token();
425-
ret->add_child(make_node(arena_, NodeType::NODE_ALIAS, alias_name.text));
432+
ret->add_child(make_node(arena_, NodeType::NODE_ALIAS,
433+
alias_name.source.empty() ? alias_name.text : alias_name.source));
426434
}
427435

428436
if (tok_.peek().type == TokenType::TK_COMMA) {

0 commit comments

Comments
 (0)