Skip to content

Commit 0b73e2c

Browse files
renecannaorenecannao
committed
feat: add PostgreSQL query extensions and structured batch parsing
Represent DISTINCT ON, aggregate FILTER, LATERAL SELECT/function references, named windows and ROWS/RANGE/GROUPS frames as structured AST nodes. Preserve frame boundaries, exclusions, aliases and source quoting during emission. Reject these constructs in the local planner where execution is unsupported. Restore emission of window nodes that previously disappeared from SQL and digests; affected digest strings and hashes consequently change. Add parse_all with ordered per-statement results and byte spans. Reset the arena once per batch so earlier ASTs remain valid while later statements are parsed. Skip empty statements, retain failures, continue at available lexical boundaries, and prevent unknown commands from making batch.ok true. The API frames SQL text, not inline COPY data or procedural bodies. Build transaction/savepoint/prepared-transaction ASTs and bounded COPY ASTs for relation or supported query sources, directions, endpoints, modern options and WHERE clauses. Add COPY and RELEASE_SAVEPOINT classifications and update the oracle mapping. Use opt-in complete-operand parsing inside COPY queries without globally tightening existing permissive parser paths. Preserve PostgreSQL doubled-quote identifiers, ordinary versus E-string boundaries, and complete dollar-quote delimiters. Reject truncated dollar strings. Preserve quoted function names and normalize MySQL comma-LIMIT children into count/offset order, recording their original syntax for consumers that must handle anonymous bind positions. Add focused round-trip, AST mutation, malformed-input, lexical-boundary, retained-lifetime and planner-rejection tests and register them in the Makefile. The final combined tree passes all 1,354 active C++ tests and builds the corpus harness; 37 backend integration tests require unavailable servers. LATERAL WITH/VALUES/compound operands and interval frame offsets remain outside this implementation. Compatibility fixtures are refreshed separately after all parser changes in this series.
1 parent cf67898 commit 0b73e2c

18 files changed

Lines changed: 1357 additions & 89 deletions

‎Makefile‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,9 @@ TEST_SRCS = $(TEST_DIR)/test_main.cpp \
5151
$(TEST_DIR)/test_update.cpp \
5252
$(TEST_DIR)/test_delete.cpp \
5353
$(TEST_DIR)/test_compound.cpp \
54+
$(TEST_DIR)/test_pg_query_features.cpp \
55+
$(TEST_DIR)/test_pg_utilities.cpp \
56+
$(TEST_DIR)/test_parse_all.cpp \
5457
$(TEST_DIR)/test_digest.cpp \
5558
$(TEST_DIR)/test_misc_stmts.cpp \
5659
$(TEST_DIR)/test_value.cpp \

‎include/sql_engine/plan_builder.h‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ class PlanBuilder {
3838
// Build a logical plan from a parsed statement AST.
3939
// Returns nullptr for non-SELECT statements.
4040
PlanNode* build(const sql_parser::AstNode* stmt_ast) {
41-
if (!stmt_ast) return nullptr;
41+
if (!stmt_ast || has_unsupported_query_feature(stmt_ast)) return nullptr;
4242

4343
if (stmt_ast->type == sql_parser::NodeType::NODE_SELECT_STMT) {
4444
return build_select(stmt_ast);
@@ -56,6 +56,29 @@ class PlanBuilder {
5656
const Catalog& catalog_;
5757
sql_parser::Arena& arena_;
5858

59+
static bool has_unsupported_query_feature(const sql_parser::AstNode* node) {
60+
using sql_parser::NodeType;
61+
switch (node->type) {
62+
case NodeType::NODE_DISTINCT_ON:
63+
case NodeType::NODE_AGGREGATE_FILTER:
64+
case NodeType::NODE_LATERAL:
65+
case NodeType::NODE_WINDOW_CLAUSE:
66+
case NodeType::NODE_WINDOW_REFERENCE:
67+
case NodeType::NODE_WINDOW_FRAME:
68+
return true;
69+
case NodeType::NODE_FUNCTION_CALL:
70+
if (node->flags & sql_parser::FLAG_FUNCTION_TABLE) return true;
71+
break;
72+
case NodeType::NODE_ORDER_BY_ITEM:
73+
if (node->flags & sql_parser::FLAG_ORDER_NULLS) return true;
74+
break;
75+
default: break;
76+
}
77+
for (const auto* child = node->first_child; child; child = child->next_sibling)
78+
if (has_unsupported_query_feature(child)) return true;
79+
return false;
80+
}
81+
5982
// Helper: find first child of given type
6083
static const sql_parser::AstNode* find_child(const sql_parser::AstNode* node,
6184
sql_parser::NodeType type) {

‎include/sql_parser/common.h‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,10 @@ static constexpr uint16_t FLAG_SET_OP_ALL = 0x01;
6262
static constexpr uint16_t FLAG_QUERY_PARENTHESIZED = 0x01;
6363
static constexpr uint16_t FLAG_TABLE_ONLY = 0x01;
6464
static constexpr uint16_t FLAG_TABLE_INHERIT = 0x02;
65+
static constexpr uint16_t FLAG_FUNCTION_TABLE = 0x01;
66+
static constexpr uint16_t FLAG_WINDOW_BETWEEN = 0x01;
67+
static constexpr uint16_t FLAG_ORDER_NULLS = 0x01;
68+
static constexpr uint16_t FLAG_LIMIT_COMMA = 0x01;
6569

6670
// -- Flags for NODE_IDENTIFIER / NODE_COLUMN_REF --
6771
// Set when the identifier was source-delimited (backtick `name` for MySQL,
@@ -104,6 +108,8 @@ enum class StmtType : uint8_t {
104108
DESCRIBE,
105109
CALL,
106110
DO_STMT,
111+
COPY,
112+
RELEASE_SAVEPOINT,
107113
};
108114

109115
// -- AST node types --
@@ -242,6 +248,20 @@ enum class NodeType : uint16_t {
242248
NODE_LITERAL_HEX,
243249
NODE_LITERAL_BIT,
244250
NODE_TABLE_QUERY, // PostgreSQL TABLE [ONLY] relation [*]
251+
NODE_TRANSACTION_STMT,
252+
NODE_TRANSACTION_OPTION,
253+
NODE_COPY_STMT,
254+
NODE_COPY_OPTION,
255+
NODE_COPY_ENDPOINT,
256+
NODE_DISTINCT_ON, // expression children, inside SELECT_OPTIONS
257+
NODE_AGGREGATE_FILTER, // function, predicate
258+
NODE_LATERAL, // child TABLE_REF
259+
NODE_WINDOW_CLAUSE, // WINDOW_DEFINITION children
260+
NODE_WINDOW_DEFINITION, // value=name, child WINDOW_SPEC
261+
NODE_WINDOW_REFERENCE, // value=name, standalone OVER or inside spec
262+
NODE_WINDOW_FRAME, // value=ROWS/RANGE/GROUPS; bounds then exclusion
263+
NODE_WINDOW_BOUND, // value=PRECEDING/FOLLOWING/CURRENT ROW/...; offset child
264+
NODE_WINDOW_EXCLUSION, // value=CURRENT ROW/GROUP/TIES/NO OTHERS
245265
};
246266

247267
} // namespace sql_parser

‎include/sql_parser/compound_query_parser.h‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,10 @@ namespace sql_parser {
88
template <Dialect D>
99
class CompoundQueryParser {
1010
public:
11-
CompoundQueryParser(Tokenizer<D>& tokenizer, Arena& arena)
12-
: tok_(tokenizer), arena_(arena), expr_parser_(tokenizer, arena) {}
11+
CompoundQueryParser(Tokenizer<D>& tokenizer, Arena& arena,
12+
bool require_complete_operands = false)
13+
: tok_(tokenizer), arena_(arena), expr_parser_(tokenizer, arena, require_complete_operands),
14+
require_complete_operands_(require_complete_operands) {}
1315

1416
void set_subquery_callback(SubqueryParseCallback<D> cb) {
1517
subquery_cb_ = cb;
@@ -19,7 +21,8 @@ class CompoundQueryParser {
1921
// The classifier normally consumes SELECT. Pass the consumed keyword for
2022
// TABLE/VALUES/'(', or TK_EOF to start at an unconsumed query operand.
2123
AstNode* parse(TokenType first = TokenType::TK_SELECT) {
22-
const bool require_operands = first == TokenType::TK_VALUES || first == TokenType::TK_TABLE;
24+
const bool require_operands = require_complete_operands_ ||
25+
first == TokenType::TK_VALUES || first == TokenType::TK_TABLE;
2326
AstNode* result = parse_compound_expr(0, first);
2427
if (!result) return nullptr;
2528
if (result->type == NodeType::NODE_SET_OPERATION ||
@@ -62,6 +65,7 @@ class CompoundQueryParser {
6265
Arena& arena_;
6366
ExpressionParser<D> expr_parser_;
6467
SubqueryParseCallback<D> subquery_cb_ = nullptr;
68+
bool require_complete_operands_;
6569

6670
static int get_set_op_precedence(TokenType type) {
6771
switch (type) {
@@ -111,7 +115,7 @@ class CompoundQueryParser {
111115
return group;
112116
}
113117
if (first == TokenType::TK_SELECT) {
114-
SelectParser<D> select(tok_, arena_, true);
118+
SelectParser<D> select(tok_, arena_, true, require_complete_operands_);
115119
select.set_subquery_callback(subquery_cb_);
116120
return select.parse();
117121
}
@@ -239,7 +243,9 @@ class CompoundQueryParser {
239243
tok_.skip();
240244
AstNode* count = expressions.parse();
241245
if (!count) return nullptr;
242-
limit->add_child(count);
246+
limit->flags |= FLAG_LIMIT_COMMA;
247+
limit->first_child = count;
248+
count->next_sibling = first;
243249
}
244250

245251
return limit;

‎include/sql_parser/emitter.h‎

Lines changed: 150 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ class Emitter {
3434

3535
void emit_node(const AstNode* node) {
3636
switch (node->type) {
37+
case NodeType::NODE_TRANSACTION_STMT: emit_transaction_stmt(node); break;
38+
case NodeType::NODE_COPY_STMT: emit_copy_stmt(node); break;
3739
// ---- SET statement ----
3840
case NodeType::NODE_SET_STMT: emit_set_stmt(node); break;
3941
case NodeType::NODE_SET_NAMES: emit_set_names(node); break;
@@ -58,6 +60,35 @@ class Emitter {
5860
case NodeType::NODE_LOCKING_CLAUSE: emit_locking(node); break;
5961
case NodeType::NODE_INTO_CLAUSE: emit_into(node); break;
6062

63+
case NodeType::NODE_DISTINCT_ON:
64+
sb_.append("DISTINCT ON ("); emit_list(node, ", "); sb_.append_char(')'); break;
65+
case NodeType::NODE_AGGREGATE_FILTER:
66+
emit_node(node->first_child); sb_.append(" FILTER (WHERE ");
67+
emit_node(node->first_child->next_sibling); sb_.append_char(')'); break;
68+
case NodeType::NODE_LATERAL:
69+
sb_.append("LATERAL "); emit_node(node->first_child); break;
70+
case NodeType::NODE_WINDOW_FUNCTION:
71+
emit_node(node->first_child); sb_.append(" OVER ");
72+
emit_node(node->first_child->next_sibling); break;
73+
case NodeType::NODE_WINDOW_SPEC:
74+
sb_.append_char('('); emit_list(node, " "); sb_.append_char(')'); break;
75+
case NodeType::NODE_WINDOW_PARTITION:
76+
sb_.append("PARTITION BY "); emit_list(node, ", "); break;
77+
case NodeType::NODE_WINDOW_ORDER:
78+
sb_.append("ORDER BY "); emit_list(node, ", "); break;
79+
case NodeType::NODE_WINDOW_CLAUSE:
80+
sb_.append(" WINDOW "); emit_list(node, ", "); break;
81+
case NodeType::NODE_WINDOW_DEFINITION:
82+
emit_value(node); sb_.append(" AS "); emit_node(node->first_child); break;
83+
case NodeType::NODE_WINDOW_REFERENCE:
84+
emit_value(node); break;
85+
case NodeType::NODE_WINDOW_FRAME: emit_window_frame(node); break;
86+
case NodeType::NODE_WINDOW_BOUND:
87+
if (node->first_child) { emit_node(node->first_child); sb_.append_char(' '); }
88+
emit_value(node); break;
89+
case NodeType::NODE_WINDOW_EXCLUSION:
90+
sb_.append("EXCLUDE "); emit_value(node); break;
91+
6192
// ---- INSERT statement ----
6293
case NodeType::NODE_INSERT_STMT: emit_insert_stmt(node); break;
6394
case NodeType::NODE_INSERT_COLUMNS: emit_insert_columns(node); break;
@@ -138,10 +169,11 @@ class Emitter {
138169
case NodeType::NODE_LITERAL_BIT:
139170
if (mode_ == EmitMode::DIGEST) { sb_.append_char('?'); break; }
140171
emit_value(node); break;
141-
case NodeType::NODE_LITERAL_NULL:
142172
case NodeType::NODE_COLUMN_REF:
143-
case NodeType::NODE_ASTERISK:
144173
case NodeType::NODE_IDENTIFIER:
174+
emit_identifier(node); break;
175+
case NodeType::NODE_LITERAL_NULL:
176+
case NodeType::NODE_ASTERISK:
145177
emit_value(node); break;
146178

147179
case NodeType::NODE_LITERAL_STRING:
@@ -153,10 +185,102 @@ class Emitter {
153185
}
154186
}
155187

188+
void emit_list(const AstNode* node, const char* separator) {
189+
for (const AstNode* child = node->first_child; child; child = child->next_sibling) {
190+
if (child != node->first_child) sb_.append(separator);
191+
emit_node(child);
192+
}
193+
}
194+
195+
void emit_window_frame(const AstNode* node) {
196+
emit_value(node);
197+
sb_.append(node->flags & FLAG_WINDOW_BETWEEN ? " BETWEEN " : " ");
198+
const AstNode* bound = node->first_child;
199+
emit_node(bound);
200+
bound = bound ? bound->next_sibling : nullptr;
201+
if (node->flags & FLAG_WINDOW_BETWEEN) {
202+
sb_.append(" AND "); emit_node(bound);
203+
bound = bound ? bound->next_sibling : nullptr;
204+
}
205+
if (bound) { sb_.append_char(' '); emit_node(bound); }
206+
}
207+
208+
void emit_identifier(const AstNode* node) {
209+
if (!(node->flags & FLAG_IDENT_DELIMITED)) { emit_value(node); return; }
210+
if (!node->source().empty()) {
211+
sb_.append(node->source_ptr, node->source_len);
212+
return;
213+
}
214+
const char quote = D == Dialect::PostgreSQL ? '"' : '`';
215+
sb_.append_char(quote);
216+
for (uint32_t i = 0; i < node->value_len; ++i) {
217+
sb_.append_char(node->value_ptr[i]);
218+
if (node->value_ptr[i] == quote) sb_.append_char(quote);
219+
}
220+
sb_.append_char(quote);
221+
}
222+
156223
void emit_value(const AstNode* node) {
157224
sb_.append(node->value_ptr, node->value_len);
158225
}
159226

227+
void emit_utility_value(const AstNode* node) {
228+
// Utility values are syntax constants. Keep source string quoting,
229+
// including E'...' and dollar quotes, even in digest mode.
230+
if (node->type == NodeType::NODE_LITERAL_STRING && !node->source().empty()) {
231+
sb_.append(node->source_ptr, node->source_len);
232+
} else emit_node(node);
233+
}
234+
235+
void emit_transaction_stmt(const AstNode* node) {
236+
emit_value(node);
237+
for (const AstNode* child = node->first_child; child; child = child->next_sibling) {
238+
sb_.append_char(' ');
239+
emit_utility_value(child);
240+
}
241+
}
242+
243+
void emit_copy_stmt(const AstNode* node) {
244+
sb_.append("COPY ", 5);
245+
const AstNode* child = node->first_child;
246+
if (!child) return;
247+
emit_node(child);
248+
child = child->next_sibling;
249+
if (child && child->type == NodeType::NODE_INSERT_COLUMNS) {
250+
sb_.append_char(' ');
251+
emit_node(child);
252+
child = child->next_sibling;
253+
}
254+
sb_.append_char(' ');
255+
emit_value(node);
256+
sb_.append_char(' ');
257+
if (!child || child->type != NodeType::NODE_COPY_ENDPOINT) return;
258+
if (!child->value().equals_ci("FILE", 4)) emit_value(child);
259+
if (child->first_child) {
260+
if (child->value().equals_ci("PROGRAM", 7)) sb_.append_char(' ');
261+
emit_utility_value(child->first_child);
262+
}
263+
child = child->next_sibling;
264+
if (child && child->type == NodeType::NODE_COPY_OPTION) {
265+
sb_.append(" WITH (", 7);
266+
bool first = true;
267+
while (child && child->type == NodeType::NODE_COPY_OPTION) {
268+
if (!first) sb_.append(", ", 2);
269+
emit_value(child);
270+
if (child->first_child) {
271+
sb_.append_char(' ');
272+
emit_utility_value(child->first_child);
273+
}
274+
first = false;
275+
child = child->next_sibling;
276+
}
277+
sb_.append_char(')');
278+
}
279+
if (child && child->type == NodeType::NODE_WHERE_CLAUSE) {
280+
emit_node(child);
281+
}
282+
}
283+
160284
void emit_user_variable(const AstNode* node) {
161285
if (mode_ == EmitMode::DIGEST) {
162286
StringRef source = node->source();
@@ -182,6 +306,13 @@ class Emitter {
182306
}
183307

184308
void emit_string_literal(const AstNode* node) {
309+
if constexpr (D == Dialect::PostgreSQL) {
310+
if (!node->source().empty() &&
311+
(node->source_ptr[0] == 'E' || node->source_ptr[0] == 'e' || node->source_ptr[0] == '$')) {
312+
sb_.append(node->source_ptr, node->source_len);
313+
return;
314+
}
315+
}
185316
sb_.append_char('\'');
186317
sb_.append(node->value_ptr, node->value_len);
187318
sb_.append_char('\'');
@@ -356,6 +487,9 @@ class Emitter {
356487
if (mode_ == EmitMode::DIGEST) return; // skip aliases in digest mode
357488
sb_.append(" AS ");
358489
emit_value(node);
490+
if (node->first_child) {
491+
sb_.append_char('('); emit_list(node, ", "); sb_.append_char(')');
492+
}
359493
}
360494

361495
void emit_qualified_name(const AstNode* node) {
@@ -430,7 +564,7 @@ class Emitter {
430564
const AstNode* expr = node->first_child;
431565
if (expr) emit_node(expr);
432566
const AstNode* dir = expr ? expr->next_sibling : nullptr;
433-
if (dir) {
567+
for (; dir; dir = dir->next_sibling) {
434568
sb_.append_char(' ');
435569
emit_node(dir);
436570
}
@@ -1129,10 +1263,14 @@ class Emitter {
11291263
}
11301264

11311265
void emit_function_call(const AstNode* node) {
1132-
emit_value(node);
1266+
const AstNode* arg = node->first_child;
1267+
if (node->flags & FLAG_FUNCTION_TABLE) {
1268+
emit_node(arg);
1269+
arg = arg ? arg->next_sibling : nullptr;
1270+
} else emit_value(node);
11331271
sb_.append_char('(');
11341272
bool first = true;
1135-
for (const AstNode* arg = node->first_child; arg; arg = arg->next_sibling) {
1273+
for (; arg; arg = arg->next_sibling) {
11361274
if (!first) sb_.append(", ");
11371275
first = false;
11381276
emit_node(arg);
@@ -1294,9 +1432,13 @@ class Emitter {
12941432
void emit_field_access(const AstNode* node) {
12951433
const AstNode* expr = node->first_child;
12961434
const AstNode* field = expr ? expr->next_sibling : nullptr;
1297-
sb_.append_char('(');
1298-
if (expr) emit_node(expr);
1299-
sb_.append(").");
1435+
if (expr && expr->type == NodeType::NODE_EXPRESSION) emit_node(expr);
1436+
else {
1437+
sb_.append_char('(');
1438+
if (expr) emit_node(expr);
1439+
sb_.append_char(')');
1440+
}
1441+
sb_.append_char('.');
13001442
if (field) emit_node(field);
13011443
}
13021444

0 commit comments

Comments
 (0)