diff --git a/go.mod b/go.mod index 53410cd3..c26c5a30 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/spf13/viper v1.10.1 - github.com/stackql-labs/omnisdk v0.1.3-beta03 + github.com/stackql-labs/omnisdk v0.1.4-alpha05 github.com/stackql/any-sdk v0.6.0-alpha01 github.com/stackql/go-suffix-map v0.0.1-alpha01 github.com/stackql/psql-wire v0.1.3-beta01 diff --git a/go.sum b/go.sum index be73baf8..661bcd3d 100644 --- a/go.sum +++ b/go.sum @@ -368,8 +368,8 @@ github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk= github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU= github.com/spiffe/go-spiffe/v2 v2.7.0 h1:uXe1MflJoHw58wAUvxVlcM7WpKtijWG7I1UidcGh6g4= github.com/spiffe/go-spiffe/v2 v2.7.0/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= -github.com/stackql-labs/omnisdk v0.1.3-beta03 h1:VAzd8cix45oiRLbexCEBfOGHrYsnmsvqjRcBqroW96o= -github.com/stackql-labs/omnisdk v0.1.3-beta03/go.mod h1:WzvNj/bVv53yGFsVJpYWCJC1xAEdmQSFJl9eVpkRpCY= +github.com/stackql-labs/omnisdk v0.1.4-alpha05 h1:Z22XGi99mjKPJEAAYv3qkW1J0iqqFRd9OANK+9cw4Xs= +github.com/stackql-labs/omnisdk v0.1.4-alpha05/go.mod h1:2P2vFi+c/wEeQ3kFgIt0QuIlefpuzIoTm6bPJAiLAvQ= github.com/stackql/any-sdk v0.6.0-alpha01 h1:mqy0bmZ1wghr7mUgLyaexoRU6rsUZCWXU9Mf60fC8dg= github.com/stackql/any-sdk v0.6.0-alpha01/go.mod h1:ahkRgwHHEn7RTmSfKrshPVk6i4X1BdHknSeBflSyRac= github.com/stackql/go-suffix-map v0.0.1-alpha01 h1:TDUDS8bySu41Oo9p0eniUeCm43mnRM6zFEd6j6VUaz8= diff --git a/internal/stackql/intrinsic/dialect.go b/internal/stackql/intrinsic/dialect.go new file mode 100644 index 00000000..38cb34e6 --- /dev/null +++ b/internal/stackql/intrinsic/dialect.go @@ -0,0 +1,27 @@ +package intrinsic + +import ( + "fmt" + + "github.com/stackql-labs/omnisdk/pkg/sqlfn" + "github.com/stackql/any-sdk/pkg/constants" + "github.com/stackql/any-sdk/pkg/dto" +) + +// backendDialect is the SQL dialect of the session's backend, which decides what each function a +// query calls means: the embedded SQLite's, or Postgres's. It is read from the backend's +// configuration; a backend omnisdk has no catalogue for is refused. +func backendDialect(ctx queryContext) (sqlfn.Dialect, error) { + cfg, err := dto.GetSQLBackendCfg(ctx.GetRuntimeContext().SQLBackendCfgRaw) + if err != nil { + return "", err + } + switch cfg.GetSQLDialect() { + case constants.SQLDialectSQLite3: + return sqlfn.SQLite, nil + case constants.SQLDialectPostgres: + return sqlfn.Postgres, nil + } + return "", fmt.Errorf("SQL backend %q has no omnisdk function catalogue (want %q or %q)", + cfg.GetSQLDialect(), constants.SQLDialectSQLite3, constants.SQLDialectPostgres) +} diff --git a/internal/stackql/intrinsic/dialect_test.go b/internal/stackql/intrinsic/dialect_test.go new file mode 100644 index 00000000..b5ebc164 --- /dev/null +++ b/internal/stackql/intrinsic/dialect_test.go @@ -0,0 +1,37 @@ +package intrinsic //nolint:testpackage // tests the unexported dialect selection + +import ( + "testing" + + "github.com/stackql-labs/omnisdk/pkg/sqlfn" + "github.com/stackql/any-sdk/pkg/dto" +) + +// backendCtx is a queryContext whose only state is the SQL backend's configuration. +type backendCtx struct { + queryContext + raw string +} + +func (c backendCtx) GetRuntimeContext() dto.RuntimeCtx { + return dto.RuntimeCtx{SQLBackendCfgRaw: c.raw} +} + +// The dialect is the backend's: the default and sqlite3 are SQLite, postgres is Postgres, and a +// backend with no catalogue is refused rather than given another's functions. +func TestBackendDialect(t *testing.T) { + cases := map[string]sqlfn.Dialect{ + ``: sqlfn.SQLite, + `{"sqlDialect": "sqlite3"}`: sqlfn.SQLite, + `{"dbEngine": "postgres_tcp", "sqlDialect": "postgres", "dsn": "postgres://u:p@h:5432/db"}`: sqlfn.Postgres, + } + for raw, want := range cases { + got, err := backendDialect(backendCtx{raw: raw}) + if err != nil || got != want { + t.Errorf("%s: got %q, %v; want %q", raw, got, err, want) + } + } + if _, err := backendDialect(backendCtx{raw: `{"sqlDialect": "snowflake"}`}); err == nil { + t.Error("snowflake: want a refusal, got none") + } +} diff --git a/internal/stackql/intrinsic/doc.go b/internal/stackql/intrinsic/doc.go index e06141c9..b75df734 100644 --- a/internal/stackql/intrinsic/doc.go +++ b/internal/stackql/intrinsic/doc.go @@ -166,7 +166,11 @@ func docSelectFunc( if previewCfg.getStagingEnabled() && needsStaging(node) { return stagedSelectFunc(ctx, node, currentProvider), true } - translated, err := translateSelect(node, currentProvider) + dialect, err := backendDialect(ctx) + if err != nil { + return refuse(err), true + } + translated, err := translateSelect(node, currentProvider, dialect) if err != nil { return refuse(err), true } @@ -190,7 +194,11 @@ func docMutationFunc( } else if err != nil { return refuse(err), true } - translated, err := translateMutation(stmt, currentProvider) + dialect, err := backendDialect(ctx) + if err != nil { + return refuse(err), true + } + translated, err := translateMutation(stmt, currentProvider, dialect) if err != nil { return refuse(err), true } @@ -224,14 +232,21 @@ func openDocQuery(ctx queryContext, translated docQuery) (omnisdk.Rows, string, } else { relation = q.From()[0].Resource().Alias() } - res, resolveErr := omnisdk.Resolve(q, tables) + dialect, dialectErr := backendDialect(ctx) + if dialectErr != nil { + return nil, "", dialectErr + } + res, resolveErr := omnisdk.ResolveIn(q, tables, dialect) if resolveErr != nil { return nil, "", resolveErr } // omnisdk takes one credential per run: the first relation's cloud - a // mutation's target - leaving the rest to the canonical environment // variables. - args := previewArgs(ctx, translated.getBundles()[0], res.Params()) + args, argsErr := previewArgs(ctx, translated.getBundles()[0], res.Params()) + if argsErr != nil { + return nil, "", argsErr + } args.Tuning.Limit = translated.getLimit() plan, planErr := omnisdk.NewGraphSelectQuery(registry, res.Graph(), args) if planErr != nil { diff --git a/internal/stackql/intrinsic/dynamic.go b/internal/stackql/intrinsic/dynamic.go index 47d6a847..616d6965 100644 --- a/internal/stackql/intrinsic/dynamic.go +++ b/internal/stackql/intrinsic/dynamic.go @@ -83,8 +83,11 @@ func dynamicSelectFunc( if err != nil { return internaldto.NewErroneousExecutorOutput(err) } - plan, planErr := omnisdk.NewGraphQuery( - registryRoot(ctx), graph, previewArgs(ctx, graphCloud(graph.Addresses()), params)) + args, argsErr := previewArgs(ctx, graphCloud(graph.Addresses()), params) + if argsErr != nil { + return internaldto.NewErroneousExecutorOutput(argsErr) + } + plan, planErr := omnisdk.NewGraphQuery(registryRoot(ctx), graph, args) if planErr != nil { return internaldto.NewErroneousExecutorOutput(planErr) } diff --git a/internal/stackql/intrinsic/iac.go b/internal/stackql/intrinsic/iac.go index 6bbc75eb..f422703a 100644 --- a/internal/stackql/intrinsic/iac.go +++ b/internal/stackql/intrinsic/iac.go @@ -86,9 +86,11 @@ func iacSelectFunc( if err != nil { return internaldto.NewErroneousExecutorOutput(err) } - plan, planErr := omnisdk.Converge( - registryRoot(ctx), collection, state, runID, resources, - previewArgs(ctx, convergeCloud(resources), params)) + args, argsErr := previewArgs(ctx, convergeCloud(resources), params) + if argsErr != nil { + return internaldto.NewErroneousExecutorOutput(argsErr) + } + plan, planErr := omnisdk.Converge(registryRoot(ctx), collection, state, runID, resources, args) if planErr != nil { return internaldto.NewErroneousExecutorOutput(planErr) } diff --git a/internal/stackql/intrinsic/preview_services.go b/internal/stackql/intrinsic/preview_services.go index 422b4a87..d79dd9ef 100644 --- a/internal/stackql/intrinsic/preview_services.go +++ b/internal/stackql/intrinsic/preview_services.go @@ -141,16 +141,21 @@ func streamPlan( } // previewArgs assembles the SDK arguments both services need: the scope left -// over after the control predicates, the credential for the cloud in play, and -// the backend tuning. -func previewArgs(ctx queryContext, cloud string, params map[string]string) omnisdk.Args { +// over after the control predicates, the credential for the cloud in play, the +// backend tuning, and the backend's SQL dialect. +func previewArgs(ctx queryContext, cloud string, params map[string]string) (omnisdk.Args, error) { + dialect, err := backendDialect(ctx) + if err != nil { + return omnisdk.Args{}, err + } input := previewCfg return omnisdk.Args{ Params: params, AuthByProvider: providerAuthByProvider(ctx, cloud), Endpoint: input.getEndpoint(), InsecureSkipTLSVerify: input.getInsecureSkipTLSVerify(), - } + Dialect: dialect, + }, nil } // previewSelectFunc routes a SELECT over an extended preview relation. diff --git a/internal/stackql/intrinsic/staged.go b/internal/stackql/intrinsic/staged.go index 4e9e75bc..8a8d0823 100644 --- a/internal/stackql/intrinsic/staged.go +++ b/internal/stackql/intrinsic/staged.go @@ -18,6 +18,7 @@ import ( "github.com/stackql-labs/omnisdk/pkg/omnisdk" "github.com/stackql-labs/omnisdk/pkg/query" + "github.com/stackql-labs/omnisdk/pkg/sqlfn" "github.com/stackql/any-sdk/pkg/dto" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" "github.com/stackql/stackql/internal/stackql/omnistaging" @@ -139,7 +140,11 @@ func stagedSelectFunc( node *sqlparser.Select, currentProvider string, ) func() internaldto.ExecutorOutput { - staged, err := planStagedSelect(node, currentProvider, ctx.GetASTFormatter()) + dialect, err := backendDialect(ctx) + if err != nil { + return refuse(err) + } + staged, err := planStagedSelect(node, currentProvider, dialect, ctx.GetASTFormatter()) if err != nil { return refuse(err) } @@ -248,9 +253,10 @@ func (r *stagedRefs) collectSelectExprs( func planStagedSelect( node *sqlparser.Select, currentProvider string, + dialect sqlfn.Dialect, formatter sqlparser.NodeFormatter, ) (stagedSelect, error) { - t, where, err := translateSource(node, currentProvider) + t, where, err := translateSource(node, currentProvider, dialect) if err != nil { return nil, err } diff --git a/internal/stackql/intrinsic/staged_test.go b/internal/stackql/intrinsic/staged_test.go index fe81d28e..f33a5c0e 100644 --- a/internal/stackql/intrinsic/staged_test.go +++ b/internal/stackql/intrinsic/staged_test.go @@ -10,6 +10,7 @@ import ( "path/filepath" "testing" + "github.com/stackql-labs/omnisdk/pkg/sqlfn" "github.com/stackql/any-sdk/pkg/dto" "github.com/stackql/any-sdk/public/sqlengine" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" @@ -84,7 +85,7 @@ func TestPlanStagedSelectOuterStatement(t *testing.T) { }, } for _, tc := range cases { - staged, err := planStagedSelect(sel, "", tc.formatter) + staged, err := planStagedSelect(sel, "", sqlfn.SQLite, tc.formatter) if err != nil { t.Fatal(err) } @@ -102,7 +103,7 @@ func TestPlanStagedSelectOuterStatement(t *testing.T) { } } if _, err := planStagedSelect(parseSelect(t, - "select * from stackql_unstable_github.orgs.members order by login"), "", + "select * from stackql_unstable_github.orgs.members order by login"), "", sqlfn.SQLite, astformat.SQLiteSelectExprsFormatter); err == nil || err.Error() != "'*' cannot be staged for stackql_unstable_* relations; name the columns" { t.Fatalf("star refusal: got %v", err) diff --git a/internal/stackql/intrinsic/translate.go b/internal/stackql/intrinsic/translate.go index a082ce9e..ad86b3f0 100644 --- a/internal/stackql/intrinsic/translate.go +++ b/internal/stackql/intrinsic/translate.go @@ -12,6 +12,7 @@ import ( "strings" "github.com/stackql-labs/omnisdk/pkg/query" + "github.com/stackql-labs/omnisdk/pkg/sqlfn" "github.com/stackql/stackql-parser/go/vt/sqlparser" ) @@ -96,7 +97,7 @@ func fromDocProviders(from sqlparser.TableExprs, currentProvider string) (bool, // relations. What omnisdk leaves to the caller - ordering, grouping, // aggregation, de-duplication - is refused until stackql applies it over the // streamed rows. -func translateSelect(node *sqlparser.Select, currentProvider string) (docQuery, error) { +func translateSelect(node *sqlparser.Select, currentProvider string, dialect sqlfn.Dialect) (docQuery, error) { if unsupported := unsupportedDocClauses(node); len(unsupported) > 0 { return nil, fmt.Errorf("%s cannot be applied to %s relations; remove %s from the query", strings.Join(unsupported, ", "), UnstablePrefix+"*", pluralClause(len(unsupported))) @@ -105,7 +106,7 @@ func translateSelect(node *sqlparser.Select, currentProvider string) (docQuery, if err != nil { return nil, err } - t, where, err := translateSource(node, currentProvider) + t, where, err := translateSource(node, currentProvider, dialect) if err != nil { return nil, err } @@ -122,19 +123,24 @@ func translateSelect(node *sqlparser.Select, currentProvider string) (docQuery, // translateSource translates a SELECT's FROM and WHERE: the joins omnisdk runs // and the conjuncts it applies to them. -func translateSource(node *sqlparser.Select, currentProvider string) (*translator, []query.Predicate, error) { - if len(node.From) != 1 { - return nil, nil, fmt.Errorf("a comma-separated FROM cannot be applied to %s relations; use JOIN ... ON", - UnstablePrefix+"*") - } - t := &translator{currentProvider: currentProvider} - if err := t.from(node.From[0], query.Base, nil); err != nil { - return nil, nil, err +func translateSource( + node *sqlparser.Select, currentProvider string, dialect sqlfn.Dialect, +) (*translator, []query.Predicate, error) { + t := &translator{currentProvider: currentProvider, dialect: dialect} + // A comma-separated FROM is a cross join of its items; WHERE then says how they relate. + for i, item := range node.From { + form := query.Cross + if i == 0 { + form = query.Base + } + if err := t.from(item, form, nil); err != nil { + return nil, nil, err + } } if node.Where == nil { return t, nil, nil } - where, err := conjuncts(node.Where.Expr) + where, err := t.conjuncts(node.Where.Expr) if err != nil { return nil, nil, err } @@ -181,6 +187,7 @@ func pushedLimit(limit *sqlparser.Limit) (int, error) { } type translator struct { + dialect sqlfn.Dialect currentProvider string joins []query.Join bundles []string @@ -210,17 +217,23 @@ func (t *translator) from(expr sqlparser.TableExpr, form query.JoinForm, on []qu if err != nil { return err } - if len(node.Condition.Using) > 0 { - return fmt.Errorf("JOIN ... USING cannot be applied to %s relations; use ON", UnstablePrefix+"*") - } if err = t.from(node.LeftExpr, form, on); err != nil { return err } var rightOn []query.Predicate - if node.Condition.On != nil { - if rightOn, err = conjuncts(node.Condition.On); err != nil { + switch { + case node.Condition.On != nil: + if rightOn, err = t.conjuncts(node.Condition.On); err != nil { + return err + } + case len(node.Condition.Using) > 0: + // USING (c) is ON left.c = right.c, left being the table joined just before. + if rightOn, err = t.using(node.RightExpr, node.Condition.Using); err != nil { return err } + case rightForm == query.Inner: + // A JOIN with no condition, CROSS JOIN among them, pairs every row with every row. + rightForm = query.Cross } return t.from(node.RightExpr, rightForm, rightOn) default: @@ -242,6 +255,29 @@ func (t *translator) resource(tableName sqlparser.TableName, as sqlparser.TableI return query.NewResource(alias, address) } +// using is the ON a JOIN ... USING spells: each column equal on the table joined just before and on +// the right side. +func (t *translator) using(right sqlparser.TableExpr, cols sqlparser.Columns) ([]query.Predicate, error) { + aliased, isAliased := right.(*sqlparser.AliasedTableExpr) + if !isAliased || len(t.joins) == 0 { + return nil, fmt.Errorf("JOIN ... USING needs a table on each side") + } + rightName, isName := aliased.Expr.(sqlparser.TableName) + if !isName { + return nil, fmt.Errorf("JOIN ... USING needs a table on each side") + } + rightAlias := aliased.As.GetRawVal() + if rightAlias == "" { + rightAlias = rightName.Name.GetRawVal() + } + leftAlias := t.joins[len(t.joins)-1].Resource().Alias() + out := make([]query.Predicate, 0, len(cols)) + for _, c := range cols { + out = append(out, query.NewEq(query.NewColumn(leftAlias, c.GetRawVal()), query.NewColumn(rightAlias, c.GetRawVal()))) + } + return out, nil +} + func qualifiedName(tableName sqlparser.TableName) string { var parts []string for _, part := range []string{ @@ -267,19 +303,19 @@ func joinForm(join string) (query.JoinForm, error) { } // conjuncts splits a condition at its top-level ANDs. -func conjuncts(expr sqlparser.Expr) ([]query.Predicate, error) { +func (t *translator) conjuncts(expr sqlparser.Expr) ([]query.Predicate, error) { if and, isAnd := expr.(*sqlparser.AndExpr); isAnd { - left, err := conjuncts(and.Left) + left, err := t.conjuncts(and.Left) if err != nil { return nil, err } - right, err := conjuncts(and.Right) + right, err := t.conjuncts(and.Right) if err != nil { return nil, err } return append(left, right...), nil } - p, err := predicate(expr) + p, err := t.predicate(expr) if err != nil { return nil, err } @@ -295,10 +331,10 @@ var compareOps = map[string]query.CompareOp{ //nolint:gochecknoglobals // fixed sqlparser.GreaterEqualStr: query.Ge, } -func predicate(expr sqlparser.Expr) (query.Predicate, error) { +func (t *translator) predicate(expr sqlparser.Expr) (query.Predicate, error) { switch node := expr.(type) { case *sqlparser.AndExpr: - parts, err := conjuncts(node) + parts, err := t.conjuncts(node) if err != nil { return nil, err } @@ -309,23 +345,27 @@ func predicate(expr sqlparser.Expr) (query.Predicate, error) { } return query.NewNot(query.NewOr(negated...)), nil case *sqlparser.OrExpr: - left, err := predicate(node.Left) + left, err := t.predicate(node.Left) if err != nil { return nil, err } - right, err := predicate(node.Right) + right, err := t.predicate(node.Right) if err != nil { return nil, err } return query.NewOr(left, right), nil case *sqlparser.NotExpr: - inner, err := predicate(node.Expr) + inner, err := t.predicate(node.Expr) if err != nil { return nil, err } return query.NewNot(inner), nil case *sqlparser.ComparisonExpr: - return comparison(node) + return t.comparison(node) + case *sqlparser.IsExpr: + return isNull(node) + case *sqlparser.RangeCond: + return between(node) case *sqlparser.FuncExpr: call, err := expression(node) if err != nil { @@ -338,7 +378,57 @@ func predicate(expr sqlparser.Expr) (query.Predicate, error) { } } -func comparison(node *sqlparser.ComparisonExpr) (query.Predicate, error) { +// isNull is never unknown: is_null is true or false, so NOT of it is exact. +func isNull(node *sqlparser.IsExpr) (query.Predicate, error) { + e, err := expression(node.Expr) + if err != nil { + return nil, err + } + switch node.Operator { + case sqlparser.IsNullStr: + return query.NewTest(query.NewCall("is_null", e)), nil + case sqlparser.IsNotNullStr: + return query.NewNot(query.NewTest(query.NewCall("is_null", e))), nil + } + return nil, fmt.Errorf("condition '%s' cannot be applied to %s relations", + sqlparser.String(node), UnstablePrefix+"*") +} + +func between(node *sqlparser.RangeCond) (query.Predicate, error) { + parts := []sqlparser.Expr{node.Left, node.From, node.To} + args := make([]query.Expr, 0, len(parts)) + for _, part := range parts { + e, err := expression(part) + if err != nil { + return nil, err + } + args = append(args, e) + } + test := query.NewTest(query.NewCall("between", args...)) + if node.Operator == sqlparser.NotBetweenStr { + return query.NewNot(test), nil + } + return test, nil +} + +// like is value LIKE pattern [ESCAPE escape] as the backend's dialect calls it: SQLite's function +// like(pattern, value[, escape]), or Postgres's like(value, pattern), an ESCAPE clause rewriting the +// pattern through like_escape(pattern, escape) as Postgres's parser does. +func (t *translator) like(value, pattern, escape query.Expr) query.Expr { + if t.dialect == sqlfn.Postgres { + if escape != nil { + pattern = query.NewCall("like_escape", pattern, escape) + } + return query.NewCall("like", value, pattern) + } + args := []query.Expr{pattern, value} + if escape != nil { + args = append(args, escape) + } + return query.NewCall("like", args...) +} + +func (t *translator) comparison(node *sqlparser.ComparisonExpr) (query.Predicate, error) { left, err := expression(node.Left) if err != nil { return nil, err @@ -352,6 +442,19 @@ func comparison(node *sqlparser.ComparisonExpr) (query.Predicate, error) { return query.NewIn(left, right), nil case sqlparser.NotInStr: return query.NewNot(query.NewIn(left, right)), nil + case sqlparser.LikeStr, sqlparser.NotLikeStr: + var escape query.Expr + if node.Escape != nil { + var escErr error + if escape, escErr = expression(node.Escape); escErr != nil { + return nil, escErr + } + } + like := query.NewTest(t.like(left, right, escape)) + if node.Operator == sqlparser.NotLikeStr { + return query.NewNot(like), nil + } + return like, nil } op, known := compareOps[node.Operator] if !known { @@ -483,8 +586,8 @@ func mutationTables(stmt sqlparser.Statement) (sqlparser.TableExprs, bool) { // translateMutation builds the omnisdk mutation for an INSERT, UPDATE or // DELETE whose target is a document-driven relation. A RETURNING list becomes // the mutation's outputs. -func translateMutation(stmt sqlparser.Statement, currentProvider string) (docQuery, error) { - t := &translator{currentProvider: currentProvider} +func translateMutation(stmt sqlparser.Statement, currentProvider string, dialect sqlfn.Dialect) (docQuery, error) { + t := &translator{currentProvider: currentProvider, dialect: dialect} var ( target query.Target where []query.Predicate @@ -582,7 +685,7 @@ func (t *translator) insertSelect(rows *sqlparser.Select) ([]sqlparser.Expr, []q if err := t.from(rows.From[0], query.Base, nil); err != nil { return nil, nil, err } - where, err := whereConjuncts(rows.Where) + where, err := t.whereConjuncts(rows.Where) if err != nil { return nil, nil, err } @@ -622,7 +725,7 @@ func (t *translator) update(node *sqlparser.Update) (query.Target, []query.Predi } assignments = append(assignments, query.NewAssignment(set.Name.Name.GetRawVal(), translated)) } - where, err := whereConjuncts(node.Where) + where, err := t.whereConjuncts(node.Where) if err != nil { return nil, nil, err } @@ -641,7 +744,7 @@ func (t *translator) delete(node *sqlparser.Delete) (query.Target, []query.Predi if err != nil { return nil, nil, err } - where, err := whereConjuncts(node.Where) + where, err := t.whereConjuncts(node.Where) if err != nil { return nil, nil, err } @@ -687,9 +790,9 @@ func (t *translator) sources(from sqlparser.TableExprs) error { } } -func whereConjuncts(where *sqlparser.Where) ([]query.Predicate, error) { +func (t *translator) whereConjuncts(where *sqlparser.Where) ([]query.Predicate, error) { if where == nil { return nil, nil } - return conjuncts(where.Expr) + return t.conjuncts(where.Expr) } diff --git a/internal/stackql/intrinsic/translate_test.go b/internal/stackql/intrinsic/translate_test.go index 4556ea8a..86660100 100644 --- a/internal/stackql/intrinsic/translate_test.go +++ b/internal/stackql/intrinsic/translate_test.go @@ -4,10 +4,12 @@ import ( "fmt" "path/filepath" "reflect" + "strings" "testing" "github.com/stackql-labs/omnisdk/pkg/omnisdk" "github.com/stackql-labs/omnisdk/pkg/query" + "github.com/stackql-labs/omnisdk/pkg/sqlfn" "github.com/stackql/stackql-parser/go/vt/sqlparser" ) @@ -31,7 +33,7 @@ func TestTranslateSelectJoins(t *testing.T) { "inner join stackql_unstable_google.cloudkms.crypto_keys c on c.keyRingsId = split_part(k.name, '/', 6) "+ "left join stackql_unstable_google.cloudkms.crypto_keys c2 on c2.name = c.name "+ "where k.projectsId = 'p' and k.locationsId = 'global' limit 5") - dq, err := translateSelect(sel, "") + dq, err := translateSelect(sel, "", sqlfn.SQLite) if err != nil { t.Fatalf("translate: %v", err) } @@ -67,7 +69,7 @@ func TestTranslateSelectResolvesAgainstRegistry(t *testing.T) { withUnstable(t, true) sel := parseSelect(t, "select login from stackql_unstable_fixture.orgs.members "+ "where org = 'dummyorg' and (type = 'User' or not id = 2) and login in ('a', 'b')") - dq, err := translateSelect(sel, "") + dq, err := translateSelect(sel, "", sqlfn.SQLite) if err != nil { t.Fatalf("translate: %v", err) } @@ -108,15 +110,11 @@ func TestTranslateSelectRefusals(t *testing.T) { "stackql_unstable_* relations", "select login from stackql_unstable_github.orgs.members limit 1, 2": "OFFSET cannot be applied to " + "stackql_unstable_* relations", - "select login from stackql_unstable_github.orgs.members where login like 'a%'": "condition " + - "'`login` like 'a%'' cannot be applied to stackql_unstable_* relations", - "select a.login from stackql_unstable_github.orgs.members a, stackql_unstable_github.orgs.members b": "a " + - "comma-separated FROM cannot be applied to stackql_unstable_* relations; use JOIN ... ON", "select a.login from stackql_unstable_github.orgs.members a right join " + "stackql_unstable_github.orgs.members b on a.login = b.login": "RIGHT JOIN cannot be applied to " + "stackql_unstable_* relations", } { - _, err := translateSelect(parseSelect(t, sql), "") + _, err := translateSelect(parseSelect(t, sql), "", sqlfn.SQLite) if err == nil || err.Error() != want { t.Errorf("%s:\n got %v\nwant %s", sql, err, want) } @@ -238,7 +236,7 @@ func TestTranslateMutations(t *testing.T) { where: 2, }, } { - dq, err := translateMutation(parseStatement(t, tc.sql), "") + dq, err := translateMutation(parseStatement(t, tc.sql), "", sqlfn.SQLite) if err != nil { t.Errorf("%s: %v", tc.sql, err) continue @@ -278,7 +276,7 @@ func TestTranslateMutationRefusals(t *testing.T) { "delete from stackql_unstable_google.compute.firewalls where project = 'p' limit 1": "ORDER BY and " + "LIMIT cannot be applied to a DELETE of stackql_unstable_* relations", } { - _, err := translateMutation(parseStatement(t, sql), "") + _, err := translateMutation(parseStatement(t, sql), "", sqlfn.SQLite) if err == nil || err.Error() != want { t.Errorf("%s:\n got %v\nwant %s", sql, err, want) } @@ -322,3 +320,117 @@ func TestDocProviderUnderOmniAll(t *testing.T) { } } } + +// Predicates SQL spells as operators become calls omnisdk evaluates: IS [NOT] NULL, [NOT] LIKE with an +// optional ESCAPE, and [NOT] BETWEEN. +func TestTranslateOperatorPredicates(t *testing.T) { + withUnstable(t, true) + for cond, want := range map[string]string{ + "login is null": "is_null(login)", + "login is not null": "not is_null(login)", + "login like 'a%'": "like(a%, login)", + "login not like 'a!%' escape '!'": "not like(a!%, login, !)", + "id between 1 and 10": "between(id, 1, 10)", + "id not between 1 and 10": "not between(id, 1, 10)", + } { + sel := parseSelect(t, "select login from stackql_unstable_github.orgs.members where "+cond) + dq, err := translateSelect(sel, "", sqlfn.SQLite) + if err != nil { + t.Fatalf("%s: %v", cond, err) + } + where := dq.getQuery().Where() + if len(where) != 1 { + t.Fatalf("%s: %d conjuncts", cond, len(where)) + } + if got := describePredicate(where[0]); got != want { + t.Errorf("%s: got %s, want %s", cond, got, want) + } + } +} + +// On a Postgres backend LIKE is Postgres's like(value, pattern), an ESCAPE clause rewriting the +// pattern through like_escape as Postgres's parser does. +func TestTranslateLikePostgres(t *testing.T) { + withUnstable(t, true) + for cond, want := range map[string]string{ + "login like 'a%'": "like(login, a%)", + "login not like 'a!%' escape '!'": "not like(login, like_escape(a!%, !))", + } { + sel := parseSelect(t, "select login from stackql_unstable_github.orgs.members where "+cond) + dq, err := translateSelect(sel, "", sqlfn.Postgres) + if err != nil { + t.Fatalf("%s: %v", cond, err) + } + if got := describePredicate(dq.getQuery().Where()[0]); got != want { + t.Errorf("%s: got %s, want %s", cond, got, want) + } + } +} + +// A comma-separated FROM, CROSS JOIN and a JOIN with no condition are cross joins; JOIN ... USING is +// an inner join on each named column. +func TestTranslateCrossAndUsing(t *testing.T) { + withUnstable(t, true) + for sql, want := range map[string][]query.JoinForm{ + "select a.login from stackql_unstable_github.orgs.members a, stackql_unstable_github.orgs.members b": {query.Base, query.Cross}, + "select a.login from stackql_unstable_github.orgs.members a cross join stackql_unstable_github.orgs.members b": {query.Base, query.Cross}, + "select a.login from stackql_unstable_github.orgs.members a join stackql_unstable_github.orgs.members b": {query.Base, query.Cross}, + } { + dq, err := translateSelect(parseSelect(t, sql), "", sqlfn.SQLite) + if err != nil { + t.Fatalf("%s: %v", sql, err) + } + var got []query.JoinForm + for _, j := range dq.getQuery().From() { + got = append(got, j.Form()) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("%s: forms %v, want %v", sql, got, want) + } + } + dq, err := translateSelect(parseSelect(t, "select a.login from stackql_unstable_github.orgs.members a "+ + "join stackql_unstable_github.orgs.members b using (login, id)"), "", sqlfn.SQLite) + if err != nil { + t.Fatal(err) + } + b := dq.getQuery().From()[1] + var on []string + for _, p := range b.On() { + on = append(on, describePredicate(p)) + } + if b.Form() != query.Inner || strings.Join(on, "; ") != "a.login = b.login; a.id = b.id" { + t.Errorf("using: form %v, on %v", b.Form(), on) + } +} + +// describePredicate renders the predicate shapes these tests build. +func describePredicate(p query.Predicate) string { + switch p := p.(type) { + case query.Not: + return "not " + describePredicate(p.Negated()) + case query.Test: + return describeExpr(p.Cond()) + case query.Compare: + return describeExpr(p.Left()) + " " + string(p.Op()) + " " + describeExpr(p.Right()) + } + return fmt.Sprintf("%T", p) +} + +func describeExpr(e query.Expr) string { + switch e := e.(type) { + case query.Column: + if e.Qualifier() == "" { + return e.Name() + } + return e.Qualifier() + "." + e.Name() + case query.Literal: + return fmt.Sprint(e.Value()) + case query.Call: + var args []string + for _, a := range e.Args() { + args = append(args, describeExpr(a)) + } + return e.Func() + "(" + strings.Join(args, ", ") + ")" + } + return fmt.Sprintf("%T", e) +} diff --git a/test/robot/functional/stackql_mocked_from_cmd_line.robot b/test/robot/functional/stackql_mocked_from_cmd_line.robot index ec64d074..854e1f69 100644 --- a/test/robot/functional/stackql_mocked_from_cmd_line.robot +++ b/test/robot/functional/stackql_mocked_from_cmd_line.robot @@ -11117,6 +11117,36 @@ Unstable Github Org Members Jsonl Row Set Matches Expectation ... {"endpoint":"https://${LOCAL_HOST_ALIAS}:${MOCKSERVER_PORT_GITHUB}", ... "insecureSkipTLSVerify":true,"unstable":true} ${expected} = Catenate SEPARATOR=\n + ... {"id":"1","login":"some-jimbo-40","type":"User"} + ... {"id":"1","login":"some-jimbo-39","type":"User"} + ... {"id":"1","login":"some-jimbo-38","type":"User"} + ... {"id":"1","login":"some-jimbo-37","type":"User"} + ... {"id":"1","login":"some-jimbo-36","type":"User"} + ... {"id":"1","login":"some-jimbo-35","type":"User"} + ... {"id":"1","login":"some-jimbo-34","type":"User"} + ... {"id":"1","login":"some-jimbo-33","type":"User"} + ... {"id":"1","login":"some-jimbo-32","type":"User"} + ... {"id":"1","login":"some-jimbo-31","type":"User"} + ... {"id":"1","login":"some-jimbo-30","type":"User"} + ... {"id":"1","login":"some-jimbo-29","type":"User"} + ... {"id":"1","login":"some-jimbo-28","type":"User"} + ... {"id":"1","login":"some-jimbo-27","type":"User"} + ... {"id":"1","login":"some-jimbo-26","type":"User"} + ... {"id":"1","login":"some-jimbo-25","type":"User"} + ... {"id":"1","login":"some-jimbo-24","type":"User"} + ... {"id":"1","login":"some-jimbo-23","type":"User"} + ... {"id":"1","login":"some-jimbo-22","type":"User"} + ... {"id":"1","login":"some-jimbo-21","type":"User"} + ... {"id":"1","login":"some-jimbo-20","type":"User"} + ... {"id":"1","login":"some-jimbo-19","type":"User"} + ... {"id":"1","login":"some-jimbo-18","type":"User"} + ... {"id":"1","login":"some-jimbo-17","type":"User"} + ... {"id":"1","login":"some-jimbo-16","type":"User"} + ... {"id":"1","login":"some-jimbo-15","type":"User"} + ... {"id":"1","login":"some-jimbo-14","type":"User"} + ... {"id":"1","login":"some-jimbo-13","type":"User"} + ... {"id":"1","login":"some-jimbo-12","type":"User"} + ... {"id":"1","login":"some-jimbo-11","type":"User"} ... {"id":"1","login":"some-jimbo-10","type":"User"} ... {"id":"1","login":"some-jimbo-9","type":"User"} ... {"id":"1","login":"some-jimbo-8","type":"User"} @@ -11218,6 +11248,103 @@ Unstable Google Kms Key Rings Joined To Crypto Keys Jsonl Row Set Matches Expect ... stdout=${CURDIR}${/}tmp${/}Unstable-Google-Kms-Key-Rings-Joined-To-Crypto-Keys.tmp ... stderr=${CURDIR}${/}tmp${/}Unstable-Google-Kms-Key-Rings-Joined-To-Crypto-Keys-stderr.tmp +Unstable Github Org Members Filtered By Like Jsonl Row Set Matches Expectation + [Documentation] LIKE is evaluated by omnisdk on the streamed rows, every page included. + [Teardown] Remove Preview Mock Environment + ${preview} = Catenate SEPARATOR= + ... {"endpoint":"https://${LOCAL_HOST_ALIAS}:${MOCKSERVER_PORT_GITHUB}", + ... "insecureSkipTLSVerify":true,"unstable":true} + ${expected} = Catenate SEPARATOR=\n + ... {"login":"some-jimbo-1","type":"User"} + ... {"login":"some-jimbo-10","type":"User"} + ... {"login":"some-jimbo-11","type":"User"} + ... {"login":"some-jimbo-12","type":"User"} + ... {"login":"some-jimbo-13","type":"User"} + ... {"login":"some-jimbo-14","type":"User"} + ... {"login":"some-jimbo-15","type":"User"} + ... {"login":"some-jimbo-16","type":"User"} + ... {"login":"some-jimbo-17","type":"User"} + ... {"login":"some-jimbo-18","type":"User"} + ... {"login":"some-jimbo-19","type":"User"} + ${query} = Catenate SEPARATOR=${SPACE} + ... select login, type from stackql_unstable_github.orgs.members + ... where org = 'dummyorg' and login like 'some-jimbo-1%'; + Should StackQL Exec Inline Jsonl Set Equal + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${query} + ... ${expected} + ... --preview\=${preview} + ... stdout=${CURDIR}${/}tmp${/}Unstable-Github-Org-Members-Filtered-By-Like.tmp + ... stderr=${CURDIR}${/}tmp${/}Unstable-Github-Org-Members-Filtered-By-Like-stderr.tmp + +Unstable Github Org Members Filtered By Between And Is Not Null Jsonl Row Set Matches Expectation + [Documentation] BETWEEN orders text as text, and IS NOT NULL holds for every member with a type. + [Teardown] Remove Preview Mock Environment + ${preview} = Catenate SEPARATOR= + ... {"endpoint":"https://${LOCAL_HOST_ALIAS}:${MOCKSERVER_PORT_GITHUB}", + ... "insecureSkipTLSVerify":true,"unstable":true} + ${expected} = Catenate SEPARATOR=\n + ... {"login":"some-jimbo-2","type":"User"} + ... {"login":"some-jimbo-20","type":"User"} + ... {"login":"some-jimbo-21","type":"User"} + ... {"login":"some-jimbo-22","type":"User"} + ... {"login":"some-jimbo-23","type":"User"} + ... {"login":"some-jimbo-24","type":"User"} + ... {"login":"some-jimbo-25","type":"User"} + ... {"login":"some-jimbo-26","type":"User"} + ... {"login":"some-jimbo-27","type":"User"} + ... {"login":"some-jimbo-28","type":"User"} + ... {"login":"some-jimbo-29","type":"User"} + ... {"login":"some-jimbo-3","type":"User"} + ${query} = Catenate SEPARATOR=${SPACE} + ... select login, type from stackql_unstable_github.orgs.members + ... where org = 'dummyorg' and login between 'some-jimbo-2' and 'some-jimbo-3' + ... and type is not null; + Should StackQL Exec Inline Jsonl Set Equal + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${query} + ... ${expected} + ... --preview\=${preview} + ... stdout=${CURDIR}${/}tmp${/}Unstable-Github-Org-Members-Filtered-By-Between.tmp + ... stderr=${CURDIR}${/}tmp${/}Unstable-Github-Org-Members-Filtered-By-Between-stderr.tmp + +Unstable Github Org Members Comma Joined To Themselves Jsonl Row Set Matches Expectation + [Documentation] A comma-separated FROM is a cross join, narrowed by WHERE. + [Teardown] Remove Preview Mock Environment + ${preview} = Catenate SEPARATOR= + ... {"endpoint":"https://${LOCAL_HOST_ALIAS}:${MOCKSERVER_PORT_GITHUB}", + ... "insecureSkipTLSVerify":true,"unstable":true} + ${expected} = Catenate SEPARATOR=\n + ... {"login":"some-jimbo-3","type":"User"} + ${query} = Catenate SEPARATOR=${SPACE} + ... select a.login, a.type from stackql_unstable_github.orgs.members a, stackql_unstable_github.orgs.members b + ... where a.org = 'dummyorg' and b.org = 'dummyorg' and a.login = b.login and a.login = 'some-jimbo-3'; + Should StackQL Exec Inline Jsonl Set Equal + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${query} + ... ${expected} + ... --preview\=${preview} + ... stdout=${CURDIR}${/}tmp${/}Unstable-Github-Org-Members-Comma-Joined.tmp + ... stderr=${CURDIR}${/}tmp${/}Unstable-Github-Org-Members-Comma-Joined-stderr.tmp + Unstable Github Org Members Filtered By In List Jsonl Row Set Matches Expectation [Documentation] A condition no request parameter can carry is applied by ... omnisdk as a filter on the streamed rows. @@ -11290,7 +11417,7 @@ Unstable Github Org Members Staged Group By Exact Match ... ${AUTH_CFG_STR} ... ${SQL_BACKEND_CFG_STR_CANONICAL} ... ${query} - ... type,member_count\nUser,10 + ... type,member_count\nUser,40 ... \-o\=csv ... --preview\=${preview} ... stdout=${CURDIR}${/}tmp${/}Unstable-Github-Org-Members-Staged-Group-By.tmp