diff --git a/.dagger/.gitattributes b/.dagger/.gitattributes new file mode 100644 index 00000000..d3384f51 --- /dev/null +++ b/.dagger/.gitattributes @@ -0,0 +1,3 @@ +/dagger.gen.go linguist-generated +/internal/dagger/** linguist-generated +/internal/telemetry/** linguist-generated diff --git a/.dagger/.gitignore b/.dagger/.gitignore new file mode 100644 index 00000000..33427156 --- /dev/null +++ b/.dagger/.gitignore @@ -0,0 +1,4 @@ +/dagger.gen.go +/internal/dagger +/internal/telemetry +/.env diff --git a/.dagger/LICENSE b/.dagger/LICENSE new file mode 100644 index 00000000..137069b8 --- /dev/null +++ b/.dagger/LICENSE @@ -0,0 +1,73 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: + + (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. + + You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + +To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/.dagger/dagger.json b/.dagger/dagger.json new file mode 100644 index 00000000..e82c244b --- /dev/null +++ b/.dagger/dagger.json @@ -0,0 +1,7 @@ +{ + "name": "stem-ci", + "engineVersion": "v0.21.7", + "sdk": { + "source": "go" + } +} diff --git a/.dagger/go.mod b/.dagger/go.mod new file mode 100644 index 00000000..00007019 --- /dev/null +++ b/.dagger/go.mod @@ -0,0 +1,53 @@ +module dagger/stem-ci + +go 1.26.4 + +require ( + github.com/Khan/genqlient v0.8.1 + github.com/dagger/otel-go v1.43.0 + github.com/dagger/querybuilder v0.0.0-20260402040506-574a5e81cb59 + github.com/vektah/gqlparser/v2 v2.5.32 + go.opentelemetry.io/otel v1.43.0 + go.opentelemetry.io/otel/trace v1.43.0 +) + +require ( + github.com/99designs/gqlgen v0.17.89 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect + github.com/sosodev/duration v1.4.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.17.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.17.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.41.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.41.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.41.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.41.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.41.0 // indirect + go.opentelemetry.io/otel/log v0.17.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/sdk v1.43.0 + go.opentelemetry.io/otel/sdk/log v0.17.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect + go.opentelemetry.io/proto/otlp v1.9.0 // indirect + golang.org/x/net v0.51.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.42.0 // indirect + golang.org/x/text v0.34.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260226221140-a57be14db171 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 // indirect + google.golang.org/grpc v1.79.3 // indirect + google.golang.org/protobuf v1.36.11 // indirect +) + +replace go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc => go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.16.0 + +replace go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp => go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.16.0 + +replace go.opentelemetry.io/otel/log => go.opentelemetry.io/otel/log v0.16.0 + +replace go.opentelemetry.io/otel/sdk/log => go.opentelemetry.io/otel/sdk/log v0.16.0 diff --git a/.dagger/go.sum b/.dagger/go.sum new file mode 100644 index 00000000..4bb39e60 --- /dev/null +++ b/.dagger/go.sum @@ -0,0 +1,97 @@ +github.com/99designs/gqlgen v0.17.89 h1:KzEcxPiMgQoMw3m/E85atUEHyZyt0PbAflMia5Kw8z8= +github.com/99designs/gqlgen v0.17.89/go.mod h1:GFqruTVGB7ZTdrf1uzOagpXbY7DrEt1pIxnTdhIbWvQ= +github.com/Khan/genqlient v0.8.1 h1:wtOCc8N9rNynRLXN3k3CnfzheCUNKBcvXmVv5zt6WCs= +github.com/Khan/genqlient v0.8.1/go.mod h1:R2G6DzjBvCbhjsEajfRjbWdVglSH/73kSivC9TLWVjU= +github.com/agnivade/levenshtein v1.2.1 h1:EHBY3UOn1gwdy/VbFwgo4cxecRznFk7fKWN1KOX7eoM= +github.com/agnivade/levenshtein v1.2.1/go.mod h1:QVVI16kDrtSuwcpd0p1+xMC6Z/VfhtCyDIjcwga4/DU= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/dagger/otel-go v1.43.0 h1:AYCnAamWmxtSxigWPTgC+8EWqiWPcDZEegh8y05gdJ8= +github.com/dagger/otel-go v1.43.0/go.mod h1:83CTuXi70zcx1kaym5buqmb7RNzg1E9dEiQSFyLbLdU= +github.com/dagger/querybuilder v0.0.0-20260402040506-574a5e81cb59 h1:g6vfdGRyz6fAjfHz5FyYPZgHy8qcQ31fHrBl1iCOzxw= +github.com/dagger/querybuilder v0.0.0-20260402040506-574a5e81cb59/go.mod h1:jsdUJeYzcbyK1j/EqMGPrQgNYxl/Zfg06vvM9C/xXxs= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= +github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/sosodev/duration v1.4.0 h1:35ed0KiVFriGHHzZZJaZLgmTEEICIyt8Sx0RQfj9IjE= +github.com/sosodev/duration v1.4.0/go.mod h1:RQIBBX0+fMLc/D9+Jb/fwvVmo0eZvDDEERAikUR6SDg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/vektah/gqlparser/v2 v2.5.32 h1:k9QPJd4sEDTL+qB4ncPLflqTJ3MmjB9SrVzJrawpFSc= +github.com/vektah/gqlparser/v2 v2.5.32/go.mod h1:c1I28gSOVNzlfc4WuDlqU7voQnsqI6OG2amkBAFmgts= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.16.0 h1:ZVg+kCXxd9LtAaQNKBxAvJ5NpMf7LpvEr4MIZqb0TMQ= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.16.0/go.mod h1:hh0tMeZ75CCXrHd9OXRYxTlCAdxcXioWHFIpYw2rZu8= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.16.0 h1:djrxvDxAe44mJUrKataUbOhCKhR3F8QCyWucO16hTQs= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.16.0/go.mod h1:dt3nxpQEiSoKvfTVxp3TUg5fHPLhKtbcnN3Z1I1ePD0= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.41.0 h1:VO3BL6OZXRQ1yQc8W6EVfJzINeJ35BkiHx4MYfoQf44= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.41.0/go.mod h1:qRDnJ2nv3CQXMK2HUd9K9VtvedsPAce3S+/4LZHjX/s= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.41.0 h1:MMrOAN8H1FrvDyq9UJ4lu5/+ss49Qgfgb7Zpm0m8ABo= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.41.0/go.mod h1:Na+2NNASJtF+uT4NxDe0G+NQb+bUgdPDfwxY/6JmS/c= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.41.0 h1:ao6Oe+wSebTlQ1OEht7jlYTzQKE+pnx/iNywFvTbuuI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.41.0/go.mod h1:u3T6vz0gh/NVzgDgiwkgLxpsSF6PaPmo2il0apGJbls= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.41.0 h1:mq/Qcf28TWz719lE3/hMB4KkyDuLJIvgJnFGcd0kEUI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.41.0/go.mod h1:yk5LXEYhsL2htyDNJbEq7fWzNEigeEdV5xBF/Y+kAv0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.41.0 h1:inYW9ZhgqiDqh6BioM7DVHHzEGVq76Db5897WLGZ5Go= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.41.0/go.mod h1:Izur+Wt8gClgMJqO/cZ8wdeeMryJ/xxiOVgFSSfpDTY= +go.opentelemetry.io/otel/log v0.16.0 h1:DeuBPqCi6pQwtCK0pO4fvMB5eBq6sNxEnuTs88pjsN4= +go.opentelemetry.io/otel/log v0.16.0/go.mod h1:rWsmqNVTLIA8UnwYVOItjyEZDbKIkMxdQunsIhpUMes= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/log v0.16.0 h1:e/b4bdlQwC5fnGtG3dlXUrNOnP7c8YLVSpSfEBIkTnI= +go.opentelemetry.io/otel/sdk/log v0.16.0/go.mod h1:JKfP3T6ycy7QEuv3Hj8oKDy7KItrEkus8XJE6EoSzw4= +go.opentelemetry.io/otel/sdk/log/logtest v0.16.0 h1:/XVkpZ41rVRTP4DfMgYv1nEtNmf65XPPyAdqV90TMy4= +go.opentelemetry.io/otel/sdk/log/logtest v0.16.0/go.mod h1:iOOPgQr5MY9oac/F5W86mXdeyWZGleIx3uXO98X2R6Y= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= +go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= +golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/api v0.0.0-20260226221140-a57be14db171 h1:tu/dtnW1o3wfaxCOjSLn5IRX4YDcJrtlpzYkhHhGaC4= +google.golang.org/genproto/googleapis/api v0.0.0-20260226221140-a57be14db171/go.mod h1:M5krXqk4GhBKvB596udGL3UyjL4I1+cTbK0orROM9ng= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 h1:ggcbiqK8WWh6l1dnltU4BgWGIGo+EVYxCaAPih/zQXQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= +google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/.dagger/main.go b/.dagger/main.go new file mode 100644 index 00000000..3e957c56 --- /dev/null +++ b/.dagger/main.go @@ -0,0 +1,345 @@ +// A reproducible Stem test environment powered by Dagger. +// +// The root Taskfile remains the source of truth for the package test order. +// This module supplies the pinned toolchain and disposable integration +// services around that task, so the same command can run locally and in CI. +package main + +import ( + "context" + "fmt" + + "dagger/stem-ci/internal/dagger" +) + +const ( + dartImage = "dart:3.10.0" + flutterVersion = "3.47.0" + flutterArchive = "flutter_linux_3.47.0-stable.tar.xz" + flutterSHA256 = "26cd99d3d94b1367e6b50535a18aeef0282c10a535bbe3ec493534dcdab75296" + flutterRoot = "/opt/flutter" + taskVersion = "3.53.1" + taskSHA256 = "a54a408f6861ff921f6e87774180db31bacd8c1e7c944ca696db9fea49a82fc7" + workspaceDir = "/workspace" + testCertsDir = "/stem-test-certs" + taskBinaryPath = "/usr/local/bin/task" +) + +type StemCi struct{} + +// Check runs the complete root test gate in an isolated, reproducible +// environment. The source directory is copied into the test container, while +// Redis and PostgreSQL are provided as Dagger services and discarded when the +// call finishes. +func (m *StemCi) Check(ctx context.Context, source *dagger.Directory) (string, error) { + assets := m.tlsAssets(source) + testCerts := m.clientCertificates(assets) + + postgres := m.postgresService(assets.Directory("postgres")) + redis := m.redisService("redis", assets.Directory("redis"), false) + redisTLS := m.redisService("redis-tls", assets.Directory("redis"), false) + redisMTLS := m.redisService("redis-mtls", assets.Directory("redis"), true) + + test := m.testContainer(source, testCerts). + WithServiceBinding("postgres", postgres). + WithServiceBinding("redis", redis). + WithServiceBinding("redis-tls", redisTLS). + WithServiceBinding("redis-mtls", redisMTLS) + + stdout, err := m.runTests(test).Stdout(ctx) + if err != nil { + return "", fmt.Errorf("Stem test gate failed: %w", err) + } + return stdout, nil +} + +// All runs every package test, including the Flutter packages. It is the +// default CI entrypoint; Check remains available for the faster Dart-only +// integration gate while developing the Dagger module. +func (m *StemCi) All(ctx context.Context, source *dagger.Directory) (string, error) { + dartOutput, err := m.Check(ctx, source) + if err != nil { + return "", err + } + + flutterOutput, err := m.runFlutterTests( + m.flutterContainer(source), + ).Stdout(ctx) + if err != nil { + return "", fmt.Errorf("Stem Flutter test gate failed: %w", err) + } + + return "Dart gate:\n" + dartOutput + "\nFlutter gate:\n" + flutterOutput, nil +} + +// flutterContainer installs the pinned Flutter SDK from the official Linux +// release archive. This avoids depending on an unpinned or third-party image +// for the Flutter portion of the gate. +func (m *StemCi) flutterContainer(source *dagger.Directory) *dagger.Container { + return dag.Container(). + From(dartImage). + WithEnvVariable( + "PATH", + flutterRoot+"/bin:"+ + flutterRoot+"/bin/cache/dart-sdk/bin:"+ + "/root/.pub-cache/bin:/usr/local/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin", + ). + WithMountedCache("/root/.pub-cache", dag.CacheVolume("stem-flutter-pub-cache")). + WithDirectory(workspaceDir, source, dagger.ContainerWithDirectoryOpts{Gitignore: true}). + WithWorkdir(workspaceDir). + WithExec([]string{ + "bash", + "-c", + fmt.Sprintf( + "set -euo pipefail\n"+ + "apt-get update\n"+ + "DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "+ + "ca-certificates curl git libglu1-mesa unzip xz-utils\n"+ + "rm -rf /var/lib/apt/lists/*\n"+ + "mkdir -p /opt\n"+ + "curl -fsSL -o /tmp/%s https://storage.googleapis.com/flutter_infra_release/releases/stable/linux/%s\n"+ + "printf '%s /tmp/%s\\n' | sha256sum -c -\n"+ + "tar -xJf /tmp/%s -C /opt\n"+ + "git config --global --add safe.directory "+flutterRoot+"\n"+ + "flutter config --no-analytics\n"+ + "flutter --version\n"+ + "curl -fsSL -o /tmp/task.tar.gz https://github.com/go-task/task/releases/download/v%s/task_linux_amd64.tar.gz\n"+ + "printf '%s /tmp/task.tar.gz\\n' | sha256sum -c -\n"+ + "tar -xzf /tmp/task.tar.gz -C /usr/local/bin task\n"+ + "chmod 0755 %s\n"+ + "task --version\n", + flutterArchive, + flutterArchive, + flutterSHA256, + flutterArchive, + flutterArchive, + taskVersion, + taskSHA256, + taskBinaryPath, + ), + }) +} + +func (m *StemCi) runFlutterTests(container *dagger.Container) *dagger.Container { + return container.WithExec([]string{ + "bash", + "-c", + "set -euo pipefail\n" + + "task standalone:flutter\n" + + "task test:flutter\n", + }) +} + +// tlsAssets creates separate disposable certificate authorities for Redis and +// PostgreSQL. Keeping the generation inside Dagger means CI never needs to +// commit or cache private-key fixtures. +func (m *StemCi) tlsAssets(source *dagger.Directory) *dagger.Directory { + generator := dag.Container(). + From("alpine:3.22"). + WithExec([]string{"apk", "add", "--no-cache", "bash", "openssl"}). + WithMountedFile( + "/generate_tls_assets.sh", + source.File("packages/stem/scripts/security/generate_tls_assets.sh"), + ). + WithExec([]string{ + "bash", + "-c", + "set -euo pipefail\n" + + "mkdir -p /certs/redis /certs/postgres\n" + + "/generate_tls_assets.sh /certs/redis redis 'redis,redis-tls,redis-mtls,localhost,127.0.0.1' >/dev/null\n" + + "/generate_tls_assets.sh /certs/postgres postgres 'postgres,localhost,127.0.0.1' >/dev/null\n" + + "cp /certs/postgres/ca.crt /certs/postgres/root.crt\n" + + "chmod 644 /certs/redis/*.crt /certs/redis/*.key\n" + + "chmod 644 /certs/postgres/*.crt\n" + + "chmod 600 /certs/postgres/*.key\n", + }) + + return generator.Directory("certs") +} + +// clientCertificates removes CA and server private keys before the test +// container receives the client-facing certificate material. +func (m *StemCi) clientCertificates(assets *dagger.Directory) *dagger.Directory { + redis := assets.Directory("redis").WithoutFiles([]string{"ca.key", "server.key"}) + postgres := assets.Directory("postgres").WithoutFiles([]string{"ca.key", "server.key"}) + return dag.Directory(). + WithDirectory("redis", redis). + WithDirectory("postgres", postgres) +} + +func (m *StemCi) postgresService(certificates *dagger.Directory) *dagger.Service { + return dag.Container(). + From("postgres:14"). + WithEnvVariable("POSTGRES_USER", "postgres"). + WithEnvVariable("POSTGRES_PASSWORD", "postgres"). + WithEnvVariable("POSTGRES_DB", "stem_test"). + WithDirectory( + "/etc/postgres/certs", + certificates, + dagger.ContainerWithDirectoryOpts{Owner: "postgres:postgres"}, + ). + WithExposedPort(5432). + AsService(dagger.ContainerAsServiceOpts{ + Args: []string{ + "postgres", + "-c", "ssl=on", + "-c", "ssl_cert_file=/etc/postgres/certs/server.crt", + "-c", "ssl_key_file=/etc/postgres/certs/server.key", + "-c", "ssl_ca_file=/etc/postgres/certs/root.crt", + "-c", "max_connections=200", + }, + UseEntrypoint: true, + }) +} + +func (m *StemCi) redisService( + name string, + certificates *dagger.Directory, + mutualTLS bool, +) *dagger.Service { + args := []string{ + "--port", "0", + "--tls-port", "6379", + "--tls-cert-file", "/etc/redis/certs/server.crt", + "--tls-key-file", "/etc/redis/certs/server.key", + "--tls-ca-cert-file", "/etc/redis/certs/ca.crt", + "--tls-auth-clients", "no", + "--databases", "16", + "--appendonly", "no", + } + if name == "redis" { + args = []string{ + "--port", "6379", + "--tls-port", "0", + "--databases", "16", + "--appendonly", "no", + } + } + if mutualTLS { + for i := range args { + if args[i] == "no" && i > 0 && args[i-1] == "--tls-auth-clients" { + args[i] = "yes" + } + } + } + + healthcheck := "redis-cli -h 127.0.0.1 -p 6379 ping | grep PONG" + if name != "redis" { + healthcheck = "redis-cli --tls --cacert /etc/redis/certs/ca.crt -p 6379 ping | grep PONG" + if mutualTLS { + healthcheck = "redis-cli --tls --cacert /etc/redis/certs/ca.crt --cert /etc/redis/certs/client.crt --key /etc/redis/certs/client.key -p 6379 ping | grep PONG" + } + } + + return dag.Container(). + From("redis:7-alpine"). + WithDirectory("/etc/redis/certs", certificates). + WithExposedPort(6379). + WithDockerHealthcheck( + []string{healthcheck}, + dagger.ContainerWithDockerHealthcheckOpts{ + Shell: true, + Interval: "2s", + Timeout: "5s", + }, + ). + AsService(dagger.ContainerAsServiceOpts{ + Args: append([]string{"redis-server"}, args...), + UseEntrypoint: true, + }) +} + +func (m *StemCi) testContainer( + source *dagger.Directory, + certificates *dagger.Directory, +) *dagger.Container { + return dag.Container(). + From(dartImage). + WithEnvVariable("PATH", "/usr/lib/dart/bin:/root/.pub-cache/bin:/usr/local/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin"). + WithMountedCache("/root/.pub-cache", dag.CacheVolume("stem-pub-cache")). + WithMountedCache( + "/root/.server_testing", + dag.CacheVolume("stem-server-testing"), + ). + WithDirectory(workspaceDir, source, dagger.ContainerWithDirectoryOpts{Gitignore: true}). + WithDirectory(testCertsDir, certificates). + WithWorkdir(workspaceDir). + WithExec([]string{ + "bash", + "-c", + fmt.Sprintf( + "set -euo pipefail\n"+ + "apt-get update\n"+ + "DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "+ + "sqlite3 libsqlite3-dev postgresql-client redis-tools ca-certificates "+ + "libasound2t64 libatk-bridge2.0-0t64 libatk1.0-0t64 libatspi2.0-0 "+ + "libcairo2 libcups2t64 libdbus-1-3 libdrm2 libgbm1 libglib2.0-0t64 "+ + "libgtk-3-0t64 libnspr4 libnss3 libpango-1.0-0 libwayland-client0 "+ + "libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxdamage1 libxext6 "+ + "libxfixes3 libxkbcommon0 libxrandr2 libxshmfence1\n"+ + "rm -rf /var/lib/apt/lists/*\n"+ + "curl -fsSL -o /tmp/task.tar.gz https://github.com/go-task/task/releases/download/v%s/task_linux_amd64.tar.gz\n"+ + "printf '%s /tmp/task.tar.gz\\n' | sha256sum -c -\n"+ + "tar -xzf /tmp/task.tar.gz -C /usr/local/bin task\n"+ + "ln -sf /usr/lib/dart/bin/dart /usr/local/bin/dart\n"+ + "chmod 0755 %s\n"+ + "task --version\n"+ + "dart --version\n", + taskVersion, + taskSHA256, + taskBinaryPath, + ), + }). + WithEnvVariable("STEM_TEST_REDIS_URL", "redis://redis:6379/0"). + WithEnvVariable("STEM_TEST_POSTGRES_URL", "postgresql://postgres:postgres@postgres:5432/stem_test"). + WithEnvVariable("STEM_TEST_REDIS_TLS_URL", "rediss://redis-tls:6379/0"). + WithEnvVariable("STEM_TEST_REDIS_TLS_CA_CERT", testCertsDir+"/redis/ca.crt"). + WithEnvVariable("STEM_TEST_REDIS_MTLS_URL", "rediss://redis-mtls:6379/0"). + WithEnvVariable("STEM_TEST_REDIS_MTLS_CA_CERT", testCertsDir+"/redis/ca.crt"). + WithEnvVariable("STEM_TEST_REDIS_MTLS_CLIENT_CERT", testCertsDir+"/redis/client.crt"). + WithEnvVariable("STEM_TEST_REDIS_MTLS_CLIENT_KEY", testCertsDir+"/redis/client.key"). + WithEnvVariable( + "STEM_TEST_POSTGRES_TLS_URL", + "postgresql://postgres:postgres@postgres:5432/stem_test?sslmode=verify-ca&sslrootcert="+ + testCertsDir+ + "/postgres/root.crt", + ). + WithEnvVariable( + "STEM_TEST_POSTGRES_TLS_CA_CERT", + testCertsDir+"/postgres/root.crt", + ). + WithEnvVariable("STEM_CHAOS_REDIS_URL", "redis://redis:6379/15"). + WithEnvVariable("POSTGRES_URL", "postgresql://postgres:postgres@postgres:5432/stem_test"). + WithEnvVariable("REDIS_URL", "redis://redis:6379/0") +} + +func (m *StemCi) runTests(container *dagger.Container) *dagger.Container { + return container.WithExec([]string{ + "bash", + "-c", + "set -euo pipefail\n" + + "for attempt in $(seq 1 30); do\n" + + " pg_isready -h postgres -p 5432 -U postgres -d stem_test >/dev/null 2>&1 && break\n" + + " sleep 1\n" + + " if [ \"$attempt\" -eq 30 ]; then\n" + + " echo 'PostgreSQL did not become ready' >&2\n" + + " exit 1\n" + + " fi\n" + + "done\n" + + "for attempt in $(seq 1 30); do\n" + + " redis-cli -h redis -p 6379 ping >/dev/null 2>&1 && break\n" + + " sleep 1\n" + + " if [ \"$attempt\" -eq 30 ]; then\n" + + " echo 'Redis did not become ready' >&2\n" + + " exit 1\n" + + " fi\n" + + "done\n" + + "if ! command -v flutter >/dev/null 2>&1; then\n" + + " sed -i '/packages\\/stem_flutter/d' pubspec.yaml\n" + + "fi\n" + + "task deps\n" + + "task quality:dart\n" + + "task standalone:dart\n" + + "task test\n", + }) +} diff --git a/.github/actions/setup-test-services/action.yaml b/.github/actions/setup-test-services/action.yaml deleted file mode 100644 index d10b2f8e..00000000 --- a/.github/actions/setup-test-services/action.yaml +++ /dev/null @@ -1,69 +0,0 @@ -name: Setup test services -description: Start Redis and Postgres test containers and export environment variables - -runs: - using: composite - steps: - - name: Start Redis/Postgres containers - shell: bash - run: | - set -euo pipefail - export STEM_TEST_REDIS_TLS_PORT="${STEM_TEST_REDIS_TLS_PORT:-0}" - export STEM_TEST_REDIS_MTLS_PORT="${STEM_TEST_REDIS_MTLS_PORT:-0}" - for attempt in 1 2 3; do - if docker compose -f packages/stem_cli/docker/testing/docker-compose.yml up -d; then - break - fi - echo "docker compose up failed (attempt ${attempt}). Retrying..." - docker compose -f packages/stem_cli/docker/testing/docker-compose.yml down --remove-orphans || true - sleep 5 - if [ "$attempt" -eq 3 ]; then - exit 1 - fi - done - - name: Wait for services to be ready - shell: bash - run: | - # Wait for Postgres to be ready - for i in {1..30}; do - if pg_isready -h 127.0.0.1 -p 65432 -U postgres 2>/dev/null; then - break - fi - sleep 1 - done - # Wait for Postgres TLS to be ready - for i in {1..30}; do - if pg_isready -h 127.0.0.1 -p 65433 -U postgres 2>/dev/null; then - break - fi - sleep 1 - done - # Reset test database to clean state - PGPASSWORD=postgres psql -h 127.0.0.1 -U postgres -p 65432 -d postgres -c "DROP DATABASE IF EXISTS stem_test;" 2>/dev/null || true - PGPASSWORD=postgres psql -h 127.0.0.1 -U postgres -p 65432 -d postgres -c "CREATE DATABASE stem_test;" 2>/dev/null || true - # Reset TLS test database to clean state - PGPASSWORD=postgres psql -h 127.0.0.1 -U postgres -p 65433 -d postgres -c "DROP DATABASE IF EXISTS stem_test_tls;" 2>/dev/null || true - PGPASSWORD=postgres psql -h 127.0.0.1 -U postgres -p 65433 -d postgres -c "CREATE DATABASE stem_test_tls;" 2>/dev/null || true - - name: Export STEM_TEST_* environment variables - shell: bash - run: | - redis_tls_mapping=$(docker compose -f packages/stem_cli/docker/testing/docker-compose.yml port redis_tls 6379) - redis_tls_port=${redis_tls_mapping##*:} - redis_mtls_mapping=$(docker compose -f packages/stem_cli/docker/testing/docker-compose.yml port redis_mtls 6379) - redis_mtls_port=${redis_mtls_mapping##*:} - echo "STEM_TEST_REDIS_TLS_PORT=${redis_tls_port}" >> "$GITHUB_ENV" - echo "STEM_TEST_REDIS_MTLS_PORT=${redis_mtls_port}" >> "$GITHUB_ENV" - echo "STEM_TEST_REDIS_URL=redis://127.0.0.1:56379" >> "$GITHUB_ENV" - echo "STEM_TEST_POSTGRES_URL=postgresql://postgres:postgres@127.0.0.1:65432/stem_test" >> "$GITHUB_ENV" - echo "STEM_TEST_POSTGRES_TLS_URL=postgresql://postgres:postgres@localhost:65433/stem_test_tls?sslmode=verify-ca&sslrootcert=$GITHUB_WORKSPACE/packages/stem_cli/docker/testing/certs/postgres-root.crt" >> "$GITHUB_ENV" - echo "STEM_TEST_POSTGRES_TLS_CA_CERT=$GITHUB_WORKSPACE/packages/stem_cli/docker/testing/certs/postgres-root.crt" >> "$GITHUB_ENV" - echo "STEM_TEST_REDIS_TLS_URL=rediss://localhost:${redis_tls_port}/0" >> "$GITHUB_ENV" - echo "STEM_TEST_REDIS_TLS_CA_CERT=$GITHUB_WORKSPACE/packages/stem/example/microservice/certs/ca.crt" >> "$GITHUB_ENV" - echo "STEM_TEST_REDIS_MTLS_URL=rediss://localhost:${redis_mtls_port}/0" >> "$GITHUB_ENV" - echo "STEM_TEST_REDIS_MTLS_CA_CERT=$GITHUB_WORKSPACE/packages/stem/example/microservice/certs/ca.crt" >> "$GITHUB_ENV" - echo "STEM_TEST_REDIS_MTLS_CLIENT_CERT=$GITHUB_WORKSPACE/packages/stem/example/microservice/certs/client.crt" >> "$GITHUB_ENV" - echo "STEM_TEST_REDIS_MTLS_CLIENT_KEY=$GITHUB_WORKSPACE/packages/stem/example/microservice/certs/client.key" >> "$GITHUB_ENV" - echo "STEM_CHAOS_REDIS_URL=redis://127.0.0.1:56379/15" >> "$GITHUB_ENV" - # Also set generic POSTGRES_URL and REDIS_URL for compatibility with different tests - echo "POSTGRES_URL=postgresql://postgres:postgres@127.0.0.1:65432/stem_test" >> "$GITHUB_ENV" - echo "REDIS_URL=redis://127.0.0.1:56379/0" >> "$GITHUB_ENV" diff --git a/.github/workflows/aggregate.yaml b/.github/workflows/aggregate.yaml new file mode 100644 index 00000000..c75b92d2 --- /dev/null +++ b/.github/workflows/aggregate.yaml @@ -0,0 +1,102 @@ +name: aggregate + +on: + push: + branches: [master, main] + pull_request: + branches: [master, main] + workflow_dispatch: + +permissions: + contents: read + +jobs: + repository-hygiene: + name: repository / hygiene + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Reject committed certificate and key fixtures + shell: bash + run: | + set -euo pipefail + matches="$(find packages -type f \( \ + -name '*.key' -o -name '*.pem' -o -name '*.crt' -o \ + -name '*.srl' -o -name '*.cnf' \ + \) -print)" + if [[ -n "$matches" ]]; then + printf 'Certificate/key-like files must be generated during test setup:\n%s\n' "$matches" + exit 1 + fi + - name: Reject duplicate legacy PostgreSQL CA path + shell: bash + run: | + set -euo pipefail + test ! -e packages/stem_cli/docker/testing/certs/postgres-root.crt + ! rg -n 'docker/testing/certs|certs/postgres-root' \ + packages .gitignore + + dagger-test: + name: integration / Dagger all gate + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + # Local `task test:dagger` and CI both call the same Go SDK module. + - name: Run Dagger-backed all-package test gate + uses: dagger/dagger-for-github@v8.4.1 + with: + version: v0.21.7 + module: .dagger + call: all --source=. + dagger-flags: --progress plain + + cross-platform-packages: + name: portable package / ${{ matrix.package }} / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [windows-latest, macos-latest] + include: + - package: packages/stem + tool: dart + test_args: --exclude-tags soak --fail-fast + - package: packages/stem_builder + tool: dart + test_args: --fail-fast + - package: packages/stem_sqlite + tool: dart + test_args: --fail-fast + - package: packages/stem_flutter + tool: flutter + test_args: --fail-fast + - package: packages/stem_flutter_sqlite + tool: flutter + test_args: --fail-fast + - package: packages/stem_memory + tool: dart + test_args: --fail-fast + - package: packages/stem_adapter_tests + tool: dart + test_args: --fail-fast + steps: + - uses: actions/checkout@v4 + # The root is a Dart workspace that also contains Flutter packages. Use + # Flutter's pub frontend for every portable package so Dart-only entries + # can resolve the workspace on both runners as well. + - uses: subosito/flutter-action@v2 + with: + channel: stable + cache: true + - name: Resolve package dependencies + run: flutter pub get + working-directory: ${{ matrix.package }} + - name: Check package formatting + run: dart format lib test --set-exit-if-changed + working-directory: ${{ matrix.package }} + - name: Analyze package + run: ${{ matrix.tool }} analyze --fatal-infos + working-directory: ${{ matrix.package }} + - name: Test package + run: ${{ matrix.tool }} test ${{ matrix.test_args }} + working-directory: ${{ matrix.package }} diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml new file mode 100644 index 00000000..cc490c94 --- /dev/null +++ b/.github/workflows/benchmarks.yaml @@ -0,0 +1,41 @@ +name: benchmarks + +on: + schedule: + - cron: '17 3 * * 1' + workflow_dispatch: + +permissions: + contents: read + +jobs: + core-throughput: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dart-lang/setup-dart@v1 + with: + sdk: 3.10.0 + - name: Resolve dependencies + run: dart pub get + - name: Check benchmark formatting + run: dart format benchmark --set-exit-if-changed + - name: Run throughput regression gate + run: dart run benchmark/stem_throughput.dart --check-baseline + + sqlite-throughput: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dart-lang/setup-dart@v1 + with: + sdk: 3.10.0 + - name: Resolve SQLite package dependencies + working-directory: packages/stem_sqlite + run: dart pub get + - name: Check SQLite benchmark formatting + working-directory: packages/stem_sqlite + run: dart format benchmark --set-exit-if-changed + - name: Run SQLite throughput regression gate + working-directory: packages/stem_sqlite + run: dart run benchmark/sqlite_throughput.dart --check-baseline diff --git a/.github/workflows/dashboard.yaml b/.github/workflows/dashboard.yaml deleted file mode 100644 index cdc368dc..00000000 --- a/.github/workflows/dashboard.yaml +++ /dev/null @@ -1,44 +0,0 @@ -name: dashboard - -on: - push: - branches: - - master - - main - paths: - - 'packages/dashboard/**' - - 'packages/stem/**' - - 'pubspec.yaml' - pull_request: - paths: - - 'packages/dashboard/**' - - 'packages/stem/**' - - 'pubspec.yaml' - workflow_dispatch: - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y sqlite3 libsqlite3-dev - - uses: ./.github/actions/setup-test-services - - uses: subosito/flutter-action@v2 - with: - channel: stable - cache: true - - name: Get Dart packages cache - uses: actions/cache@v4 - with: - path: ~/.pub-cache - key: ${{ runner.os }}-pub-${{ hashFiles('**/pubspec.yaml') }} - restore-keys: | - ${{ runner.os }}-pub- - - name: Install dependencies - run: flutter pub get --offline - continue-on-error: true - - name: Run dashboard tests - run: echo "Dashboard tests skipped - external dependencies unavailable in CI" diff --git a/.github/workflows/docusaurus-site.yaml b/.github/workflows/docusaurus-site.yaml index 2d20a7c2..1ac60922 100644 --- a/.github/workflows/docusaurus-site.yaml +++ b/.github/workflows/docusaurus-site.yaml @@ -4,13 +4,18 @@ on: push: branches: - master + - main + pull_request: + branches: + - master + - main workflow_dispatch: jobs: build-docs: runs-on: ubuntu-latest permissions: - contents: write + contents: read defaults: run: working-directory: .site @@ -28,11 +33,33 @@ jobs: - name: Install dependencies run: npm ci + - name: Type-check site configuration + run: npm run typecheck + - name: Build Docusaurus site run: npm run build + - name: Store generated site + uses: actions/upload-artifact@v4 + with: + name: docusaurus-site + path: .site/build + if-no-files-found: error + + deploy-docs: + needs: build-docs + if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main') + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Download generated site + uses: actions/download-artifact@v4 + with: + name: docusaurus-site + path: .site/build + - name: Deploy to GitHub Pages branch - if: github.ref == 'refs/heads/master' uses: peaceiris/actions-gh-pages@v4 with: github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index ab51f9fc..abd579ca 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -1,22 +1,41 @@ name: Publish -# When a pull request is opened or updated the job will run the validation -# steps (using firehose, this checks for correct pubspec versioning, changelog entries, etc.). -# When you push a tag (for a package), firehose will attempt to publish that package. +# Pull requests and release tags run Stem's repository-owned release gate. +# Tags then continue through Dart's trusted publishing workflow. on: pull_request: branches: [ master ] types: [ opened, synchronize, reopened, labeled, unlabeled ] push: # For mono repos the tag must be prefixed with the package name, - # e.g. "routed-v1.0.0". Adjust the regular expression as needed. + # e.g. "stem-v0.3.0". Every published package uses this form. tags: - - 'v[0-9]+.[0-9]+.[0-9]+' - - '[A-z]+-v[0-9]+.[0-9]+.[0-9]+' + - '[A-Za-z0-9_]+-v[0-9]+.[0-9]+.[0-9]+' workflow_dispatch: jobs: + release-gate: + name: release gate + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: dart-lang/setup-dart@v1 + with: + sdk: 3.10.0 + - uses: subosito/flutter-action@v2 + with: + channel: stable + cache: true + - name: Resolve workspace dependencies + run: dart pub get + - name: Validate the complete release train + run: dart run tool/publish.dart --skip-published --include-unchanged + publish: + needs: release-gate + if: startsWith(github.ref, 'refs/tags/') && needs.release-gate.result == 'success' # This job uses the common publish workflow provided by the ecosystem. # See https://github.com/dart-lang/ecosystem/wiki/Publishing-automation uses: dart-lang/ecosystem/.github/workflows/publish.yaml@main diff --git a/.github/workflows/stem.yaml b/.github/workflows/stem.yaml deleted file mode 100644 index 976a65d6..00000000 --- a/.github/workflows/stem.yaml +++ /dev/null @@ -1,50 +0,0 @@ -name: stem - -on: - push: - branches: - - master - - main - paths: - - 'packages/stem/**' - - 'packages/stem_adapter_tests/**' - - 'pubspec.yaml' - pull_request: - paths: - - 'packages/stem/**' - - 'packages/stem_adapter_tests/**' - - 'pubspec.yaml' - workflow_dispatch: - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y sqlite3 libsqlite3-dev lcov - - uses: ./.github/actions/setup-test-services - - uses: subosito/flutter-action@v2 - with: - channel: stable - cache: true - - name: Get Dart packages cache - uses: actions/cache@v4 - with: - path: ~/.pub-cache - key: ${{ runner.os }}-pub-${{ hashFiles('**/pubspec.yaml') }} - restore-keys: | - ${{ runner.os }}-pub- - - name: Install dependencies - run: flutter pub get - - name: Run stem package tests - working-directory: packages/stem - run: dart test --exclude-tags soak --coverage=coverage - - name: Format coverage - working-directory: packages/stem - run: dart run coverage:format_coverage --lcov --in=coverage --out=coverage/lcov.info --report-on=lib - - name: Check coverage threshold - working-directory: packages/stem - run: dart run ../../tool/coverage/coverage_badge.dart --lcov coverage/lcov.info --out coverage/coverage.json --min 75 diff --git a/.github/workflows/stem_cli.yaml b/.github/workflows/stem_cli.yaml deleted file mode 100644 index f018a09a..00000000 --- a/.github/workflows/stem_cli.yaml +++ /dev/null @@ -1,54 +0,0 @@ -name: stem_cli - -on: - push: - branches: - - master - - main - paths: - - 'packages/stem_cli/**' - - 'packages/stem/**' - - 'packages/stem_adapter_tests/**' - - 'pubspec.yaml' - pull_request: - paths: - - 'packages/stem_cli/**' - - 'packages/stem/**' - - 'packages/stem_adapter_tests/**' - - 'pubspec.yaml' - workflow_dispatch: - -jobs: - test: - runs-on: ubuntu-latest - env: - STEM_CLI_RUN_MULTI: 'false' - steps: - - uses: actions/checkout@v4 - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y sqlite3 libsqlite3-dev lcov - - uses: ./.github/actions/setup-test-services - - uses: subosito/flutter-action@v2 - with: - channel: stable - cache: true - - name: Get Dart packages cache - uses: actions/cache@v4 - with: - path: ~/.pub-cache - key: ${{ runner.os }}-pub-${{ hashFiles('**/pubspec.yaml') }} - restore-keys: | - ${{ runner.os }}-pub- - - name: Install dependencies - run: flutter pub get - - name: Run stem_cli tests - working-directory: packages/stem_cli - run: dart test --coverage=coverage - - name: Format coverage - working-directory: packages/stem_cli - run: dart run coverage:format_coverage --lcov --in=coverage --out=coverage/lcov.info --report-on=lib - - name: Check coverage threshold - working-directory: packages/stem_cli - run: dart run ../../tool/coverage/coverage_badge.dart --lcov coverage/lcov.info --out coverage/coverage.json --min 50 diff --git a/.github/workflows/stem_flutter.yaml b/.github/workflows/stem_flutter.yaml deleted file mode 100644 index 6f8b7c91..00000000 --- a/.github/workflows/stem_flutter.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: stem_flutter - -on: - push: - branches: - - master - - main - paths: - - 'packages/stem_flutter/**' - - 'packages/stem/**' - - 'pubspec.yaml' - pull_request: - paths: - - 'packages/stem_flutter/**' - - 'packages/stem/**' - - 'pubspec.yaml' - workflow_dispatch: - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: subosito/flutter-action@v2 - with: - channel: stable - cache: true - - name: Get Dart packages cache - uses: actions/cache@v4 - with: - path: ~/.pub-cache - key: ${{ runner.os }}-pub-${{ hashFiles('**/pubspec.yaml') }} - restore-keys: | - ${{ runner.os }}-pub- - - name: Install dependencies - run: flutter pub get - - name: Analyze stem_flutter - working-directory: packages/stem_flutter - run: flutter analyze - - name: Run stem_flutter tests - working-directory: packages/stem_flutter - run: flutter test diff --git a/.github/workflows/stem_flutter_sqlite.yaml b/.github/workflows/stem_flutter_sqlite.yaml deleted file mode 100644 index af3efb3e..00000000 --- a/.github/workflows/stem_flutter_sqlite.yaml +++ /dev/null @@ -1,50 +0,0 @@ -name: stem_flutter_sqlite - -on: - push: - branches: - - master - - main - paths: - - 'packages/stem_flutter_sqlite/**' - - 'packages/stem_flutter/**' - - 'packages/stem_sqlite/**' - - 'packages/stem/**' - - 'pubspec.yaml' - pull_request: - paths: - - 'packages/stem_flutter_sqlite/**' - - 'packages/stem_flutter/**' - - 'packages/stem_sqlite/**' - - 'packages/stem/**' - - 'pubspec.yaml' - workflow_dispatch: - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y sqlite3 libsqlite3-dev - - uses: subosito/flutter-action@v2 - with: - channel: stable - cache: true - - name: Get Dart packages cache - uses: actions/cache@v4 - with: - path: ~/.pub-cache - key: ${{ runner.os }}-pub-${{ hashFiles('**/pubspec.yaml') }} - restore-keys: | - ${{ runner.os }}-pub- - - name: Install dependencies - run: flutter pub get - - name: Analyze stem_flutter_sqlite - working-directory: packages/stem_flutter_sqlite - run: flutter analyze - - name: Run stem_flutter_sqlite tests - working-directory: packages/stem_flutter_sqlite - run: flutter test diff --git a/.github/workflows/stem_memory.yaml b/.github/workflows/stem_memory.yaml deleted file mode 100644 index 9ddc67e9..00000000 --- a/.github/workflows/stem_memory.yaml +++ /dev/null @@ -1,47 +0,0 @@ -name: stem_memory - -on: - push: - branches: - - master - - main - paths: - - 'packages/stem_memory/**' - - 'packages/stem/**' - - 'packages/stem_adapter_tests/**' - - 'pubspec.yaml' - pull_request: - paths: - - 'packages/stem_memory/**' - - 'packages/stem/**' - - 'packages/stem_adapter_tests/**' - - 'pubspec.yaml' - workflow_dispatch: - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: subosito/flutter-action@v2 - with: - channel: stable - cache: true - - name: Get Dart packages cache - uses: actions/cache@v4 - with: - path: ~/.pub-cache - key: ${{ runner.os }}-pub-${{ hashFiles('**/pubspec.yaml') }} - restore-keys: | - ${{ runner.os }}-pub- - - name: Install dependencies - run: flutter pub get - - name: Run stem_memory tests - working-directory: packages/stem_memory - run: dart test --coverage=coverage --fail-fast - - name: Format coverage - working-directory: packages/stem_memory - run: dart run coverage:format_coverage --lcov --in=coverage --out=coverage/lcov.info --report-on=lib - - name: Check coverage threshold - working-directory: packages/stem_memory - run: dart run ../../tool/coverage/coverage_badge.dart --lcov coverage/lcov.info --out coverage/coverage.json --min 80 diff --git a/.github/workflows/stem_postgres.yaml b/.github/workflows/stem_postgres.yaml deleted file mode 100644 index 505d464d..00000000 --- a/.github/workflows/stem_postgres.yaml +++ /dev/null @@ -1,52 +0,0 @@ -name: stem_postgres - -on: - push: - branches: - - master - - main - paths: - - 'packages/stem_postgres/**' - - 'packages/stem/**' - - 'packages/stem_adapter_tests/**' - - 'pubspec.yaml' - pull_request: - paths: - - 'packages/stem_postgres/**' - - 'packages/stem/**' - - 'packages/stem_adapter_tests/**' - - 'pubspec.yaml' - workflow_dispatch: - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y sqlite3 libsqlite3-dev lcov - - uses: ./.github/actions/setup-test-services - - uses: subosito/flutter-action@v2 - with: - channel: stable - cache: true - - name: Get Dart packages cache - uses: actions/cache@v4 - with: - path: ~/.pub-cache - key: ${{ runner.os }}-pub-${{ hashFiles('**/pubspec.yaml') }} - restore-keys: | - ${{ runner.os }}-pub- - - name: Install dependencies - run: flutter pub get - - name: Run stem_postgres tests - working-directory: packages/stem_postgres - run: dart test --coverage=coverage - - name: Format coverage - working-directory: packages/stem_postgres - run: dart run coverage:format_coverage --lcov --in=coverage --out=coverage/lcov.info --report-on=lib - - name: Check coverage threshold - working-directory: packages/stem_postgres - run: dart run ../../tool/coverage/coverage_badge.dart --lcov coverage/lcov.info --out coverage/coverage.json --min 45 diff --git a/.github/workflows/stem_redis.yaml b/.github/workflows/stem_redis.yaml deleted file mode 100644 index 2a79a745..00000000 --- a/.github/workflows/stem_redis.yaml +++ /dev/null @@ -1,52 +0,0 @@ -name: stem_redis - -on: - push: - branches: - - master - - main - paths: - - 'packages/stem_redis/**' - - 'packages/stem/**' - - 'packages/stem_adapter_tests/**' - - 'pubspec.yaml' - pull_request: - paths: - - 'packages/stem_redis/**' - - 'packages/stem/**' - - 'packages/stem_adapter_tests/**' - - 'pubspec.yaml' - workflow_dispatch: - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y sqlite3 libsqlite3-dev lcov - - uses: ./.github/actions/setup-test-services - - uses: subosito/flutter-action@v2 - with: - channel: stable - cache: true - - name: Get Dart packages cache - uses: actions/cache@v4 - with: - path: ~/.pub-cache - key: ${{ runner.os }}-pub-${{ hashFiles('**/pubspec.yaml') }} - restore-keys: | - ${{ runner.os }}-pub- - - name: Install dependencies - run: flutter pub get - - name: Run stem_redis tests - working-directory: packages/stem_redis - run: dart test --coverage=coverage - - name: Format coverage - working-directory: packages/stem_redis - run: dart run coverage:format_coverage --lcov --in=coverage --out=coverage/lcov.info --report-on=lib - - name: Check coverage threshold - working-directory: packages/stem_redis - run: dart run ../../tool/coverage/coverage_badge.dart --lcov coverage/lcov.info --out coverage/coverage.json --min 75 diff --git a/.github/workflows/stem_sqlite.yaml b/.github/workflows/stem_sqlite.yaml deleted file mode 100644 index 4c20e39c..00000000 --- a/.github/workflows/stem_sqlite.yaml +++ /dev/null @@ -1,52 +0,0 @@ -name: stem_sqlite - -on: - push: - branches: - - master - - main - paths: - - 'packages/stem_sqlite/**' - - 'packages/stem/**' - - 'packages/stem_adapter_tests/**' - - 'pubspec.yaml' - pull_request: - paths: - - 'packages/stem_sqlite/**' - - 'packages/stem/**' - - 'packages/stem_adapter_tests/**' - - 'pubspec.yaml' - workflow_dispatch: - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y sqlite3 libsqlite3-dev lcov - - uses: ./.github/actions/setup-test-services - - uses: subosito/flutter-action@v2 - with: - channel: stable - cache: true - - name: Get Dart packages cache - uses: actions/cache@v4 - with: - path: ~/.pub-cache - key: ${{ runner.os }}-pub-${{ hashFiles('**/pubspec.yaml') }} - restore-keys: | - ${{ runner.os }}-pub- - - name: Install dependencies - run: flutter pub get - - name: Run stem_sqlite tests - working-directory: packages/stem_sqlite - run: dart test --coverage=coverage - - name: Format coverage - working-directory: packages/stem_sqlite - run: dart run coverage:format_coverage --lcov --in=coverage --out=coverage/lcov.info --report-on=lib - - name: Check coverage threshold - working-directory: packages/stem_sqlite - run: dart run ../../tool/coverage/coverage_badge.dart --lcov coverage/lcov.info --out coverage/coverage.json --min 50 diff --git a/.gitignore b/.gitignore index d7c1f300..cedf6d84 100644 --- a/.gitignore +++ b/.gitignore @@ -5,8 +5,12 @@ pubspec.lock build/ **/.flutter-plugins-dependencies textwrap/ -examples/microservice/certs/ -examples/security/*/certs/ +packages/stem/example/microservice/certs/ +packages/stem/example/postgres_tls/certs/ +packages/stem/example/security/*/certs/ +packages/stem_cli/docker/testing/postgres/certs/*.crt +packages/stem_cli/docker/testing/postgres/certs/*.key +packages/stem_cli/docker/testing/postgres/certs/*.srl packages/dashboard/storage/ .beads/ mcp_agent_mail/ diff --git a/.site/docs/about/index.md b/.site/docs/about/index.md new file mode 100644 index 00000000..f2074919 --- /dev/null +++ b/.site/docs/about/index.md @@ -0,0 +1,16 @@ +--- +title: About Stem +slug: /about +sidebar_position: 0 +--- + +Stem is an experimental Dart-native background work platform. This section +explains the project's current scope and the work that is deliberately being +prioritised before new platform features. + +- **[Roadmap](./roadmap.md)** – current hardening work, proven capabilities and + deliberately deferred features. + +Stem is not presented as production-ready. Production use requires validating +the selected broker, backend, failure semantics and operational procedures for +the workload being deployed. diff --git a/.site/docs/about/roadmap.md b/.site/docs/about/roadmap.md new file mode 100644 index 00000000..79e83eb0 --- /dev/null +++ b/.site/docs/about/roadmap.md @@ -0,0 +1,90 @@ +--- +title: Roadmap +sidebar_label: Roadmap +slug: /about/roadmap +sidebar_position: 1 +--- + +## Direction + +The next release cycle is a hardening cycle. Stem already has a broad set of +queue, worker, scheduler, workflow and adapter features; the priority is to +narrow the public surface, make release artifacts reproducible and prove +failure behaviour before expanding the platform. + +## Delivered foundations + +These capabilities exist in the repository and are covered by package tests, +adapter contract tests or integration tests as applicable: + +- Typed task definitions, DTO codecs and generated task/workflow definitions. +- Redis, Postgres, SQLite and in-memory broker/backend implementations. +- Explicit worker lifecycle, isolate execution and documented timeout modes. +- Retries, leases, acknowledgements, uniqueness coordination and revocation. +- Durable workflow checkpoints, suspension/resumption and external events. +- Scheduler time zones, jitter, persistence, leases and CLI control commands. +- OpenTelemetry context propagation and task/result instrumentation. +- Redis/Postgres distributed rate-limiter implementations and a Postgres + transactional outbox integration. +- Adapter contract suites, standalone package staging and workspace example + checks in CI. +- Atomic terminal-result arbitration in the built-in result backends, with + worker tests covering lease-loss redelivery and late cross-worker + completion. +- SQLite workflow persistence now has a restart-recovery integration test that + reopens the store and resumes a checkpoint with replacement runtime and + broker instances. +- SQLite and Postgres migration suites now cover current adapters reading + legacy-shaped queue or lock records after upgrade. +- Core in-memory and SQLite file-backed throughput benchmarks now have checked + minimum baselines and scheduled CI regression gates. + +The existence of a feature does not mean that every adapter offers identical +guarantees. Read the broker and backend caveats before choosing a deployment. + +## Current hardening + +Work in this phase is focused on evidence and boundaries: + +1. Exercise crash, lease-loss, duplicate-delivery, retry-storm and shutdown + interleavings with failure-injection and soak tests. +2. Test schema upgrades, mixed-version workers and standalone package + resolution without workspace dependency overrides. +3. Stabilise the public entrypoints and keep low-level compatibility APIs out + of the recommended onboarding path. +4. Publish reproducible benchmark baselines for queue throughput, SQLite + contention, workflow checkpoints and adapter recovery. +5. Document adapter-specific delivery, lease, delay, priority and recovery + guarantees. + +## Deliberately deferred + +The following are not release priorities while the reliability work is in +progress: + +- Compensation/saga primitives for workflows. +- Additional broker integrations. +- A broader dashboard product; the current dashboard remains experimental. +- Exactly-once execution claims for external side effects. +- More Canvas policies or composition features beyond the semantics already + implemented. + +## Contribution gate + +Before proposing a release-facing change, run the package checks relevant to +the change, including: + +```sh +dart run tool/check_examples.dart --skip-diff +dart run tool/publish.dart --plan +``` + +For package-level work, also run `dart format`, `dart analyze --fatal-infos` +and the package test suite. Changes to generated definitions must leave the +working tree clean after generation. + +## Status language + +Stem should be described as experimental until the project has accumulated +long-running fault-test evidence, compatibility guarantees, migration stories +and recovery reports for the supported adapters. diff --git a/.site/docs/about/testing.md b/.site/docs/about/testing.md new file mode 100644 index 00000000..c03f4e26 --- /dev/null +++ b/.site/docs/about/testing.md @@ -0,0 +1,71 @@ +--- +title: Testing and Quality Gates +sidebar_label: Testing +sidebar_position: 2 +slug: /about/testing +--- + +Stem treats tests as evidence for runtime guarantees, not as a substitute for +adapter-specific operational validation. Run the checks relevant to the +package and behavior you changed. + +## Core development loop + +From the repository root: + +```sh +dart pub get +dart run tool/check_examples.dart --skip-diff +``` + +Then run the core package gates: + +```sh +cd packages/stem +dart format lib test --set-exit-if-changed +dart analyze --fatal-infos +dart test --exclude-tags soak --fail-fast +``` + +Examples are part of the user-facing API. The example checker discovers every +example under the root workspace packages, resolves its dependencies, runs +code generation where needed and analyzes the result. + +## Package and adapter validation + +Run `dart format`, `dart analyze --fatal-infos` and the test suite for every +package affected by a change. Adapter changes should include the shared +contract tests and, when infrastructure is available, the real Redis or +Postgres integration tests. + +The adapters do not all provide identical delivery, lease, delay or priority +guarantees. Read the broker caveats and test the selected deployment rather +than treating an in-memory pass as proof of distributed recovery. + +## Reliability tests + +Soak tests are tagged and excluded from the normal fast suite: + +```sh +cd packages/stem +dart test --tags soak +``` + +Failure-oriented tests should cover duplicate delivery, acknowledgement +failure, lease loss, retry storms, worker shutdown, checkpoint recovery and +broker restart. Changes to persistence or scheduling should also run migration +upgrade and lease/fencing tests. + +## Release gate + +The release planner derives the package graph from workspace manifests and +checks package metadata, generated output, standalone resolution and publish +archives: + +```sh +dart run tool/publish.dart --plan +``` + +A real release requires a clean tree and the exact commit used to produce the +artifacts. The aggregate CI workflow repeats package checks on Linux, Windows +and macOS, tests standalone source staging, and checks workspace examples. diff --git a/.site/docs/brokers/caveats.md b/.site/docs/brokers/caveats.md index d2cf17cb..36d8b51c 100644 --- a/.site/docs/brokers/caveats.md +++ b/.site/docs/brokers/caveats.md @@ -9,6 +9,72 @@ This page highlights broker-specific constraints that affect routing, priorities and control-plane behavior. These caveats are based on the adapter implementations. +## Capability interfaces + +The broad `Broker` contract remains the compatibility surface for existing +adapters. New integrations should expose only the optional capabilities they +actually implement: + +- `QueueBroker` — publish, consume, acknowledge, negative acknowledge, and + close. +- `LeaseBroker` — extend an active delivery lease. +- `InspectableBroker` — pending and in-flight queue counts. +- `DeadLetterBroker` — dead-letter, list, retrieve, replay, and purge dead + letters. + +`BrokerCapabilities` defaults optional behavior to `false`. Built-in adapters +explicitly declare lease extension and dead-letter support. A queue-only +adapter therefore remains valid: terminal failures are persisted in the +result backend and discarded without requeueing when no dead-letter store is +available. Built-in adapters advertise `atLeastOnce`; applications must still +make external side effects idempotent because a crash after side-effect +completion and before acknowledgement can produce a duplicate delivery. + +Built-in adapters declare these interfaces in addition to `Broker`. A new +queue-only adapter can implement `QueueBroker` without implementing the +optional operations; consumers that need one can check its capability +interface or use the adapter's `BrokerCapabilities` snapshot instead of +assuming every transport has identical operational semantics. The historical +`Broker` facade supplies compatibility defaults that throw +`UnsupportedError` for operations an adapter does not support. + +## Delivery and recovery matrix + +The lease values below are configuration defaults. Actual recovery time also +includes the adapter's poll, claim, or sweeper interval and any broker/network +latency. + +| Adapter | Lease creation | Expired-delivery recovery | Delivery guarantee | +| --- | --- | --- | --- | +| In-memory | `defaultVisibilityTimeout` | In-process claim timer (`claimInterval`) | At least once while the process is alive | +| SQLite | `defaultVisibilityTimeout` and row lock | Expiry sweeper (`sweeperInterval`) plus polling (`pollInterval`) | At least once for durable queue rows | +| Redis Streams | `defaultVisibilityTimeout` and consumer-group pending entry | `XAUTOCLAIM` on `claimInterval` | At least once while Redis data is retained | +| Postgres | `defaultVisibilityTimeout` and `locked_until` | Expiry sweeper (`sweeperInterval`) plus polling (`pollInterval`) | At least once for committed queue rows | + +An automatic lease renewal is scheduled at roughly half of the remaining +lease. Short leases use that shorter interval even when the normal minimum is +one second. A renewal failure is transient from the worker's perspective: +later attempts continue, but a lease that remains expired may be delivered to +another worker. Set the visibility timeout long enough for the handler's +normal execution, terminal result handling, and broker round-trip time; +renewal starts when a delivery enters the worker and remains active through +acknowledgement. This also covers slow consume middleware and pre-execution +validation. Renewal is a safety mechanism, not a substitute for idempotent +side effects. + +The worker suppresses a duplicate envelope that arrives while its original +delivery is active in that same process. A process crash, or the same task +running on different workers, remains at-least-once behavior and must be +handled with idempotent application effects. + +Built-in result backends also expose `AtomicTerminalResultBackend`. Worker +terminal writes use it to ensure that a concurrent late completion cannot +replace an already persisted terminal state. This arbitration protects the +stored task result and worker-owned terminal side effects; it does not make +external HTTP calls, emails, or other application side effects exactly once. +Custom backends may omit the capability, in which case terminal persistence +uses the legacy unconditional `ResultBackend.set` path. + ## In-memory broker - **No priority buckets**: `supportsPriority` is false, so priorities are not @@ -51,8 +117,8 @@ implementations. - **Single-queue consumption**: only one queue can be consumed per subscription. - **Polling-based delivery**: workers poll for due jobs on an interval. - **Visibility timeouts**: tasks are locked with a `locked_until` lease; if a - worker dies or stops heartbeating, jobs become visible again after the lease - expires. + worker dies or stops renewing its lease, jobs become visible again after the + lease expires. - **Dead letter retention**: dead letters are retained for a default window (7 days) unless configured otherwise. - **Broadcast channels**: broadcasts are stored in a separate table and read @@ -71,8 +137,12 @@ implementations. requeues). If a worker stops without acking a delivery, the task becomes visible again after the visibility lease expires (Redis reclaim interval / Postgres `locked_until`). -- **Long-running tasks** should emit heartbeats or extend leases so the broker - does not re-deliver them mid-execution. +- **Long-running tasks** should emit heartbeats for liveness and rely on the + worker's automatic renewal or call `context.extendLease(...)` when they need + explicit lease time. A heartbeat alone does not extend the broker lease. +- **Renewal failures** are exposed through `stem.lease.renewal_failed` and + structured worker logs. Renewal attempts continue after a transient failure; + a lease that remains lost can still produce an at-least-once redelivery. ## Tips diff --git a/.site/docs/brokers/sqlite.md b/.site/docs/brokers/sqlite.md index 48aa5ebd..99289fe6 100644 --- a/.site/docs/brokers/sqlite.md +++ b/.site/docs/brokers/sqlite.md @@ -26,7 +26,7 @@ Add the adapter package: ```yaml dependencies: - stem_sqlite: ^0.1.1 + stem_sqlite: ^0.2.0 ``` ## Quick start (broker) @@ -80,6 +80,12 @@ contention: - **Keep producers off the backend** (let workers be the only writers). - **Do not share a single SQLite file** between broker and backend. +Stem serializes its own transactional broker/backend mutations when multiple +handles point at the same file, which prevents savepoint corruption and +in-process writer races. This coordination does not cover unrelated processes +or tools opening the file, and it cannot turn SQLite into a multi-host queue. +Use separate files when throughput matters. + A simple layout: ``` @@ -116,6 +122,9 @@ SQLite brokers are intentionally minimal: - **Single-queue subscriptions only** (one queue per worker subscription). - **Polling-based delivery** (latency depends on `pollInterval`). - **Single-writer constraint** (plan your processes and DB files accordingly). +- Stem coordinates in-process writes across its broker, backend, workflow, and + control handles, but external writers and separate processes still need + SQLite-compatible locking discipline. If you need cross-process broadcast control, multi-queue consumption, or multi-host scaling, diff --git a/.site/docs/core-concepts/canvas.md b/.site/docs/core-concepts/canvas.md index 18573185..8088fb58 100644 --- a/.site/docs/core-concepts/canvas.md +++ b/.site/docs/core-concepts/canvas.md @@ -3,15 +3,16 @@ title: Canvas Patterns sidebar_label: Canvas sidebar_position: 5 slug: /core-concepts/canvas +description: Compose task work with typed chains, groups, batches and chords. --- This guide walks through Stem's task composition primitives—chains, groups, and chords—using in-memory brokers and backends. Each snippet references a runnable file under `packages/stem/example/docs_snippets/` so you can experiment locally with `dart run`. If you bootstrap with `StemApp`, use `app.canvas` to reuse the -same broker, backend, task handlers, and encoder registry. `StemApp` lazy-starts -its managed worker for canvas dispatch too, so the common path does not need an -explicit `await app.start()`. +same broker, backend, task handlers, and encoder registry. Start the worker +explicitly in processes that should consume work; constructing a canvas or +inspecting its state never starts execution as a side effect. ## Chains @@ -26,6 +27,33 @@ Chains execute tasks serially. Each step receives the previous result via If any step fails, the chain stops immediately. Retry by invoking `canvas.chain` again with the same signatures. +For heterogeneous transitions, use the typed fluent API. Each `then` accepts a +`TaskDefinition` whose argument type must match the previous result type, and +the final result is decoded into the last task's result type: + +```dart +final download = TaskDefinition.codec( + name: 'download', + argsCodec: downloadRequestCodec, + resultCodec: downloadResultCodec, +); +final resize = TaskDefinition.codec( + name: 'resize', + argsCodec: downloadResultCodec, + resultCodec: resizeResultCodec, +); + +final result = await canvas + .typedChain(download, request) + .then(resize) + .run(); +``` + +The compiler rejects a `then` whose argument type does not accept the previous +task's result. The existing homogeneous `Canvas.chain` remains available for +raw signatures and migration compatibility; it transports the previous result +through `chainPrevResult`. + ## Groups Groups fan out work and persist each branch in the result backend. @@ -49,8 +77,9 @@ state: ## Chords -Chords combine a group with a callback. Once all body tasks succeed, the -callback runs with `context.meta['chordResults']` populated. Prefer +Chords combine a group with a callback. By default, once all body tasks +succeed, the callback runs with `context.meta['chordResults']` populated. +Prefer `context.meta.valueListOr('chordResults', const [])` over manual list casts when reading those results. @@ -58,10 +87,27 @@ when reading those results. ``` -If any branch fails, the callback is skipped and the chord group is marked as -failed. Inspect the latest group status via `StemApp.getGroupStatus(...)` or -`StemClient.getGroupStatus(...)` before retrying. If you are operating below -the runtime layer, read the raw backend directly. +The default `ChordPolicy.allOrFail()` skips the callback when any branch fails. +Use an explicit policy when a callback should receive terminal failures: + +```dart +await canvas.chord( + body: body, + callback: summarize, + policy: const ChordPolicy.collectTerminalResults(), +); +``` + +`ChordPolicy.collectTerminalResults()` waits for every body task and passes +`null` in `chordResults` for failed or cancelled branches. The callback also +receives failure summaries in `context.meta['stem.chord.failures']`. +`ChordPolicy.allowPartial(minSuccessful: 2)` has the same terminal-result +behavior but dispatches only when the required number of body tasks succeeded. +If the policy cannot be satisfied, the callback is skipped and the chord +operation fails. Inspect the latest group status via +`StemApp.getGroupStatus(...)` or `StemClient.getGroupStatus(...)` before +retrying. If you are operating below the runtime layer, read the raw backend +directly. ## Dependency semantics @@ -69,8 +115,8 @@ the runtime layer, read the raw backend directly. the previous one succeeds. - **Groups** model fan-out dependencies: a group is “complete” once all child tasks finish. The expected count is stored in the backend. -- **Chords** combine both: a callback depends on the entire group finishing - successfully. +- **Chords** combine both: a callback depends on the entire group reaching a + terminal state and the configured `ChordPolicy` being satisfied. ## Child result retrieval diff --git a/.site/docs/core-concepts/cli-control.md b/.site/docs/core-concepts/cli-control.md index 4084d11b..ec2cb4d2 100644 --- a/.site/docs/core-concepts/cli-control.md +++ b/.site/docs/core-concepts/cli-control.md @@ -194,6 +194,7 @@ stem schedule apply \ --yes stem schedule list +stem schedule trigger cleanup stem schedule dry-run --spec "every:5m" ``` @@ -205,8 +206,10 @@ Run Beat from a Dart entrypoint wired to your schedule store: Requirements: -- `stem schedule apply/list/dry-run` use `STEM_SCHEDULE_STORE_URL` when set, +- `stem schedule apply/list/trigger/dry-run` use `STEM_SCHEDULE_STORE_URL` when set, otherwise they operate on local schedule files. +- `schedule trigger` also requires `STEM_BROKER_URL` and publishes one task + without changing the recurring entry's next-run or execution counters. - Beat needs a broker, schedule store, and (for HA) a lock store. Expected output (schedule list): @@ -245,6 +248,7 @@ Use this table to sanity-check which connection strings are required: | `stem worker shutdown` | ✅ | ❌ | ❌ | ❌ | ❌ | | `stem worker pause/resume` | ✅ | ❌ | ❌ | optional | ❌ | | `stem schedule apply/list/dry-run` | ❌ | ❌ | ✅ | ❌ | ❌ | +| `stem schedule trigger` | ✅ | optional | ✅ | ❌ | ❌ | | `stem health` | ✅ | optional | ❌ | ❌ | ❌ | Notes: diff --git a/.site/docs/core-concepts/observability.md b/.site/docs/core-concepts/observability.md index 2a590e95..5d55ed42 100644 --- a/.site/docs/core-concepts/observability.md +++ b/.site/docs/core-concepts/observability.md @@ -49,6 +49,11 @@ Traces include spans for `stem.enqueue`, `stem.consume`, and task execution. Use attributes (`stem.task`, `stem.queue`, `stem.retry.attempt`) to filter in your tracing backend. +Canvas fan-out also emits a `stem.canvas.group` composition span. Each group +body's `stem.consume` span links back to that composition span while retaining +its normal task trace parent, so groups and chords remain understandable in +backends that support OpenTelemetry span links. + ## Signals `StemSignals` fire lifecycle hooks for tasks, workers, scheduler events, and @@ -82,13 +87,16 @@ your own tracing/logging systems. class LoggingWorkflowIntrospectionSink implements WorkflowIntrospectionSink { @override Future recordStepEvent(WorkflowStepEvent event) async { - stemLogger.info('workflow.execution', { - 'run': event.runId, - 'workflow': event.workflow, - 'step': event.stepId, - 'type': event.type.name, - 'iteration': event.iteration, - }); + stemLogger.info( + 'workflow.execution', + fields: { + 'run': event.runId, + 'workflow': event.workflow, + 'step': event.stepId, + 'type': event.type.name, + 'iteration': event.iteration, + }, + ); } } ``` @@ -114,14 +122,16 @@ Persisted worker heartbeats expose the same typed decode path on `extras` via ## Logging -Use `stemLogger` (Contextual logger) for structured logs. +Import `package:stem/observability.dart` when you need Stem's structured +logging facade. It accepts Stem-owned severity and field types; the underlying +logging dependency is kept out of the public API. ```dart file=/../packages/stem/example/docs_snippets/lib/observability.dart#observability-logging ``` The shared `stemLogger` starts silent by default, so opt in explicitly with -`configureStemLogging(level: Level.info, format: StemLogFormat.pretty)`. +`configureStemLogging(level: StemLogLevel.info, format: StemLogFormat.pretty)`. When you want machine-oriented output for production log shipping, switch to `configureStemLogging(format: StemLogFormat.plain)`. diff --git a/.site/docs/core-concepts/producer.md b/.site/docs/core-concepts/producer.md index 168b318c..9ca2d2fb 100644 --- a/.site/docs/core-concepts/producer.md +++ b/.site/docs/core-concepts/producer.md @@ -69,6 +69,26 @@ If you later inspect the raw `Envelope`, prefer `envelope.argsJson(...)`, `envelope.argsVersionedJson(...)`, `envelope.metaJson(...)`, or `envelope.metaVersionedJson(...)` over manual map casts. +## Split process roles + +When a process only publishes or observes work, depend on the narrow role +interfaces rather than passing around a full `StemApp`: + +```dart +Future publish(StemProducer producer) async { + await producer.enqueue('tasks.email', args: {'to': 'ops@example.com'}); +} + +Future inspect(StemObserver observer, String taskId) { + return observer.getTaskStatus(taskId); +} +``` + +`StemWorkerHost` is only exposed by managed application wrappers and owns the +explicit `start()`/`shutdown()` lifecycle. A producer or observer reference +does not provide worker lifecycle methods, which keeps API and dashboard +processes from accidentally becoming consumers. + ## Enqueue options Use `TaskEnqueueOptions` to override scheduling, routing, retry behavior, and diff --git a/.site/docs/core-concepts/rate-limiting.md b/.site/docs/core-concepts/rate-limiting.md index e3b29fea..aadcd44a 100644 --- a/.site/docs/core-concepts/rate-limiting.md +++ b/.site/docs/core-concepts/rate-limiting.md @@ -120,19 +120,30 @@ Run the `rate_limit_delay` example for a full demo: - `packages/stem/example/rate_limit_delay` -## Rate limit syntax +## Rate limit values -`rateLimit` accepts short strings like: +In Dart code, use the typed `RateLimit` value object: + +```dart +const TaskOptions( + rateLimit: RateLimit.perMinute(100), + groupRateLimit: RateLimit.perSecond(5), +) +``` + +String values remain supported at JSON/YAML and environment-configuration +boundaries: - `10/s` — 10 tokens per second - `100/m` — 100 tokens per minute - `500/h` — 500 tokens per hour -`groupRateLimit` uses the same syntax. +`groupRateLimit` uses the same syntax. The worker receives a validated +`RateLimit` value rather than parsing strings during task execution. ## How it works -- The worker parses `rateLimit` for each task. +- The worker asks the configured limiter to acquire the typed `rateLimit`. - The worker asks the `RateLimiter` for an acquire decision. - If denied, the task is retried with backoff and `rateLimited=true` metadata. - Retry delays come from the limiter `retryAfter` if provided, otherwise the @@ -157,18 +168,13 @@ Group rate limits share a limiter bucket across related tasks. ## Redis-backed limiter example -The `packages/stem/example/rate_limit_delay` demo ships a Redis fixed-window limiter. It: +The `packages/stem/example/rate_limit_delay` demo uses the shipped Redis +token-bucket limiter. It: - shares tokens across multiple workers, -- logs when a token is granted or denied, +- uses Redis server time and an atomic Lua refill/acquire operation, - reschedules denied tasks with retry metadata. -Inspect it here: - -```dart title="lib/rate_limiter.dart" file=/../packages/stem/example/rate_limit_delay/lib/rate_limiter.dart#rate-limit-redis-limiter - -``` - ## Observability When a task is rate limited: @@ -194,9 +200,10 @@ The `rate_limit_delay` example reads `STEM_RATE_LIMIT_URL` to point the limiter at Redis. Use a dedicated Redis DB or key prefix to keep limiter state isolated from your broker/result backend. -```dart title="lib/shared.dart" file=/../packages/stem/example/rate_limit_delay/lib/shared.dart#rate-limit-redis-connector - -``` +The Redis limiter is constructed with `RedisRateLimiter.connect(...)` from +`stem_redis`. `stem_postgres` provides the equivalent +`PostgresRateLimiter.connect(...)`; it uses a server-clock token bucket with a +row lock inside one transaction. ## Tips diff --git a/.site/docs/core-concepts/stem-builder.md b/.site/docs/core-concepts/stem-builder.md index f745e275..541ee647 100644 --- a/.site/docs/core-concepts/stem-builder.md +++ b/.site/docs/core-concepts/stem-builder.md @@ -194,8 +194,9 @@ final taskApp = await client.createApp(); ``` Plain `StemApp` bootstrap infers task queue subscriptions from the bundled or -explicitly supplied task handlers when `workerConfig.subscription` is omitted, -and it lazy-starts on the first enqueue or wait call. +explicitly supplied task handlers when `workerConfig.subscription` is omitted. +Worker startup is always an explicit lifecycle operation: call `start()` in the +process that is intended to consume work. If you already centralize broker/backend wiring in a `StemClient`, stay on the shared-client path: diff --git a/.site/docs/core-concepts/tasks.md b/.site/docs/core-concepts/tasks.md index b97a0332..9ff80328 100644 --- a/.site/docs/core-concepts/tasks.md +++ b/.site/docs/core-concepts/tasks.md @@ -149,7 +149,9 @@ every retry signal and shows how the strategy interacts with broker timings. `TaskContext` provides metadata and control helpers: - `context.attempt` – current attempt number (0-based). -- `context.heartbeat()` – extend the lease to avoid timeouts. +- `context.heartbeat()` – emit a liveness signal for monitoring. Automatic + lease renewal is handled by the worker; use `context.extendLease(...)` when + task code needs an explicit lease extension. - `context.extendLease(Duration by)` – request additional processing time. - `context.progress(percent, data: {...})` – emit progress signals for UI hooks. - `context.progressJson(percent, dto)` – emit DTO progress payloads without @@ -245,8 +247,21 @@ Set soft/hard timeouts to guard against runaway tasks: - **Soft timeouts** trigger `WorkerEventType.timeout` so you can log or notify. - **Hard timeouts** raise `TimeoutException` to force retries or failure. -- Provide an `isolateEntrypoint` to run the task in a dedicated isolate when - enforcing hard limits or dealing with CPU-intensive code. +- Task handlers declare `TaskExecutionMode.inline` or + `TaskExecutionMode.isolate`. Isolate mode requires a top-level + `isolateEntrypoint`; generated handlers set both values for you. +- Use isolate mode for CPU-intensive code or when enforcing a hard limit. Use + inline mode for handlers that need coordinator-isolate state and can observe + cooperative cancellation. + +Timeout guarantees depend on the execution mode: + +- For isolate-backed handlers, a hard timeout can terminate the execution + isolate, so the timed-out handler does not continue running there. +- For inline handlers, a hard timeout stops the worker from awaiting the + result, but Dart cannot forcibly cancel an arbitrary `Future`; the handler + may continue until it returns. Use `context.cancellation` and cooperative + checkpoints for inline cancellation. ## Idempotency Checklist diff --git a/.site/docs/getting-started/first-steps.md b/.site/docs/getting-started/first-steps.md index 8749265a..1c57f20c 100644 --- a/.site/docs/getting-started/first-steps.md +++ b/.site/docs/getting-started/first-steps.md @@ -19,9 +19,8 @@ Create a task handler (StemApp will register it for you): ## 2. Bootstrap the in-memory runtime -Use `StemApp` to create the broker, backend, and worker in memory. The worker -lazy-starts on the first enqueue or wait call, so the common path does not need -an explicit `await app.start()`: +Use the in-memory shortcut to create the broker, backend, and worker. It does +not start the worker implicitly; start it explicitly before enqueueing work: ```dart file=/../packages/stem/example/docs_snippets/lib/first_steps.dart#first-steps-bootstrap diff --git a/.site/docs/getting-started/intro.md b/.site/docs/getting-started/intro.md index 3d6693a9..0efe52bf 100644 --- a/.site/docs/getting-started/intro.md +++ b/.site/docs/getting-started/intro.md @@ -10,10 +10,10 @@ aliases: import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -Stem is a Dart-native background work platform that gives you Celery‑level -capabilities without leaving the Dart ecosystem. This onboarding path assumes -you have never touched Stem before and walks you from “what is this?” to “I can -ship a production deployment.” +Stem is an experimental Dart-native background work platform with Celery-like +building blocks for queues, workers, and durable workflows. This onboarding +path assumes you have never touched Stem before and walks you from “what is +this?” to “I can evaluate a production-shaped deployment.” ## What is a task queue? diff --git a/.site/docs/getting-started/production-checklist.md b/.site/docs/getting-started/production-checklist.md index 1709d891..9b2f8758 100644 --- a/.site/docs/getting-started/production-checklist.md +++ b/.site/docs/getting-started/production-checklist.md @@ -139,7 +139,8 @@ stem worker diagnose --node web-1 \ Before every deployment run through these guardrails: -- **Quality gates** – run `packages/stem/example/quality_gates` (`just quality`) to execute +- **Quality gates** – run the aggregate workflow or execute `dart format`, + `dart analyze --fatal-infos`, and the package test suites locally to execute format, analyze, unit/chaos/perf tests, and coverage targets. - **Observability** – confirm Grafana dashboards (task success rate, latency p95, queue depth) and OpenTelemetry exporters are healthy. diff --git a/.site/docs/getting-started/reliability.md b/.site/docs/getting-started/reliability.md index e60bec80..ed34f50a 100644 --- a/.site/docs/getting-started/reliability.md +++ b/.site/docs/getting-started/reliability.md @@ -20,14 +20,68 @@ import TabItem from '@theme/TabItem'; ## Broker fetch notes - **Redis Streams** uses consumer groups plus `XAUTOCLAIM` to reclaim idle - deliveries; long-running tasks should emit heartbeats or extend leases. + deliveries; long-running tasks should rely on automatic renewal or explicitly + extend their leases. - **Postgres** uses polling with `locked_until` leases; tasks become visible again after the lease expires. +Worker lease renewal is isolated from the task loop. A failed automatic renewal +is contained, recorded as the `stem.lease.renewal_failed` metric, and logged with +the task, queue, and broker receipt; a shorter retry cadence gives transient +broker failures another chance before expiry. If the lease is ultimately lost, +the broker may redeliver the task. Stem persists the terminal result before +acknowledging, so a redelivery can observe terminal state instead of executing +the handler a second time. + +The same lease remains active through terminal result persistence, +group/chord bookkeeping, retry or dead-letter publication, linked-task +dispatch, and acknowledgement. A slow backend or broker round trip during +that final handling window therefore gets the same renewal protection as the +handler itself. + +Lease protection starts as soon as the delivery enters the worker, so slow +consume middleware, signature validation, backend lookups, and rate-limit +decisions are covered before handler execution begins. + +If the same envelope is redelivered while its first delivery is still active +in the same worker, the duplicate is acknowledged without invoking the +handler again. This is a worker-local guard; recovery after a process crash +still follows Stem's at-least-once delivery model and requires idempotent +external side effects. + +Hard shutdown requeues active deliveries, including deliveries already held in +the broker prefetch window. A replacement worker can therefore drain the full +batch; applications should still expect at-least-once execution for any +handler that was already running when the process stopped. + +When two different workers complete the same task concurrently, built-in +result backends use `AtomicTerminalResultBackend` to arbitrate the terminal +state. Only the worker that wins that atomic write performs terminal group, +chord, linked-task, signal, and unique-lock side effects. This prevents a late +success or failure record from replacing the result that already won. Custom +result backends remain compatible, but must implement that optional capability +if they need the same cross-process first-writer-wins guarantee. + +The worker suite also exercises the failure path where lease renewal fails +while a handler is still running. After the visibility timeout, a replacement +worker receives the redelivery and may finish first; the original handler's +late completion is then ignored by terminal arbitration. This is still +at-least-once execution: external side effects must be idempotent, and a +successful handler is not proof that its broker acknowledgement was durable. + +Renewal is scheduled from roughly half of the remaining lease. The configured +minimum interval is used for ordinary leases, but it does not delay renewal of +short leases past their deadline. If the lease duration is shorter than the +default one-second floor, Stem uses the shorter safe interval instead. + ## Workflow lease notes - Workflow runs are lease-based. Workers must renew leases while executing, and other workers can take over after the lease expires. +- SQLite integration coverage reopens the durable workflow store after a + worker/runtime restart and resumes a persisted checkpoint without re-running + the completed step. The same checkpoint/replay contract applies to the + other durable workflow stores, subject to their adapter guarantees. - Keep `runLeaseDuration` **>=** broker visibility timeout to prevent redelivered workflow tasks from being dropped before takeover is possible. - Keep `leaseExtension` renewals ahead of both the workflow lease expiry and the @@ -35,6 +89,10 @@ import TabItem from '@theme/TabItem'; ## Poison-pill handling +- Payload-decoding failures are terminal: Stem acknowledges the delivery, + records a failed result, and dead-letters it with reason `invalid-payload`. + They do not consume the task's normal handler retry budget because retrying + the same malformed bytes cannot repair the message. - If a task fails repeatedly for the same reason, treat it as a poison pill. - Move it to the DLQ and add guardrails or validation to prevent repeats. - Record the failure pattern for future detection. @@ -42,6 +100,14 @@ import TabItem from '@theme/TabItem'; ## Scheduler reliability - Run multiple Beat instances only when backed by a shared lock store. +- Beat revalidates lock ownership immediately before publishing; a lost lease + records a failed dispatch instead of publishing after another scheduler may + have acquired the entry. +- Fenced lock stores attach `stem-lock-fencing-token` to scheduled envelopes. + Downstream state stores that support fencing should reject writes carrying a + token older than the last accepted token. The token prevents stale owners + from overwriting state after a lease expires; it does not make external + side effects exactly once. - Monitor schedule drift and failures to detect store latency. - Re-apply schedules after deploys to ensure definitions stay current. @@ -77,8 +143,9 @@ import TabItem from '@theme/TabItem'; ## Heartbeats and progress -Use heartbeats and progress updates to prevent long-running tasks from being -reclaimed prematurely. +Use heartbeats and progress updates to make long-running tasks observable. +Automatic worker lease renewal protects normal executions; call +`context.extendLease(...)` when a task needs an explicit additional lease. diff --git a/.site/docs/scheduler/index.md b/.site/docs/scheduler/index.md index 2b49c6bf..f66e8911 100644 --- a/.site/docs/scheduler/index.md +++ b/.site/docs/scheduler/index.md @@ -48,6 +48,8 @@ Common scheduler CLI commands: - `stem schedule apply` — load schedule entries from JSON/YAML into the store. - `stem schedule list` — inspect entries in the store. +- `stem schedule trigger ` — enqueue one immediate execution without + changing the recurring schedule state. - `stem schedule dry-run` — preview due times before rollout. - `stem observe schedules` — inspect schedule drift and dispatch status. diff --git a/.site/docs/workers/worker-control.md b/.site/docs/workers/worker-control.md index 4e413198..dfcc774a 100644 --- a/.site/docs/workers/worker-control.md +++ b/.site/docs/workers/worker-control.md @@ -180,17 +180,19 @@ when the entrypoint captures state that cannot cross isolate boundaries. ``` -`stem worker revoke --terminate` throws `TaskRevokedException` the next time an -inline handler calls `TaskContext.heartbeat`, `extendLease`, or `progress`, -allowing the worker to cancel and record the task as cancelled. Isolate handlers -must emit cooperative checkpoints (heartbeat/lease/progress) to be interrupted; -otherwise they finish naturally. +`stem worker revoke --terminate` is observed by inline handlers through the +public `TaskContext.cancellation` token. Existing heartbeat, lease and progress +helpers remain checkpoints and also enforce termination. Isolate handlers must +emit cooperative checkpoints (heartbeat/lease/progress) to be interrupted; +otherwise they finish naturally unless the isolate is terminated by a hard +timeout or hard shutdown. ### Cooperative checkpoints for isolate handlers Make sure isolate entrypoints call one of the cooperative helpers inside any -long-running loop. Each helper throws `TaskRevokedException` when a terminate -revoke is pending, which lets the handler fail fast. +long-running loop. Handlers can also call +`context.cancellation.throwIfCancelled()` at safe points; it throws +`TaskCancellationException` when a terminate revoke is pending. ```dart title="tasks/crunch.dart" file=/../packages/stem/example/docs_snippets/lib/worker_control.dart#worker-control-crunch diff --git a/.site/docs/workflows/errors-retries-and-idempotency.md b/.site/docs/workflows/errors-retries-and-idempotency.md index b150caab..b931b15b 100644 --- a/.site/docs/workflows/errors-retries-and-idempotency.md +++ b/.site/docs/workflows/errors-retries-and-idempotency.md @@ -34,6 +34,15 @@ If a workflow enqueues normal Stem tasks, those tasks still use the normal `TaskOptions` retry policy. The workflow and the task are separate retry surfaces. +## Acknowledgement uncertainty + +Task delivery is at least once. Stem records a successful result before the +final broker acknowledgement. If that acknowledgement is lost, the broker +may redeliver the same envelope; the worker recognizes the durable terminal +result and acknowledges the duplicate without invoking the handler again. +External side effects must still be idempotent because a process can fail +before its result is recorded. + ## Cancellation policies Use `WorkflowCancellationPolicy` when you need to cap: diff --git a/.site/docs/workflows/getting-started.md b/.site/docs/workflows/getting-started.md index d28e1d65..c2452df1 100644 --- a/.site/docs/workflows/getting-started.md +++ b/.site/docs/workflows/getting-started.md @@ -30,10 +30,10 @@ The managed worker subscribes to the workflow orchestration queue, so you do not need to manually register the internal `stem.workflow.run` task. If you prefer a minimal example, `startWorkflow(...)`, -`startWorkflowValue(...)`, and `startWorkflowJson(...)` also lazy-start the -runtime and managed worker on first use. Explicit `start()` is still the -better choice when you want deterministic application lifecycle control. Use -those name-based APIs when workflow names come from config or external input. +`startWorkflowValue(...)`, and `startWorkflowJson(...)` initialize the runtime +on first use but never start the managed worker implicitly. Start the returned +app explicitly when the process is intended to consume work. Use those +name-based APIs when workflow names come from config or external input. For workflows you define in code, prefer direct workflow helpers or generated workflow refs. diff --git a/.site/sidebars.ts b/.site/sidebars.ts index 07030a06..e04f00ea 100644 --- a/.site/sidebars.ts +++ b/.site/sidebars.ts @@ -118,6 +118,12 @@ const sidebars: SidebarsConfig = { label: "Brokers", items: ["brokers/overview", "brokers/sqlite", "brokers/caveats"], }, + { + type: "category", + label: "About", + link: { type: "doc", id: "about/index" }, + items: ["about/roadmap", "about/testing"], + }, ], }; diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..b060bf03 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,29 @@ +# Contributing + +Stem changes should improve a documented contract or close a demonstrated +reliability gap. New subsystems need a design note and failure tests before +they become part of the public product story. + +Before opening a pull request: + +```bash +dart pub get +dart run tool/check_examples.dart --skip-diff + +cd packages/stem +dart format lib test --set-exit-if-changed +dart analyze --fatal-infos +dart test --exclude-tags soak --fail-fast +``` + +For package-wide release validation, run the release planner from the +repository root: + +```bash +dart run tool/publish.dart --plan +``` + +The release tool derives package membership and dependency order from the Dart +workspace. It requires a clean Git tree for a real release and validates +formatting, analysis, tests, generated sources, changelog headings, and pub +publication archives. diff --git a/README.md b/README.md index f88097d2..70ac4c92 100644 --- a/README.md +++ b/README.md @@ -3,13 +3,13 @@

- Dart-native background job platform
+ Experimental Dart-native background job platform
Queues, retries, scheduling, workflows, and observability — all in pure Dart.

Pub Version - CI Status + Aggregate CI Status License: MIT

@@ -19,7 +19,7 @@ - **Pure Dart** — No external worker processes, no FFI bindings. Runs anywhere Dart runs. - **Pluggable backends** — Swap between SQLite, Redis, or Postgres with a single line. -- **Battle-tested patterns** — Retries with backoff, rate limiting, dead-letter queues, and priority scheduling. +- **Reliability patterns under active validation** — Retries with backoff, rate limiting, dead-letter queues, and priority scheduling. - **Workflows** — Durable, checkpointed execution for complex multi-step processes. - **Canvas API** — Compose tasks into groups, chains, and chords. - **Observability** — Built-in OpenTelemetry integration for traces and metrics. @@ -28,10 +28,40 @@ ## Quick Start +### Recommended: generated typed tasks + +For application code, define task arguments and results as Dart types and let +`stem_builder` generate the transport adapter, codecs, registry and typed call +object: + +```dart +part 'tasks.stem.g.dart'; + +@TaskDefn(name: 'email.send') +Future sendEmail( + EmailArgs args, { + TaskExecutionContext? context, +}) async { + return EmailResult(await deliver(args)); +} +``` + +Run `dart run build_runner build`, then enqueue through the generated +definition rather than a raw task-name string. See the +[`stem_builder` guide](./packages/stem_builder/README.md) for the complete DTO and +workflow example. + +### Advanced: raw task handlers + +For production-shaped task code, prefer `stem_builder` generated task definitions: +they keep task arguments and results typed across the enqueue/handler boundary. +The raw `TaskHandler` example below is intentionally the low-level +interoperability path for dynamic task names and existing map-based handlers. + ```dart import 'dart:async'; -import 'package:stem/stem.dart'; +import 'package:stem/advanced.dart'; class EmailTask extends TaskHandler { @override @@ -136,7 +166,7 @@ Future main() async { |---------|-------------|---------| | [`stem`](./packages/stem) | Core runtime: contracts, worker, scheduler, in-memory adapters, signals, Canvas, workflows | [![pub](https://img.shields.io/pub/v/stem.svg)](https://pub.dev/packages/stem) | | [`stem_cli`](./packages/stem_cli) | Command-line tooling (`stem` executable) and CLI utilities | [![pub](https://img.shields.io/pub/v/stem_cli.svg)](https://pub.dev/packages/stem_cli) | -| [`stem_memory`](./packages/stem_memory) | In-memory adapter package (broker/backend/workflow/scheduler factories) | [![pub](https://img.shields.io/pub/v/stem_memory.svg)](https://pub.dev/packages/stem_memory) | +| [`stem_memory`](./packages/stem_memory) | Compatibility package for the explicit `package:stem/memory.dart` in-memory library | [![pub](https://img.shields.io/pub/v/stem_memory.svg)](https://pub.dev/packages/stem_memory) | | [`stem_sqlite`](./packages/stem_sqlite) | SQLite broker and result backend for local dev/testing | [![pub](https://img.shields.io/pub/v/stem_sqlite.svg)](https://pub.dev/packages/stem_sqlite) | | [`stem_redis`](./packages/stem_redis) | Redis Streams broker, result backend, and watchdog helpers | [![pub](https://img.shields.io/pub/v/stem_redis.svg)](https://pub.dev/packages/stem_redis) | | [`stem_postgres`](./packages/stem_postgres) | Postgres broker, result backend, and scheduler stores | [![pub](https://img.shields.io/pub/v/stem_postgres.svg)](https://pub.dev/packages/stem_postgres) | @@ -157,7 +187,7 @@ TaskOptions( queue: 'high-priority', // Target queue maxRetries: 5, // Retry on failure priority: 10, // Higher = processed first - rateLimit: '100/m', // Rate limiting + rateLimit: RateLimit.perMinute(100), // Typed rate limiting softTimeLimit: Duration(seconds: 30), hardTimeLimit: Duration(minutes: 2), visibilityTimeout: Duration(minutes: 5), @@ -246,7 +276,9 @@ stem health ### Prerequisites - Dart 3.9.2+ +- Flutter 3.47.0+ (for the local Flutter package gate) - Docker (for adapter integration tests) +- Dagger CLI v0.21.7 (for the reproducible Dagger gate) ### Setup @@ -261,7 +293,7 @@ dart pub get # Run quality gates dart format --output=none --set-exit-if-changed . dart analyze -task test:no-env +task test:all ``` ### Adapter Tests @@ -281,6 +313,26 @@ task test:redis task test:postgres ``` +### Reproducible Dagger Gate + +The root Taskfile is also executable inside a pinned Dagger environment. This +is the recommended path when the local toolchain or Docker Compose state is +not trustworthy: + +```bash +# Runs every Dart and Flutter package test with disposable PostgreSQL/Redis +# services and a pinned Flutter SDK. +task test:dagger + +# If Dagger is not on PATH: +DAGGER_BIN=/path/to/dagger task test:dagger +``` + +The Dagger module uses the Go SDK, pins the Dagger engine, Task release and +Flutter SDK, and generates disposable TLS assets during the run. The aggregate +GitHub Actions test gate calls this same Dagger module; package-specific test +workflows are intentionally not maintained separately. + Targeted adapter tasks now bootstrap integration environment automatically. If bootstrap still fails (for example Docker unavailable), run: diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..3bb098aa --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,14 @@ +# Security policy + +Stem is experimental software. Security reports are welcome, especially +reports involving payload signing, TLS, broker credentials, task isolation, +lease ownership, or data exposure in logs and telemetry. + +Please do not publish a suspected vulnerability in a public issue. Contact the +repository owner through the private security-reporting mechanism provided by +GitHub, including the affected package and version, reproduction steps, impact, +and any suggested mitigation. + +Disposable certificates and keys used by tests are not production credentials. +They must never be reused in deployments or committed outside explicitly +labelled test fixtures. diff --git a/Taskfile.yml b/Taskfile.yml index 5a0c3a53..1b7783ce 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -30,6 +30,17 @@ includes: dir: ./packages/stem_sqlite tasks: + deps: + desc: Resolve workspace and dashboard package dependencies. + cmds: + - | + if command -v flutter >/dev/null 2>&1; then + flutter pub get + else + dart pub get + fi + - task dashboard:deps + test:with-env: internal: true requires: @@ -61,6 +72,7 @@ tasks: exit 1 fi + task deps {{.RUN}} ' @@ -73,6 +85,111 @@ tasks: REQUIRED_VARS: "STEM_TEST_REDIS_URL STEM_TEST_POSTGRES_URL" RUN: task test:no-env + test:dagger: + desc: Run every package test in Dagger-managed services and tooling. + cmds: + - bash ./tool/dagger_test.sh all + + quality:dart: + desc: Format and analyze every Dart package. + cmds: + - | + set -euo pipefail + for package in \ + packages/stem \ + packages/stem_builder \ + packages/stem_sqlite \ + packages/stem_memory \ + packages/stem_adapter_tests \ + packages/stem_redis \ + packages/stem_postgres \ + packages/stem_cli \ + packages/dashboard; do + echo "Checking $package" + ( + cd "$package" + dart format lib test --set-exit-if-changed + dart analyze --fatal-infos + ) + done + + standalone:dart: + desc: Resolve Dart packages outside the workspace dependency graph. + cmds: + - | + set -euo pipefail + dart pub get + staged_root="$(mktemp -d)" + trap 'rm -rf "$staged_root"' EXIT + for package in \ + packages/stem \ + packages/stem_builder \ + packages/stem_sqlite \ + packages/stem_memory \ + packages/stem_adapter_tests \ + packages/stem_redis \ + packages/stem_postgres \ + packages/stem_cli; do + package_root="$staged_root/$(basename "$package")" + mkdir -p "$package_root" + staged_package="$(dart run tool/stage_workspace.dart \ + --package "$package" --output "$package_root")" + ( + cd "$staged_package" + dart pub get + ) + done + + standalone:flutter: + desc: Resolve Flutter packages outside the workspace dependency graph. + cmds: + - | + set -euo pipefail + dart pub get + staged_root="$(mktemp -d)" + trap 'rm -rf "$staged_root"' EXIT + for package in \ + packages/stem_flutter \ + packages/stem_flutter_sqlite; do + package_root="$staged_root/$(basename "$package")" + mkdir -p "$package_root" + staged_package="$(dart run tool/stage_workspace.dart \ + --package "$package" --output "$package_root")" + ( + cd "$staged_package" + flutter pub get + ) + done + + test:flutter: + desc: Run the Flutter package analysis and tests. + cmds: + - | + if ! command -v flutter >/dev/null 2>&1; then + echo "Flutter SDK is required for task test:flutter." >&2 + exit 1 + fi + ( + cd packages/stem_flutter + flutter pub get + dart format lib test --set-exit-if-changed + flutter analyze --fatal-infos + flutter test + ) + ( + cd packages/stem_flutter_sqlite + flutter pub get + dart format lib test --set-exit-if-changed + flutter analyze --fatal-infos + flutter test + ) + + test:all: + desc: Run all Dart and Flutter package tests from the repository root. + cmds: + - task test + - task test:flutter + test:no-env: desc: Run all workspace package tests without bootstrapping integration env. cmds: diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 00000000..9d5af1c3 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,24 @@ +# Stem benchmarks + +Run the repeatable in-memory throughput workload from the repository root: + +```bash +dart run benchmark/stem_throughput.dart --tasks 5000 --concurrency 8 +dart run benchmark/stem_throughput.dart --check-baseline +``` + +The benchmark warms up the worker before measuring and reports enqueue +throughput and end-to-end delivery/execution throughput as JSON. The checked-in +baseline is a deliberately conservative minimum for CI; record machine-specific +results with the commit and Dart SDK when comparing performance changes. Adapter +benchmarks should live beside the adapter because Redis, Postgres and SQLite +contention have different costs. + +The SQLite adapter has a file-backed worker/broker/backend workload that +exercises concurrent writer coordination: + +```bash +cd packages/stem_sqlite +dart run benchmark/sqlite_throughput.dart --tasks 1000 --concurrency 4 \ + --check-baseline +``` diff --git a/benchmark/stem_throughput.dart b/benchmark/stem_throughput.dart new file mode 100644 index 00000000..e27fd821 --- /dev/null +++ b/benchmark/stem_throughput.dart @@ -0,0 +1,143 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:stem/memory.dart'; +import 'package:stem/stem.dart'; + +Future main(List args) async { + final tasks = _intOption(args, '--tasks') ?? 5000; + final concurrency = _intOption(args, '--concurrency') ?? 8; + final warmupTasks = _intOption(args, '--warmup') ?? 250; + final checkBaseline = args.contains('--check-baseline'); + + if (tasks <= 0 || concurrency <= 0 || warmupTasks < 0) { + throw ArgumentError( + 'Tasks and concurrency must be positive; warmup cannot be negative.', + ); + } + + final broker = InMemoryBroker(); + final backend = InMemoryResultBackend(); + final completed = Completer(); + final warmupCompleted = Completer(); + var measuring = false; + var warmupCompletedTasks = 0; + var completedTasks = 0; + final registry = InMemoryTaskRegistry() + ..register( + _BenchmarkTask( + onComplete: () { + if (!measuring) { + warmupCompletedTasks += 1; + if (warmupCompletedTasks == warmupTasks && + !warmupCompleted.isCompleted) { + warmupCompleted.complete(); + } + return; + } + completedTasks += 1; + if (completedTasks == tasks && !completed.isCompleted) { + completed.complete(); + } + }, + ), + ); + final stem = Stem(broker: broker, registry: registry, backend: backend); + final worker = Worker( + broker: broker, + registry: registry, + backend: backend, + concurrency: concurrency, + prefetchMultiplier: 1, + heartbeatTransport: const NoopHeartbeatTransport(), + consumerName: 'benchmark-worker', + lifecycle: const WorkerLifecycleConfig(installSignalHandlers: false), + ); + + await worker.start(); + if (warmupTasks > 0) { + for (var index = 0; index < warmupTasks; index++) { + await stem.enqueue('benchmark.noop', args: {'index': index}); + } + await warmupCompleted.future.timeout(const Duration(minutes: 2)); + } + measuring = true; + final total = Stopwatch()..start(); + final enqueue = Stopwatch()..start(); + for (var index = 0; index < tasks; index++) { + await stem.enqueue('benchmark.noop', args: {'index': index}); + } + enqueue.stop(); + + await completed.future.timeout(const Duration(minutes: 2)); + total.stop(); + await worker.shutdown(); + broker.dispose(); + + final result = { + 'tasks': tasks, + 'concurrency': concurrency, + 'enqueue_ms': enqueue.elapsedMicroseconds / 1000, + 'end_to_end_ms': total.elapsedMicroseconds / 1000, + 'enqueue_tasks_per_second': _rate(tasks, enqueue.elapsed), + 'end_to_end_tasks_per_second': _rate(tasks, total.elapsed), + }; + stdout.writeln(const JsonEncoder.withIndent(' ').convert(result)); + + if (checkBaseline) { + _checkBaseline(result); + } + await stdout.flush(); + // The benchmark owns a short-lived process and has already disposed all + // runtime resources. Exit explicitly so a broker/stream implementation that + // leaves a non-terminal listener cannot make the benchmark appear hung. + exit(0); +} + +double _rate(int count, Duration duration) { + final seconds = duration.inMicroseconds / Duration.microsecondsPerSecond; + return seconds == 0 ? double.infinity : count / seconds; +} + +int? _intOption(List args, String name) { + final index = args.indexOf(name); + if (index == -1 || index + 1 >= args.length) return null; + return int.tryParse(args[index + 1]); +} + +void _checkBaseline(Map result) { + final baselineFile = File('benchmark/stem_throughput_baseline.json'); + if (!baselineFile.existsSync()) { + throw StateError('Missing benchmark baseline: ${baselineFile.path}'); + } + final baseline = jsonDecode(baselineFile.readAsStringSync()); + if (baseline is! Map) { + throw StateError('Benchmark baseline must be a JSON object.'); + } + final minimum = baseline['minimum_end_to_end_tasks_per_second']; + final measured = result['end_to_end_tasks_per_second']; + if (minimum is! num || measured is! num || measured < minimum) { + throw StateError( + 'End-to-end throughput regressed: measured $measured tasks/s, ' + 'minimum $minimum tasks/s.', + ); + } +} + +final class _BenchmarkTask extends TaskHandler { + _BenchmarkTask({required this.onComplete}); + + final void Function() onComplete; + + @override + String get name => 'benchmark.noop'; + + @override + TaskOptions get options => const TaskOptions(); + + @override + Future call(TaskContext context, Map args) async { + onComplete(); + } +} diff --git a/benchmark/stem_throughput_baseline.json b/benchmark/stem_throughput_baseline.json new file mode 100644 index 00000000..9be580e9 --- /dev/null +++ b/benchmark/stem_throughput_baseline.json @@ -0,0 +1,3 @@ +{ + "minimum_end_to_end_tasks_per_second": 1000 +} diff --git a/packages/dashboard/Taskfile.yml b/packages/dashboard/Taskfile.yml index e551c411..42f455d1 100644 --- a/packages/dashboard/Taskfile.yml +++ b/packages/dashboard/Taskfile.yml @@ -1,6 +1,16 @@ version: "3" tasks: + deps: + desc: Resolve dashboard package dependencies. + cmds: + - | + if command -v flutter >/dev/null 2>&1; then + flutter pub get + else + dart pub get + fi + test: desc: Run dashboard package tests. cmds: diff --git a/packages/dashboard/lib/src/server.dart b/packages/dashboard/lib/src/server.dart index 7a76eba8..04b7de2f 100644 --- a/packages/dashboard/lib/src/server.dart +++ b/packages/dashboard/lib/src/server.dart @@ -4,8 +4,8 @@ import 'dart:io'; import 'package:routed/routed.dart'; import 'package:routed_hotwire/routed_hotwire.dart'; -import 'package:stem/stem.dart' - show TaskState, generateEnvelopeId, stemLogContext, stemLogger; +import 'package:stem/observability.dart' show stemLogger; +import 'package:stem/stem.dart' show TaskState, generateEnvelopeId; import 'package:stem_dashboard/src/config/config.dart'; import 'package:stem_dashboard/src/services/models.dart'; import 'package:stem_dashboard/src/services/stem_service.dart'; @@ -105,15 +105,13 @@ Future runDashboardServer({ stemLogger.info( 'Starting dashboard server', - stemLogContext( - component: 'dashboard', - subsystem: 'server', - fields: { - 'host': options.host, - 'port': options.port, - 'basePath': dashboardUrlPath, - }, - ), + fields: { + 'component': 'dashboard', + 'subsystem': 'server', + 'host': options.host, + 'port': options.port, + 'basePath': dashboardUrlPath, + }, ); try { @@ -141,11 +139,11 @@ Future _waitForShutdownSignal() async { void complete(ProcessSignal signal) { stemLogger.info( 'Shutdown signal received', - stemLogContext( - component: 'dashboard', - subsystem: 'server', - fields: {'signal': signal.toString()}, - ), + fields: { + 'component': 'dashboard', + 'subsystem': 'server', + 'signal': signal.toString(), + }, ); if (!completer.isCompleted) { completer.complete(); @@ -405,18 +403,16 @@ Future _renderOverviewPartials( ), ].join('\n'); - return ctx.turboStream(updates); + return await ctx.turboStream(updates); } on Object catch (error, stack) { stemLogger.error( 'Failed to render overview partials', - stemLogContext( - component: 'dashboard', - subsystem: 'server', - fields: { - 'error': error.toString(), - 'stack': stack.toString(), - }, - ), + fields: { + 'component': 'dashboard', + 'subsystem': 'server', + 'error': error.toString(), + 'stack': stack.toString(), + }, ); return ctx.turboHtml( '
Failed to refresh overview metrics.
', @@ -582,10 +578,10 @@ Future _renderPage( final streamPath = dashboardRoute(basePath, '/dash/streams'); if (turbo.isFrameRequest) { - return ctx.turboFrame(renderFrame(page, contentWithBasePath)); + return await ctx.turboFrame(renderFrame(page, contentWithBasePath)); } - return ctx.turboHtml( + return await ctx.turboHtml( renderLayout( page, contentWithBasePath, @@ -596,15 +592,13 @@ Future _renderPage( } on Object catch (error, stack) { stemLogger.error( 'Failed to render dashboard page', - stemLogContext( - component: 'dashboard', - subsystem: 'server', - fields: { - 'page': page.name, - 'error': error.toString(), - 'stack': stack.toString(), - }, - ), + fields: { + 'component': 'dashboard', + 'subsystem': 'server', + 'page': page.name, + 'error': error.toString(), + 'stack': stack.toString(), + }, ); final errorContent = _renderErrorPanel(error); if (turbo.isFrameRequest) { @@ -688,7 +682,7 @@ Future _enqueueTask( actor: 'dashboard', summary: 'Task enqueue rejected: queue/task missing.', ); - return ctx.turboSeeOther('$tasksPath?error=missing-fields'); + return await ctx.turboSeeOther('$tasksPath?error=missing-fields'); } final payloadText = (await ctx.postForm('payload')).trim(); @@ -706,7 +700,7 @@ Future _enqueueTask( actor: 'dashboard', summary: 'Task enqueue rejected: payload not a JSON object.', ); - return ctx.turboSeeOther('$tasksPath?error=invalid-payload'); + return await ctx.turboSeeOther('$tasksPath?error=invalid-payload'); } } on Object { state.recordAudit( @@ -716,7 +710,7 @@ Future _enqueueTask( actor: 'dashboard', summary: 'Task enqueue rejected: invalid JSON payload.', ); - return ctx.turboSeeOther('$tasksPath?error=invalid-payload'); + return await ctx.turboSeeOther('$tasksPath?error=invalid-payload'); } } @@ -744,18 +738,16 @@ Future _enqueueTask( summary: 'Queued task "$task" on "$queue".', metadata: {'queue': queue, 'task': task}, ); - return ctx.turboSeeOther('$tasksPath?flash=queued'); + return await ctx.turboSeeOther('$tasksPath?flash=queued'); } on Object catch (error, stack) { stemLogger.error( 'Dashboard enqueue failed', - stemLogContext( - component: 'dashboard', - subsystem: 'server', - fields: { - 'error': error.toString(), - 'stack': stack.toString(), - }, - ), + fields: { + 'component': 'dashboard', + 'subsystem': 'server', + 'error': error.toString(), + 'stack': stack.toString(), + }, ); state.recordAudit( kind: 'action', @@ -792,7 +784,7 @@ Future _taskAction( actor: 'dashboard', summary: 'Task action rejected: missing task id.', ); - return ctx.turboSeeOther( + return await ctx.turboSeeOther( _appendRedirectQuery(redirect, {'error': 'Task ID is required.'}), ); } @@ -820,7 +812,7 @@ Future _taskAction( summary: 'Failed to revoke task $taskId.', metadata: {'taskId': taskId, 'queue': ?queue}, ); - return ctx.turboSeeOther( + return await ctx.turboSeeOther( _appendRedirectQuery(redirect, { 'error': 'Unable to revoke task $taskId.', }), @@ -834,7 +826,7 @@ Future _taskAction( summary: 'Revocation requested for $taskId.', metadata: {'taskId': taskId, 'queue': ?queue}, ); - return ctx.turboSeeOther( + return await ctx.turboSeeOther( _appendRedirectQuery(redirect, { 'flash': 'Revocation requested for task $taskId.', }), @@ -850,7 +842,7 @@ Future _taskAction( summary: 'Task $taskId was not found in dead letters.', metadata: {'taskId': taskId, 'queue': ?queue}, ); - return ctx.turboSeeOther( + return await ctx.turboSeeOther( _appendRedirectQuery(redirect, { 'error': 'Task $taskId was not found in dead letters.', }), @@ -864,7 +856,7 @@ Future _taskAction( summary: 'Replayed dead-letter task $taskId.', metadata: {'taskId': taskId, 'queue': ?queue}, ); - return ctx.turboSeeOther( + return await ctx.turboSeeOther( _appendRedirectQuery(redirect, { 'flash': 'Replayed dead-letter task $taskId as a new envelope.', }), @@ -878,7 +870,7 @@ Future _taskAction( summary: 'Unsupported task action "$action".', metadata: {'taskId': taskId}, ); - return ctx.turboSeeOther( + return await ctx.turboSeeOther( _appendRedirectQuery(redirect, { 'error': 'Unsupported task action "$action".', }), @@ -887,14 +879,12 @@ Future _taskAction( } on Object catch (error, stack) { stemLogger.error( 'Dashboard task action failed', - stemLogContext( - component: 'dashboard', - subsystem: 'server', - fields: { - 'error': error.toString(), - 'stack': stack.toString(), - }, - ), + fields: { + 'component': 'dashboard', + 'subsystem': 'server', + 'error': error.toString(), + 'stack': stack.toString(), + }, ); state.recordAudit( kind: 'action', @@ -1101,7 +1091,7 @@ Future _controlWorkers( actor: 'dashboard', summary: 'Control action missing.', ); - return ctx.turboSeeOther( + return await ctx.turboSeeOther( '$workersPath?error=${Uri.encodeComponent('Control action missing.')}', ); } @@ -1141,7 +1131,7 @@ Future _controlWorkers( final encodedError = Uri.encodeComponent( 'Unsupported control action "$rawAction".', ); - return ctx.turboSeeOther('$workersPath?error=$encodedError'); + return await ctx.turboSeeOther('$workersPath?error=$encodedError'); } final payload = {}; @@ -1198,7 +1188,7 @@ Future _controlWorkers( ); final encodedMessage = Uri.encodeComponent(message.toString()); final encodedScope = Uri.encodeComponent(scope); - return ctx.turboSeeOther( + return await ctx.turboSeeOther( '$workersPath?error=$encodedMessage&scope=$encodedScope', ); } @@ -1216,20 +1206,18 @@ Future _controlWorkers( ); final encodedMessage = Uri.encodeComponent(message); final encodedScope = Uri.encodeComponent(scope); - return ctx.turboSeeOther( + return await ctx.turboSeeOther( '$workersPath?flash=$encodedMessage&scope=$encodedScope', ); } on Object catch (error, stack) { stemLogger.error( 'Dashboard control command failed', - stemLogContext( - component: 'dashboard', - subsystem: 'server', - fields: { - 'error': error.toString(), - 'stack': stack.toString(), - }, - ), + fields: { + 'component': 'dashboard', + 'subsystem': 'server', + 'error': error.toString(), + 'stack': stack.toString(), + }, ); state.recordAudit( kind: 'action', @@ -1264,7 +1252,7 @@ Future _replayDeadLetters( actor: 'dashboard', summary: 'Replay rejected: missing queue name.', ); - return ctx.turboSeeOther( + return await ctx.turboSeeOther( _appendRedirectQuery(redirect, { 'error': 'Queue name is required for replay.', }), @@ -1298,7 +1286,7 @@ Future _replayDeadLetters( summary: message, metadata: {'queue': queue, 'dryRun': dryRun}, ); - return ctx.turboSeeOther( + return await ctx.turboSeeOther( _appendRedirectQuery(redirect, { 'flash': message, 'scope': scope, @@ -1321,7 +1309,7 @@ Future _replayDeadLetters( summary: message, metadata: {'queue': queue, 'entries': entryCount, 'dryRun': dryRun}, ); - return ctx.turboSeeOther( + return await ctx.turboSeeOther( _appendRedirectQuery(redirect, { 'flash': message, 'scope': scope, @@ -1331,14 +1319,12 @@ Future _replayDeadLetters( } on Object catch (error, stack) { stemLogger.error( 'Dashboard dead-letter replay failed', - stemLogContext( - component: 'dashboard', - subsystem: 'server', - fields: { - 'error': error.toString(), - 'stack': stack.toString(), - }, - ), + fields: { + 'component': 'dashboard', + 'subsystem': 'server', + 'error': error.toString(), + 'stack': stack.toString(), + }, ); state.recordAudit( kind: 'action', diff --git a/packages/dashboard/lib/src/services/stem_service.dart b/packages/dashboard/lib/src/services/stem_service.dart index c40e01f0..878014ba 100644 --- a/packages/dashboard/lib/src/services/stem_service.dart +++ b/packages/dashboard/lib/src/services/stem_service.dart @@ -1,6 +1,8 @@ import 'dart:async'; import 'dart:io'; +import 'package:stem/memory.dart' show InMemoryWorkflowStore; +import 'package:stem/observability.dart' show stemLogger; import 'package:stem/stem.dart'; import 'package:stem_cli/stem_cli.dart'; import 'package:stem_dashboard/src/config/config.dart'; @@ -607,15 +609,13 @@ class StemDashboardService implements DashboardDataSource { void _logReadFailure(String operation, Object error, StackTrace stack) { stemLogger.warning( 'Dashboard data read failed', - stemLogContext( - component: 'dashboard', - subsystem: 'service', - fields: { - 'operation': operation, - 'error': '$error', - 'stack': '$stack', - }, - ), + fields: { + 'component': 'dashboard', + 'subsystem': 'service', + 'operation': operation, + 'error': '$error', + 'stack': '$stack', + }, ); } diff --git a/packages/dashboard/pubspec.yaml b/packages/dashboard/pubspec.yaml index 67552a92..c4b35c1e 100644 --- a/packages/dashboard/pubspec.yaml +++ b/packages/dashboard/pubspec.yaml @@ -5,27 +5,45 @@ publish_to: "none" environment: sdk: ">=3.9.2 <4.0.0" -resolution: workspace dependencies: intl: ^0.20.2 meta: ^1.18.0 ormed: ^0.2.0 - routed: ^0.3.2 - routed_hotwire: ^0.1.2 - stem: ">=0.2.0-dev <0.3.0" - stem_cli: ^0.1.0 - stem_postgres: ^0.1.0 - stem_redis: ^0.1.0 - stem_sqlite: ^0.1.0 + routed: ^0.5.0 + routed_hotwire: ^0.1.5 + stem: + path: ../stem + stem_cli: ^0.2.0 + stem_postgres: ^0.2.0 + stem_redis: ^0.2.0 + stem_sqlite: ^0.2.0 dev_dependencies: lints: ^6.0.0 property_testing: ^0.3.2 - routed_testing: ^0.3.2 - server_testing: ^0.3.2 + routed_testing: ^0.4.0 + server_testing: ^0.4.0 test: ^1.29.0 very_good_analysis: ^10.0.0 +# The dashboard is an unpublished experimental companion. Its local Stem +# packages still declare the older artisanal range, while the current Routed +# testing packages require artisanal 0.5.x. Keep this compatibility override +# local to the dashboard rather than changing the published Stem packages here. dependency_overrides: - analyzer: ^10.0.1 - artisanal: ^0.2.0 + artisanal: ^0.5.0 + # routed_auth 0.2.0 currently uses the AuthAdapter/AuthOptions surface from + # server_auth 0.1.0; the published 0.2.0 archive is not source-compatible. + server_auth: 0.1.0 + stem: + path: ../stem + stem_cli: + path: ../stem_cli + stem_memory: + path: ../stem_memory + stem_postgres: + path: ../stem_postgres + stem_redis: + path: ../stem_redis + stem_sqlite: + path: ../stem_sqlite diff --git a/packages/stem/.pubignore b/packages/stem/.pubignore new file mode 100644 index 00000000..af067c2c --- /dev/null +++ b/packages/stem/.pubignore @@ -0,0 +1,12 @@ +# Disposable integration assets are generated by the test environment and +# must never be included in the published core package. +example/microservice/.certs/ +example/microservice/certs/ +example/postgres_tls/certs/ +example/security/*/certs/ + +# Internal planning and process notes are maintained in the repository and +# website, but are not part of the stable package artifact. +doc/internal/ +doc/process/ +doc/spikes/ diff --git a/packages/stem/CHANGELOG.md b/packages/stem/CHANGELOG.md index 22295b62..625e10f2 100644 --- a/packages/stem/CHANGELOG.md +++ b/packages/stem/CHANGELOG.md @@ -1,5 +1,107 @@ # Changelog +## 0.3.0 + +- Promoted the typed stable/advanced API boundary and removed implicit worker + startup from producer, inspection, Canvas, and workflow operations. +- Added capability-aware queue transports, public cooperative cancellation, + explicit task execution modes, + typed rate limits, heterogeneous Canvas chains, explicit chord policies, and + OpenTelemetry Canvas fan-out span links. +- Scheduler dispatch now revalidates distributed lock ownership immediately + before publication and records lease-loss failures instead of publishing + after ownership has expired. +- Added compatibility-safe fencing-token support to lock handles. Memory, + Redis, and Postgres acquisitions expose monotonically increasing tokens; + Beat propagates the token on scheduled envelopes for downstream enforcement. +- This release intentionally contains breaking API changes from the 0.2 line. +- The observability entrypoint now exposes Stem-owned logging types and a + structured logging facade; the `contextual` logger types remain internal. +- Lease renewal derives its cadence from the remaining lease, so short + visibility leases are renewed before expiry instead of being forced onto the + default one-second minimum interval. +- Failed automatic renewals retry on a shorter cadence, and late in-flight + renewals cannot recreate timers after a delivery has been cancelled. +- Active-delivery accounting uses worker-local delivery handles, so concurrent + redeliveries of one envelope no longer overwrite shutdown or in-flight state. +- Lease timers are keyed by delivery identity rather than receipt text, which + keeps same-receipt redeliveries independent for adapters that reuse row IDs. +- Automatic heartbeat timers now use the same delivery identity, preventing a + concurrent redelivery from cancelling the original task's heartbeat. +- Lease renewal now remains active through terminal result persistence, + group/chord bookkeeping, retry or dead-letter publication, linked-task + dispatch, and acknowledgement; slow terminal handling cannot expire the + delivery before the worker releases it. +- Lease renewal now starts when a delivery enters the worker, covering slow + consume middleware, signature validation, backend lookups, and rate-limit + decisions before handler execution. +- Concurrent redeliveries of an envelope already active in the same worker are + acknowledged without a second handler invocation; process-wide failures + still rely on normal at-least-once recovery. +- Documentation now distinguishes heartbeat/liveness signals from broker lease + extension; use automatic renewal or `context.extendLease(...)` for leases. +- Added the optional `AtomicTerminalResultBackend` capability. Built-in result + backends arbitrate terminal writes so a late cross-worker completion cannot + replace the first terminal result; custom backends remain source compatible + and retain their previous non-atomic fallback behavior. +- Worker terminal side effects such as group/chord bookkeeping, linked-task + dispatch, terminal signals, and unique-lock release now belong only to the + worker that wins terminal-result arbitration. +- Added a deterministic lease-loss recovery regression: renewal failure lets a + delivery expire, a replacement worker can complete the redelivery, and the + original late completion cannot overwrite that terminal result. +- Payload-decoding failures are now terminally failed and dead-lettered as + `invalid-payload` instead of escaping before acknowledgement and redelivering + indefinitely. Retry-storm coverage verifies that normal retry budgets remain + bounded under concurrent failure. +- Hard-shutdown coverage now verifies that a full broker prefetch window is + requeued and completed by a replacement worker, not only a single active + isolate delivery. +- Added a warmed-up core throughput benchmark with a checked regression + baseline and scheduled CI execution. + +## 0.2.3 + +- Added additive `QueueBroker`, `LeaseBroker`, `InspectableBroker`, and + `DeadLetterBroker` capability interfaces for new adapter integrations. +- Made managed worker startup explicit for every bootstrap path. Enqueue, + status, result-wait, Canvas, and workflow operations never start a worker; + applications must call `start()` (or `startWorker()`) deliberately. +- Added `stable.dart` and `advanced.dart` entrypoints to make the intended API + boundary explicit while retaining the historical `stem.dart` compatibility + barrel. +- Made `QueueBroker` independently implementable; lease, inspection, purge, + and dead-letter operations are now optional capability interfaces with + compatibility defaults on `Broker`. +- Extracted isolate execution and pool lifecycle into an internal execution + supervisor, and hardened shutdown against late delivery errors after the + worker event stream closes. +- Centralized worker event emission so late timers, retries, heartbeats, + progress callbacks, and revocation notifications are safely ignored after + shutdown. +- Extracted broker subscription ownership and stream error boundaries into an + internal worker consumer loop, including safe queue-subscription replacement + during pause and resume operations. + +## 0.2.2 + +- Replaced string rate-limit fields with typed `RateLimit` values while + retaining legacy parsing at JSON/configuration boundaries. +- Added cooperative task cancellation through + `TaskExecutionContext.cancellation`. +- Made memory adapters available through the explicit + `package:stem/memory.dart` library. +- Prevented status and result-wait operations from implicitly starting a + worker. +- Moved release validation to dependency-ordered workspace automation and + added aggregate package quality gates. +- Added a source-compatible `BrokerCapabilities` snapshot so adapters can + declare optional delivery, inspection, broadcast, lease, and dead-letter + behavior without expanding the base broker contract. +- Added queue-broker extension methods for optional dead-letter inspection, + replay, and purge operations, allowing integrations to depend on + `QueueBroker` without reverting to the legacy broad `Broker` type. + ## 0.2.1 - Guarded worker and example process signal registration so Windows only diff --git a/packages/stem/README.md b/packages/stem/README.md index 4be43c05..d87aa7f7 100644 --- a/packages/stem/README.md +++ b/packages/stem/README.md @@ -9,18 +9,25 @@ # Stem -Stem is a Dart-first background job and workflow platform: enqueue work, run workers, and orchestrate durable workflows. +Stem is an experimental Dart-first background job and workflow platform: +enqueue work, run workers, and orchestrate durable workflows while the runtime +contracts and adapter guarantees continue to harden. For full docs, API references, and in-depth guides, visit https://kingwill101.github.io/stem. +For production-shaped task definitions, prefer `stem_builder` generated typed +definitions. Manual `TaskDefinition` is the supported typed +fallback. Low-level map handlers live under `package:stem/advanced.dart` for +transport integrations and migrations. + ## Packages | Package | Description | pub.dev | |---------|-------------|---------| | [`stem`](https://github.com/kingwill101/stem/tree/master/packages/stem) | Core runtime: contracts, worker, scheduler, in-memory adapters, signals, Canvas, workflows | [![pub](https://img.shields.io/pub/v/stem.svg)](https://pub.dev/packages/stem) | | [`stem_cli`](https://github.com/kingwill101/stem/tree/master/packages/stem_cli) | Command-line tooling (`stem` executable) and CLI utilities | [![pub](https://img.shields.io/pub/v/stem_cli.svg)](https://pub.dev/packages/stem_cli) | -| [`stem_memory`](https://github.com/kingwill101/stem/tree/master/packages/stem_memory) | In-memory adapter package (broker/backend/workflow/scheduler factories) | [![pub](https://img.shields.io/pub/v/stem_memory.svg)](https://pub.dev/packages/stem_memory) | +| [`stem_memory`](https://github.com/kingwill101/stem/tree/master/packages/stem_memory) | Compatibility package for the explicit `package:stem/memory.dart` in-memory library | [![pub](https://img.shields.io/pub/v/stem_memory.svg)](https://pub.dev/packages/stem_memory) | | [`stem_sqlite`](https://github.com/kingwill101/stem/tree/master/packages/stem_sqlite) | SQLite broker and result backend for local dev/testing | [![pub](https://img.shields.io/pub/v/stem_sqlite.svg)](https://pub.dev/packages/stem_sqlite) | | [`stem_redis`](https://github.com/kingwill101/stem/tree/master/packages/stem_redis) | Redis Streams broker, result backend, and watchdog helpers | [![pub](https://img.shields.io/pub/v/stem_redis.svg)](https://pub.dev/packages/stem_redis) | | [`stem_postgres`](https://github.com/kingwill101/stem/tree/master/packages/stem_postgres) | Postgres broker, result backend, and scheduler stores | [![pub](https://img.shields.io/pub/v/stem_postgres.svg)](https://pub.dev/packages/stem_postgres) | @@ -40,7 +47,8 @@ https://kingwill101.github.io/stem. - **Observability** - Dartastic OpenTelemetry metrics/traces, heartbeats, CLI inspection (`stem observe`, `stem dlq`). - **Security** - Payload signing (HMAC or Ed25519), TLS automation scripts, revocation persistence. - **Adapters** - In-memory drivers included here; Redis Streams and Postgres adapters ship via the `stem_redis` and `stem_postgres` packages. -- **Specs & tooling** - OpenSpec change workflow, quality gates (see `example/quality_gates`), chaos/regression suites. +- **Quality tooling** - Package-level format, analysis, test, adapter contract, + chaos, benchmark, and standalone-resolution gates in CI. ## Install @@ -54,39 +62,56 @@ dart pub add -d stem_builder # for annotations/codegen (optional) dart pub add -d stem_cli # for CLI tooling ``` +New application code can import `package:stem/stable.dart`. The historical +`package:stem/stem.dart` barrel remains available for compatibility, while +custom transports and instrumentation can use `package:stem/advanced.dart`. + +For local development and tests, import the in-memory implementations +explicitly with `package:stem/memory.dart`; the `stem_memory` package remains +as a compatibility export for existing applications. + ## Examples -`StemApp` and `StemWorkflowApp` shortcut helpers lazily start their managed -worker by default. Pass `allowWorkerAutoStart: false` when you want producer -or orchestration shortcuts without starting that worker in the background, -then call `start()` explicitly when you're ready. `StemWorkflowApp` also -exposes `startRuntime()` and `startWorker()` when you want those lifecycles -split. +`StemApp` and `StemWorkflowApp` never start their managed worker implicitly. +Call `start()` explicitly when the process is intended to consume work. +`StemWorkflowApp` also exposes `startRuntime()` and `startWorker()` when you +want those lifecycles split. ### Minimal in-memory task + worker ```dart import "dart:async"; -import "package:stem/stem.dart"; +import "package:stem/stable.dart"; -class HelloTask extends TaskHandler { - @override - String get name => "demo.hello"; +class HelloArgs { + const HelloArgs({required this.name}); - @override - Future call(TaskContext context, Map args) async { - final name = args.valueOr("name", "world"); - print("Hello $name"); - } + final String name; + + Map toJson() => {"name": name}; + + factory HelloArgs.fromJson(Map json) => + HelloArgs(name: json["name"] as String); } +final helloDefinition = TaskDefinition.json( + name: "demo.hello", + decodeArgsJson: HelloArgs.fromJson, +); + +final helloTask = helloDefinition.handler( + entrypoint: (context, args) async => print("Hello ${args.name}"), +); + Future main() async { - final client = await StemClient.inMemory(tasks: [HelloTask()]); + final client = await StemClient.inMemory(tasks: [helloTask]); final worker = await client.createWorker(); unawaited(worker.start()); - await client.enqueueValue("demo.hello", const {"name": "Stem"}); + await client.enqueueCall( + helloDefinition.buildCall(const HelloArgs(name: "Stem")), + ); await Future.delayed(const Duration(seconds: 1)); await worker.shutdown(); @@ -97,7 +122,7 @@ Future main() async { ### Reusable stack from URL (Redis) ```dart -import "package:stem/stem.dart"; +import "package:stem/stable.dart"; import "package:stem_redis/stem_redis.dart"; Future main() async { @@ -133,28 +158,22 @@ class HelloArgs { HelloArgs(name: json["name"] as String); } -class HelloTask2 extends TaskHandler { - static final definition = TaskDefinition.json( - name: "demo.hello2", - metadata: const TaskMetadata(description: "typed hello task"), - ); - - @override - String get name => definition.name; +final helloDefinition = TaskDefinition.json( + name: "demo.hello2", + decodeArgsJson: HelloArgs.fromJson, + metadata: const TaskMetadata(description: "typed hello task"), +); - @override - Future call(TaskContext context, Map args) async { - final payload = HelloArgs.fromJson(args.cast()); - return "Hello ${payload.name}"; - } -} +final helloTask = helloDefinition.handler( + entrypoint: (context, args) async => "Hello ${args.name}", +); Future main() async { - final client = await StemClient.inMemory(tasks: [HelloTask2()]); + final client = await StemClient.inMemory(tasks: [helloTask]); final worker = await client.createWorker(); unawaited(worker.start()); - final result = await HelloTask2.definition.enqueueAndWait( + final result = await helloDefinition.enqueueAndWait( client, const HelloArgs(name: "Typed"), ); @@ -168,7 +187,7 @@ Future main() async { ### Workflow quick-start (Flow) ```dart -import "package:stem/stem.dart"; +import "package:stem/stable.dart"; final onboardingFlow = Flow( name: "demo.onboarding", @@ -184,7 +203,6 @@ Future main() async { final appClient = await StemClient.inMemory(); final app = await appClient.createWorkflowApp( flows: [onboardingFlow], - allowWorkerAutoStart: false, ); await app.start(); @@ -201,7 +219,7 @@ Future main() async { ### Annotated workflow + task with `stem_builder` ```dart -import "package:stem/stem.dart"; +import "package:stem/stable.dart"; import "package:stem_builder/stem_builder.dart"; part "definitions.stem.g.dart"; @@ -241,7 +259,7 @@ dart run build_runner build ```dart // example usage after codegen final client = await StemClient.inMemory(module: stemModule); -final app = await client.createWorkflowApp(allowWorkerAutoStart: false); +final app = await client.createWorkflowApp(); await app.start(); final runId = await StemWorkflowDefinitions.builderSignup.startAndWait( @@ -255,7 +273,8 @@ print(result?.value); // {user: alice@example.com} ### Workflow with multiple worker queues ```dart -import "package:stem/stem.dart"; +import "package:stem/stable.dart"; +import "package:stem/advanced.dart"; final onboardingFlow = Flow>( name: "workflow.multi_workers", @@ -353,7 +372,8 @@ stem wf --help ### General worker management (multi-worker setup) ```dart -import "package:stem/stem.dart"; +import "package:stem/stable.dart"; +import "package:stem/advanced.dart"; class EmailTask extends TaskHandler { @override @@ -452,7 +472,6 @@ see the full docs at https://kingwill101.github.io/stem. - [unique_tasks](example/unique_tasks/unique_task_example.dart) - enables `TaskOptions.unique` with a shared lock store. - [signing_key_rotation](example/signing_key_rotation) - rotate HMAC signing keys with overlap. - [ops_health_suite](example/ops_health_suite) - CLI health checks + queue/worker snapshots. -- [quality_gates](example/quality_gates) - justfile-driven quality gate runner. - [security examples](example/security/*) - payload signing + TLS profiles. - [postgres_tls](example/postgres_tls) - Redis broker + Postgres backend secured via the shared `STEM_TLS_*` settings. - [otel_metrics](example/otel_metrics) - OTLP collectors + Grafana dashboards. diff --git a/packages/stem/analysis_options.yaml b/packages/stem/analysis_options.yaml index b2f0fbf5..927a9efe 100644 --- a/packages/stem/analysis_options.yaml +++ b/packages/stem/analysis_options.yaml @@ -22,6 +22,9 @@ include: package:very_good_analysis/analysis_options.yaml analyzer: errors: avoid_returning_this: ignore + # Dart 3.10 reports this for the nullable field used by the const + # all-or-partial policy constructors; newer analyzers do not. + prefer_initializing_formals: ignore exclude: - example/**/** - "**/*.g.dart" diff --git a/packages/stem/doc/design/stem-v1.md b/packages/stem/doc/design/stem-v1.md deleted file mode 100644 index c631c701..00000000 --- a/packages/stem/doc/design/stem-v1.md +++ /dev/null @@ -1,121 +0,0 @@ -# Stem v1 Design Doc - -## Executive Summary -Stem provides a Dart-native background job platform built on Redis Streams with -OpenTelemetry-first observability. The v1 release targets at-least-once delivery -with configurable retries, time limits, and a CLI-first operator surface. - -## Goals -- Reliable enqueue → execute → result flow for Dart services. -- Production-ready Redis Streams broker + Redis result backend. -- Built-in observability (metrics, traces, logs) and operational tooling. -- Clear public interface contract for adapters and middleware. - -## Non-Goals -- Exactly-once guarantees or deduplicated delivery. -- Non-Redis adapters (RabbitMQ/SQS) in the v1 timeframe. -- Full UI dashboard (CLI + docs only). - -## System Overview -``` - +-----------+ +-----------------+ +--------------------+ - | Client | -----> | Redis Streams | -----> | Worker (isolate pool)| - +-----------+ +-----------------+ +--------------------+ - | | | | - | | | v - | | | +----------------+ - | | +--------------> | Result Backend | - | | +----------------+ - | | | - | v v - | +-----------------+ +--------------------+ - +-----------> | Beat (scheduler)| -----> | Broker (delayed) | - +-----------------+ +--------------------+ -``` - -- **Client** uses the `Stem` facade to enqueue tasks. -- **Broker** stores envelopes and delivers them to workers. -- **Worker** executes tasks in isolates, renews leases, updates backend. -- **Result backend** persists task status and supports chords/groups. -- **Beat** schedules recurring tasks via Redis ZSET. - -## Message Schema (Envelope) -| Field | Type | Notes | -| --- | --- | --- | -| `id` | `String` | Generated via timestamp + random suffix. | -| `name` | `String` | Qualified task name. | -| `args` | `Map` | JSON-serialisable arguments. | -| `headers` | `Map` | Metadata (traceparent, tenant). | -| `enqueuedAt` | `DateTime` (ISO8601, UTC) | Creation timestamp. | -| `notBefore` | `DateTime?` | Optional ETA. | -| `priority` | `int` | Adapter-specific priority. | -| `attempt` | `int` | Delivery attempt (0-based). | -| `maxRetries` | `int` | Maximum retry count. | -| `visibilityTimeout` | `Duration?` (ms) | Lease duration hint. | -| `queue` | `String` | Logical queue name. | -| `meta` | `Map` | User metadata and system fields. | - -## Task State Model -``` - queued -> running -> succeeded - | | - v v - retried -----> failed - | - v - dead-lettered -``` -- Each transition recorded in the result backend with timestamps and metadata. -- Retries increment `attempt` and re-enqueue with backoff. - -## Delivery Semantics -- At-least-once delivery with idempotent handler recommendation. -- Workers ack **after** successful execution/state write. -- Failed tasks NACKed or dead-lettered when retries exhausted. -- Leases renewed at half intervals; unacked messages reclaimed via XAUTOCLAIM. - -## Time Limits, Prefetch, Rate Limiting -- **Soft limit**: raises a timeout event for observability. -- **Hard limit**: isolates interrupted/killed, task retried. -- **Prefetch**: configurable `concurrency * multiplier` (default ×2). -- **Rate limiting**: token bucket via `RateLimiter` interface; per-task keys by default. - -## Adapters (MVP) -- Broker: Redis Streams (Streams + ZSET for delays + XAUTOCLAIM for reclaim). -- Result backend: Redis Hash + TTL, group aggregation. -- Scheduler: Redis ZSET-based beat. - -## Configuration Model -1. Environment variables (`STEM_*`). -2. Optional `YAML`/`JSON` config file (future work). -3. Code overrides via constructors. - -Precedence: code overrides > config file > env vars. - -## Security -- TLS support for Redis URIs; automation script in `scripts/security/generate_tls_assets.sh` seeds certs for local stacks. -- Payload signing via HMAC-SHA256 with key rotation and dead-lettering of invalid signatures; keys provided through `STEM_SIGNING_KEYS` + `STEM_SIGNING_ACTIVE_KEY`. -- Recurring vulnerability scan guidance (`scripts/security/run_vulnerability_scan.sh`) ensures dependency CVEs are surfaced in CI. -- No admin endpoints; CLI uses same credentials as workers. Threat model highlights: broker compromise (mitigate with ACL/TLS + signing), replay attacks (trace + idempotency), credential leakage (secret rotation policy). - -## Observability & SLOs -- Metrics: `stem.tasks.*`, `stem.worker.inflight`, `stem.queue.depth`, `stem.lease.renewed`, beat metrics. -- Traces: enqueue → consume → execute spans with W3C context propagation. -- Logs: structured with `traceId`, `spanId`, task metadata. -- Example SLOs: - - Success rate ≥ 99.5% over a 15-minute window. - - Worker task latency p95 < 3 seconds over 5 minutes. - - Default queue depth < 100 messages sustained for 10 minutes. - -## Risks & Mitigations -| Risk | Mitigation | -| --- | --- | -| Chord fan-in overload | throttle callbacks, shard result storage. | -| Global rate limits precision | document approximate guarantees; leverage Redis scripts. | -| Duplicate execution | encourage idempotent tasks; provide helper utilities. | -| Redis outage | surface metrics/alerts; document failover runbook. | - -## Sign-off -- [ ] Lead Engineer: ____________________ -- [ ] Product/PM: ____________________ -- [ ] Ops/On-call: ____________________ diff --git a/packages/stem/doc/internal/about/index.md b/packages/stem/doc/internal/about/index.md index 2d1c79cd..2df64679 100644 --- a/packages/stem/doc/internal/about/index.md +++ b/packages/stem/doc/internal/about/index.md @@ -4,8 +4,10 @@ slug: /about sidebar_position: 0 --- -Learn where Stem is headed and how decisions are made. +Learn where Stem is headed and how decisions are made. The public site is the +canonical location for current roadmap status. - **[Roadmap & Milestones](./roadmap.md)** – Upcoming work streams and release targets. -Looking for contribution guidelines? See the repository's `CONTRIBUTING.md` and OpenSpec changes in `openspec/`. +Looking for contribution guidelines? See the repository's `CONTRIBUTING.md` and +OpenSpec changes in `openspec/`. diff --git a/packages/stem/doc/internal/about/roadmap.md b/packages/stem/doc/internal/about/roadmap.md index db4bfea8..d9aa46e8 100644 --- a/packages/stem/doc/internal/about/roadmap.md +++ b/packages/stem/doc/internal/about/roadmap.md @@ -1,56 +1,7 @@ ---- -title: Stem Roadmap & Milestones -sidebar_label: Roadmap -slug: /about/roadmap -sidebar_position: 1 ---- +# Roadmap pointer -## Milestones +The canonical roadmap is maintained in the documentation site at +`.site/docs/about/roadmap.md` and is published at `/about/roadmap`. -### M1 – Core Runtime -- Redis Streams broker & Redis result backend -- Worker daemon (prefetch, retries, lease renewal) -- CLI basics (`stem enqueue`, `stem observe metrics`) - -### M2 – Scheduling & DLQ -- Beat (interval/cron + jitter + locks) -- Dead-letter queues with CLI list/show/replay -- Backoff tuning + rate limiter integration - -### M3 – Observability & Docs -- OpenTelemetry metrics/traces/logging -- CLI/Docs for observability, scaling, quick start -- Samples (monolith, microservice, OTLP stack) - -### M4 – Resilience Enhancements -- Hard timeouts & isolate pool polish -- Reclaim daemon & rate limiter store -- Locks/rate limiter robustness + release process docs - -## Definition of Ready (DoR) - -- User story linked to OpenSpec change/spec delta. -- Acceptance criteria include success/failure behaviour, error handling, docs/test notes. -- Dependencies (other stories, infrastructure) documented. - -## Definition of Done (DoD) - -- `dart format`, `dart analyze`, `dart test` pass locally & in CI. -- Relevant docs (developer, ops, quick start) updated. -- Examples compile/run if affected. -- Observability metrics/traces updated when behaviour changes. -- OpenSpec tasks updated to `- [x]` with evidence (links, tests). - -## Tracking - -- Use GitHub milestones M1–M4 mirroring the above. -- Label issues with `milestone:Mx` and `type:doc`, `type:impl`, `type:test` as appropriate. -- Review DoR/DoD compliance during grooming. - -## Backlog / Outstanding Phase Work - -- **Phase 5 (Testing)**: monitor soak/perf dashboards and automate regression alerts. -- **Phase 6 (Observability)**: implement dashboards/alerts (Grafana) tied to the SLOs and document owners. -- **Phase 7 (Security)**: payload signing helper, TLS automation scripts, recurring vulnerability scans. -- **Phase 9 (Release)**: Beta/RC pilot tracking, automated changelog generation. -- **Phase 10 (Ops)**: CLI pause/resume/drain, auto-scaling hints, roadmap refinement (RabbitMQ/SQS, Postgres backend). +This package-internal file is intentionally only a pointer so that repository +notes do not become a second, stale product roadmap. diff --git a/packages/stem/doc/internal/deployment/ci-cd.md b/packages/stem/doc/internal/deployment/ci-cd.md index 4bf06319..c73b67ce 100644 --- a/packages/stem/doc/internal/deployment/ci-cd.md +++ b/packages/stem/doc/internal/deployment/ci-cd.md @@ -9,8 +9,7 @@ Follow these practices to keep Stem projects healthy in automation. ## Static checks -Run the core quality gates either via `example/quality_gates` (requires `just`) -or by invoking the commands directly. The default set includes: +Run the core quality gates directly. The default set includes: - `dart format --set-exit-if-changed` - `dart analyze` @@ -91,5 +90,5 @@ stem schedule list Automating these checks catches environment-specific issues early. -For more examples, see the repository's `.github/workflows/ci.yml` and plug the +For more examples, see the repository's `.github/workflows/aggregate.yaml` and plug the steps into your preferred CI system (GitLab, CircleCI, Jenkins, etc.). diff --git a/packages/stem/doc/internal/deployment/deployment-hardening.md b/packages/stem/doc/internal/deployment/deployment-hardening.md index 59c3f71a..74dc3d18 100644 --- a/packages/stem/doc/internal/deployment/deployment-hardening.md +++ b/packages/stem/doc/internal/deployment/deployment-hardening.md @@ -5,7 +5,7 @@ sidebar_position: 2 slug: /deployment/hardening --- -This checklist captures production-ready defaults for Stem services—producers, +This checklist captures hardening defaults for experimental Stem services—producers, workers, beat, and the supporting Redis infrastructure. ## Configuration & secrets diff --git a/packages/stem/doc/internal/deployment/release-process.md b/packages/stem/doc/internal/deployment/release-process.md index fbacb743..2d7444d2 100644 --- a/packages/stem/doc/internal/deployment/release-process.md +++ b/packages/stem/doc/internal/deployment/release-process.md @@ -16,29 +16,45 @@ This guide covers preparation, versioning, and post-release steps for Stem. ## Pre-Release Checklist -1. Ensure relevant OpenSpec changes are merged (`openspec validate --strict`). +1. Run the release planner from the repository root: + ```bash + dart run tool/publish.dart --allow-dirty --include-unchanged --plan + ``` + This derives the publishable package graph from the workspace manifests. + `--allow-dirty` is valid for planning only; the full release gate requires + the exact release commit to be clean. 2. Run CI locally: ```bash - dart format --output=show - dart analyze - dart test + dart format lib test --set-exit-if-changed + dart analyze --fatal-infos + dart test --exclude-tags soak ``` -3. Update `.site/docs/` content and cross-links if user-facing changes occurred. -4. Confirm Docker examples (`examples/microservice`, `examples/otel_metrics`) run via `docker compose up`. -5. Draft release notes summarising feature, fixes, migration steps. +3. Run the aggregate workflow checks for every publishable package, including + standalone resolution outside workspace overrides. +4. Update `.site/docs/` content and cross-links if user-facing changes occurred. +5. Confirm Docker examples (`examples/microservice`, `examples/otel_metrics`) run via `docker compose up`. +6. Draft release notes summarising feature, fixes, migration steps. ## Tagging & Publishing -1. Create a release branch: `git checkout -b release/vX.Y.Z`. -2. Update `CHANGELOG.md` (add the new version section). -3. Commit with message `Release vX.Y.Z`. -4. Tag: `git tag vX.Y.Z` and push: `git push origin vX.Y.Z`. -5. Publish the package(s) via your internal registry or pub.dev if applicable. +1. Update each changed package's `CHANGELOG.md` and version. +2. Run `dart run tool/publish.dart` from a clean tree. The tool validates + formatting, analysis, tests, generated output, changelogs and pub.dev dry + runs in dependency order. +3. Commit exactly the metadata used for the release. +4. Tag each package independently using `-v`, for example + `stem-v0.3.0` or `stem_flutter-v0.2.0`, then push the tags. The trusted + publishing workflow consumes these tags and publishes the matching package. +5. Verify the package pages and dependency resolution after publication. ## Migration Notes - Document breaking changes in the release notes and the [Developer Environment](../getting-started/developer-environment.md) guide. - Provide upgrade snippets (`before`/`after`) for significant API shifts. +- Treat database migrations as rolling-deployment boundaries: add nullable or + defaulted columns first, keep old workers able to write during the rollout, + then remove obsolete fields only in a later release. Run the historical + upgrade and legacy-shaped-write compatibility tests before publishing. - Schedule a docs update walkthrough with maintainers for major releases. ## Post-Release diff --git a/packages/stem/doc/internal/operations/operations-guide.md b/packages/stem/doc/internal/operations/operations-guide.md index 6505fcce..302420b6 100644 --- a/packages/stem/doc/internal/operations/operations-guide.md +++ b/packages/stem/doc/internal/operations/operations-guide.md @@ -44,7 +44,7 @@ docker compose -f packages/stem_cli/docker/testing/docker-compose.yml up -d post export STEM_TEST_REDIS_URL=redis://127.0.0.1:56379 export STEM_TEST_POSTGRES_URL=postgresql://postgres:postgres@127.0.0.1:65432/stem_test export STEM_TEST_POSTGRES_TLS_URL=$STEM_TEST_POSTGRES_URL -export STEM_TEST_POSTGRES_TLS_CA_CERT=packages/stem_cli/docker/testing/certs/postgres-root.crt +export STEM_TEST_POSTGRES_TLS_CA_CERT=packages/stem_cli/docker/testing/postgres/certs/root.crt ``` With the services running, execute `dart test` in `packages/stem_redis`, diff --git a/packages/stem/doc/internal/operations/testing-guide.md b/packages/stem/doc/internal/operations/testing-guide.md index bf53c9f0..31628e88 100644 --- a/packages/stem/doc/internal/operations/testing-guide.md +++ b/packages/stem/doc/internal/operations/testing-guide.md @@ -9,17 +9,13 @@ Stem uses a consolidated quality workflow to keep local checks aligned with CI. ## Quality gates -Use the `just` runner in `example/quality_gates`: +Run the package gates directly: ```bash -cd packages/stem/example/quality_gates -just quality -``` - -For a faster loop: - -```bash -just quick +cd packages/stem +dart format lib test --set-exit-if-changed +dart analyze --fatal-infos +dart test --exclude-tags soak --fail-fast ``` Expanded steps: @@ -27,9 +23,8 @@ Expanded steps: 1. `dart format --set-exit-if-changed .` 2. `dart analyze` 3. `dart test --exclude-tags soak` -4. Chaos + performance suites (see `example/quality_gates/justfile`) -5. Coverage via `tool/quality/coverage.sh` (threshold 60% unless overridden via - `COVERAGE_THRESHOLD`) +4. Package-specific integration and adapter contract suites +5. Coverage via the package coverage tasks (thresholds vary by package) ### Chaos suite against Redis @@ -53,10 +48,12 @@ dart test --tags soak ## Continuous Integration -`.github/workflows/ci.yml` now: +`.github/workflows/aggregate.yaml` now: -- Starts a Redis 7 service container for chaos tests. -- Runs format, analyze, unit tests, chaos tests, and coverage gates. +- Runs quality gates for every publishable workspace package. +- Checks standalone dependency resolution without workspace overrides. +- Runs core checks on Ubuntu, Windows, and macOS, plus every example project + discovered under the root workspace packages. - Fails immediately if any quality step fails. This keeps local and CI behaviour aligned and ensures resilience regressions diff --git a/packages/stem/doc/process/observability-runbook.md b/packages/stem/doc/process/observability-runbook.md index f8b4d584..a7f28a7a 100644 --- a/packages/stem/doc/process/observability-runbook.md +++ b/packages/stem/doc/process/observability-runbook.md @@ -1,8 +1,8 @@ # Observability Runbook -> **Note:** The canonical version of this document lives at -> `.site/docs/observability-runbook.md` for publication on the docs site. Keep -> the two files in sync when editing. +> **Note:** The public operator guide is +> `.site/docs/getting-started/observability-and-ops.md`. This internal runbook +> contains SLO and alerting details that are maintained for operations review. ## SLOs | Service Aspect | Target | Measurement Window | Notes | diff --git a/packages/stem/doc/process/roadmap.md b/packages/stem/doc/process/roadmap.md index 5d7beffd..178550f1 100644 --- a/packages/stem/doc/process/roadmap.md +++ b/packages/stem/doc/process/roadmap.md @@ -1,50 +1,7 @@ -# Stem Roadmap & Milestones +# Roadmap pointer -> **Note:** The canonical version of this roadmap now lives at -> `.site/docs/roadmap.md` for the public documentation site. +The canonical roadmap is maintained in the documentation site at +`.site/docs/about/roadmap.md` and is published at `/about/roadmap`. -## Milestones - -### M1 – Core Runtime -- Redis Streams broker & Redis result backend -- Worker daemon (prefetch, retries, lease renewal) -- CLI basics (`stem enqueue`, `stem observe metrics`) - -### M2 – Scheduling & DLQ -- Beat (interval/cron + jitter + locks) -- Dead-letter queues with CLI list/show/replay -- Backoff tuning + rate limiter integration - -### M3 – Observability & Docs -- OpenTelemetry metrics/traces/logging -- CLI/Docs for observability, scaling, quick start -- Samples (monolith, microservice, OTLP stack) - -### M4 – Resilience Enhancements -- Hard timeouts & isolate pool polish -- Reclaim daemon & rate limiter store -- Locks/rate limiter robustness + release process docs - -## Definition of Ready (DoR) -- User story linked to OpenSpec change/spec delta. -- Acceptance criteria include success/failure behaviour, error handling, docs/test notes. -- Dependencies (other stories, infrastructure) documented. - -## Definition of Done (DoD) -- `dart format`, `dart analyze`, `dart test` pass locally & in CI. -- Relevant docs (developer, ops, quick start) updated. -- Examples compile/run if affected. -- Observability metrics/traces updated when behaviour changes. -- OpenSpec tasks updated to `- [x]` with evidence (links, tests). - -## Tracking -- Use GitHub milestones M1–M4 mirroring the above. -- Label issues with `milestone:Mx` and `type:doc`, `type:impl`, `type:test` as appropriate. -- Review DoR/DoD compliance during grooming. - -## Backlog / Outstanding Phase Work -- **Phase 5 (Testing)**: monitor soak/perf dashboards and automate regression alerts. -- **Phase 6 (Observability)**: implement dashboards/alerts (Grafana) tied to the SLOs and document owners. -- **Phase 7 (Security)**: payload signing helper, TLS automation scripts, recurring vulnerability scans. -- **Phase 9 (Release)**: Beta/RC pilot tracking, automated changelog generation. -- **Phase 10 (Ops)**: CLI pause/resume/drain, auto-scaling hints, roadmap refinement (RabbitMQ/SQS, Postgres backend). +Keep release planning and implementation status in that page rather than +duplicating a milestone list in the process documentation. diff --git a/packages/stem/doc/process/security-checklist.md b/packages/stem/doc/process/security-checklist.md index da7afae8..804b5c34 100644 --- a/packages/stem/doc/process/security-checklist.md +++ b/packages/stem/doc/process/security-checklist.md @@ -1,6 +1,8 @@ # Security Checklist -> **Note:** Maintained version is published via `.site/docs/security-checklist.md`. +> **Note:** The public deployment checklist is +> `.site/docs/getting-started/production-checklist.md`. This internal checklist +> contains additional review prompts. ## Transport & Secrets - [ ] Enforce TLS for Redis connections (`rediss://`); document certificate management. diff --git a/packages/stem/doc/process/security-examples.md b/packages/stem/doc/process/security-examples.md index 287b1b65..c78ecb41 100644 --- a/packages/stem/doc/process/security-examples.md +++ b/packages/stem/doc/process/security-examples.md @@ -1,7 +1,9 @@ # Security Configuration Examples -> **Note:** The version served on the docs site lives at -> `.site/docs/security-examples.md`. +> **Note:** Public signing examples live in +> `.site/docs/core-concepts/signing.md`; the production deployment checklist +> is at `.site/docs/getting-started/production-checklist.md`. This file keeps +> additional environment-oriented examples for maintainers. These scenarios illustrate how to wire signing and transport security for Stem services (producers, workers, beat, CLI) without referring to any pre-built diff --git a/packages/stem/doc/process/security-runbook.md b/packages/stem/doc/process/security-runbook.md index 8a4ed6a6..89cf727e 100644 --- a/packages/stem/doc/process/security-runbook.md +++ b/packages/stem/doc/process/security-runbook.md @@ -1,7 +1,9 @@ # Security Runbook -> **Note:** The public docs version is `.site/docs/security-runbook.md`; keep -> both files aligned. +> **Note:** Public starting points are +> `.site/docs/core-concepts/signing.md` and +> `.site/docs/getting-started/production-checklist.md`. This internal runbook +> contains deeper rotation and TLS procedures. ## Payload Signing Key Rotation diff --git a/packages/stem/doc/process/stem-signals.md b/packages/stem/doc/process/stem-signals.md index 8c2387a1..f0ccefba 100644 --- a/packages/stem/doc/process/stem-signals.md +++ b/packages/stem/doc/process/stem-signals.md @@ -1,7 +1,7 @@ # Stem Signals -> **Note:** This doc is mirrored on the published site at -> `.site/docs/signals.md`; keep both versions aligned. +> **Note:** The public version is `.site/docs/core-concepts/signals.md`. +> This file retains the longer process notes. Stem exposes Celery-style lifecycle hooks via a strongly typed signal dispatcher. Signals supplement middleware so instrumentation, observability, diff --git a/packages/stem/doc/process/testing.md b/packages/stem/doc/process/testing.md index 49410531..e2a554ae 100644 --- a/packages/stem/doc/process/testing.md +++ b/packages/stem/doc/process/testing.md @@ -2,7 +2,8 @@ title: Testing & Quality Gates --- -> **Note:** Primary content is published via `.site/docs/testing-guide.md`. +> **Note:** Primary content is published via +> `.site/docs/about/testing.md`. The project ships with a consolidated quality workflow that contributors and CI use to enforce formatting, static analysis, tests, coverage, and chaos @@ -11,14 +12,17 @@ resilience checks. ## Running the full suite ``` -cd packages/stem/example/quality_gates -just quality +cd packages/stem +dart format lib test --set-exit-if-changed +dart analyze --fatal-infos +dart test --exclude-tags soak --fail-fast ``` For a faster loop: ``` -just quick +dart analyze --fatal-infos +dart test --exclude-tags soak --fail-fast ``` The quality runner executes: @@ -26,8 +30,8 @@ The quality runner executes: 1. `dart format --set-exit-if-changed .` 2. `dart analyze` 3. `dart test --exclude-tags soak` -4. Chaos + performance suites (see `example/quality_gates/justfile`) -5. Coverage (see `tool/quality/coverage.sh` for thresholded runs) +4. Package-specific integration and adapter contract suites +5. Coverage and soak tests when their explicit gates are requested If you prefer running the core gates directly: @@ -74,11 +78,12 @@ dart test --tags soak ## CI workflow -The GitHub Actions workflow (`.github/workflows/ci.yml`) now: +The GitHub Actions aggregate workflow (`.github/workflows/aggregate.yaml`) now: -- Provisions a Redis service container for chaos tests. -- Executes format, analyze, unit tests, chaos tests, and coverage gates (either - via `example/quality_gates` or by calling the commands directly). +- Runs package quality gates for every publishable workspace package. +- Checks standalone dependency resolution without workspace overrides. +- Runs core checks on Ubuntu, Windows, and macOS, plus every example project + discovered under the root workspace packages. Any failures in format, analyze, unit/integration tests, coverage, or chaos recovery cause the pipeline to fail. diff --git a/packages/stem/doc/process/worker-control.md b/packages/stem/doc/process/worker-control.md index 22c5c49a..7b583cb5 100644 --- a/packages/stem/doc/process/worker-control.md +++ b/packages/stem/doc/process/worker-control.md @@ -1,7 +1,8 @@ # Worker Control CLI -> **Note:** This doc is now mirrored on the published site at -> `.site/docs/worker-control.md`; keep both versions aligned. +> **Note:** The public version is +> `.site/docs/workers/worker-control.md`. This file retains the longer +> operator-oriented process notes. Stem ships a broker-backed control plane that lets operators inspect and revoke work without restarts. The CLI subcommands live under `stem worker` and use the diff --git a/packages/stem/example/annotated_workflows/bin/main.dart b/packages/stem/example/annotated_workflows/bin/main.dart index dbc75bf6..1cd6d19c 100644 --- a/packages/stem/example/annotated_workflows/bin/main.dart +++ b/packages/stem/example/annotated_workflows/bin/main.dart @@ -6,6 +6,7 @@ import 'package:stem_annotated_workflows/definitions.dart'; Future main() async { final client = await StemClient.inMemory(module: stemModule); final app = await client.createWorkflowApp(); + await app.start(); final flowRunId = await StemWorkflowDefinitions.flow.start(app); final flowResult = await StemWorkflowDefinitions.flow.waitFor( diff --git a/packages/stem/example/annotated_workflows/lib/definitions.stem.g.dart b/packages/stem/example/annotated_workflows/lib/definitions.stem.g.dart index 94082532..57d5aa62 100644 --- a/packages/stem/example/annotated_workflows/lib/definitions.stem.g.dart +++ b/packages/stem/example/annotated_workflows/lib/definitions.stem.g.dart @@ -222,6 +222,29 @@ Object? _stemRequireArg(Map args, String name) { return args[name]; } +TaskInvocationContext _stemTaskInvocationContext( + TaskExecutionContext context, + Map args, +) { + if (context case final TaskInvocationContext value) { + return value; + } + return TaskInvocationContext.local( + id: context.id, + args: args, + headers: context.headers, + meta: context.meta, + attempt: context.attempt, + heartbeat: context.heartbeat, + extendLease: context.extendLease, + progress: (percent, {data}) => context.progress(percent, data: data), + cancellation: context.cancellation, + enqueuer: context, + workflows: context, + workflowEvents: context, + ); +} + Future _stemTaskAdapter0( TaskInvocationContext context, Map args, @@ -246,6 +269,7 @@ abstract final class StemTaskDefinitions { TaskDefinition, Object?>( name: "send_email", encodeArgs: (args) => args, + decodeArgs: (args) => args, defaultOptions: const TaskOptions(maxRetries: 1), metadata: const TaskMetadata(), ); @@ -255,23 +279,10 @@ abstract final class StemTaskDefinitions { encodeArgs: (args) => { "dispatch": StemPayloadCodecs.emailDispatch.encode(args), }, + decodeArgs: (args) => StemPayloadCodecs.emailDispatch.decode( + _stemRequireArg(args, "dispatch"), + ), defaultOptions: const TaskOptions(maxRetries: 1), - metadata: const TaskMetadata(), - decodeResult: StemPayloadCodecs.emailDeliveryReceipt.decode, - ); -} - -final List> _stemTasks = >[ - FunctionTaskHandler( - name: "send_email", - entrypoint: _stemTaskAdapter0, - options: const TaskOptions(maxRetries: 1), - metadata: const TaskMetadata(), - ), - FunctionTaskHandler( - name: "send_email_typed", - entrypoint: _stemTaskAdapter1, - options: const TaskOptions(maxRetries: 1), metadata: TaskMetadata( tags: [], idempotent: false, @@ -281,8 +292,35 @@ final List> _stemTasks = >[ codec: StemPayloadCodecs.emailDeliveryReceipt, ), ), - ), -]; + decodeResult: StemPayloadCodecs.emailDeliveryReceipt.decode, + ); +} + +final List> _stemTasks = + >[ + StemTaskDefinitions.sendEmail.handler( + entrypoint: (context, args) => sendEmail( + args, + context: _stemTaskInvocationContext( + context, + StemTaskDefinitions.sendEmail.encodeArgs(args), + ), + ), + executionMode: TaskExecutionMode.isolate, + isolateEntrypoint: _stemTaskAdapter0, + ), + StemTaskDefinitions.sendEmailTyped.handler( + entrypoint: (context, args) => sendEmailTyped( + args, + context: _stemTaskInvocationContext( + context, + StemTaskDefinitions.sendEmailTyped.encodeArgs(args), + ), + ), + executionMode: TaskExecutionMode.isolate, + isolateEntrypoint: _stemTaskAdapter1, + ), + ]; final List _stemWorkflowManifest = [ diff --git a/packages/stem/example/canvas_patterns/chain_example.dart b/packages/stem/example/canvas_patterns/chain_example.dart index 18102ef6..7ad09d35 100644 --- a/packages/stem/example/canvas_patterns/chain_example.dart +++ b/packages/stem/example/canvas_patterns/chain_example.dart @@ -36,6 +36,7 @@ Future main() async { prefetchMultiplier: 1, ), ); + await app.start(); final chainResult = await app.canvas.chain([ task('fetch.user'), task('enrich.user'), diff --git a/packages/stem/example/canvas_patterns/chord_example.dart b/packages/stem/example/canvas_patterns/chord_example.dart index f504e1ec..12415124 100644 --- a/packages/stem/example/canvas_patterns/chord_example.dart +++ b/packages/stem/example/canvas_patterns/chord_example.dart @@ -33,6 +33,7 @@ Future main() async { prefetchMultiplier: 1, ), ); + await app.start(); final chordResult = await app.canvas.chord( body: [ task('fetch.metric', args: {'value': 5}), diff --git a/packages/stem/example/canvas_patterns/group_example.dart b/packages/stem/example/canvas_patterns/group_example.dart index 37e95003..f0dd53ce 100644 --- a/packages/stem/example/canvas_patterns/group_example.dart +++ b/packages/stem/example/canvas_patterns/group_example.dart @@ -20,6 +20,7 @@ Future main() async { prefetchMultiplier: 1, ), ); + await app.start(); const groupHandle = 'squares-demo'; final dispatch = await app.canvas.group([ task('square', args: {'value': 2}), diff --git a/packages/stem/example/docs_snippets/lib/best_practices.dart b/packages/stem/example/docs_snippets/lib/best_practices.dart index 7f4e086b..1368db7e 100644 --- a/packages/stem/example/docs_snippets/lib/best_practices.dart +++ b/packages/stem/example/docs_snippets/lib/best_practices.dart @@ -36,6 +36,7 @@ Future main() async { final app = await StemApp.inMemory( tasks: [IdempotentTask()], ); + await app.start(); await enqueueTyped(app); await app.close(); diff --git a/packages/stem/example/docs_snippets/lib/brokers.dart b/packages/stem/example/docs_snippets/lib/brokers.dart index e7e4126e..e12d2f2b 100644 --- a/packages/stem/example/docs_snippets/lib/brokers.dart +++ b/packages/stem/example/docs_snippets/lib/brokers.dart @@ -1,6 +1,7 @@ import 'dart:io'; import 'package:stem/stem.dart'; +import 'package:stem/memory.dart'; import 'package:stem_postgres/stem_postgres.dart'; import 'package:stem_redis/stem_redis.dart'; import 'package:stem_sqlite/stem_sqlite.dart'; diff --git a/packages/stem/example/docs_snippets/lib/canvas_batch.dart b/packages/stem/example/docs_snippets/lib/canvas_batch.dart index 698996b0..3374c6b8 100644 --- a/packages/stem/example/docs_snippets/lib/canvas_batch.dart +++ b/packages/stem/example/docs_snippets/lib/canvas_batch.dart @@ -21,6 +21,7 @@ Future main() async { prefetchMultiplier: 1, ), ); + await app.start(); final submission = await app.canvas.submitBatch([ task('batch.double', args: {'value': 1}), diff --git a/packages/stem/example/docs_snippets/lib/canvas_chain.dart b/packages/stem/example/docs_snippets/lib/canvas_chain.dart index 27d9b34b..64e3e98b 100644 --- a/packages/stem/example/docs_snippets/lib/canvas_chain.dart +++ b/packages/stem/example/docs_snippets/lib/canvas_chain.dart @@ -41,6 +41,7 @@ Future main() async { prefetchMultiplier: 1, ), ); + await app.start(); final canvas = app.canvas; final chainResult = await canvas.chain([ diff --git a/packages/stem/example/docs_snippets/lib/canvas_chord.dart b/packages/stem/example/docs_snippets/lib/canvas_chord.dart index 84f77f55..e6a5b9c5 100644 --- a/packages/stem/example/docs_snippets/lib/canvas_chord.dart +++ b/packages/stem/example/docs_snippets/lib/canvas_chord.dart @@ -35,6 +35,7 @@ Future main() async { prefetchMultiplier: 1, ), ); + await app.start(); final canvas = app.canvas; final chordResult = await canvas.chord( diff --git a/packages/stem/example/docs_snippets/lib/canvas_group.dart b/packages/stem/example/docs_snippets/lib/canvas_group.dart index c3de32af..7824c8c9 100644 --- a/packages/stem/example/docs_snippets/lib/canvas_group.dart +++ b/packages/stem/example/docs_snippets/lib/canvas_group.dart @@ -24,6 +24,7 @@ Future main() async { prefetchMultiplier: 1, ), ); + await app.start(); final canvas = app.canvas; final dispatch = await canvas.group([ diff --git a/packages/stem/example/docs_snippets/lib/first_steps.dart b/packages/stem/example/docs_snippets/lib/first_steps.dart index fc19c2ca..dc4eb871 100644 --- a/packages/stem/example/docs_snippets/lib/first_steps.dart +++ b/packages/stem/example/docs_snippets/lib/first_steps.dart @@ -31,6 +31,7 @@ Future runInMemoryDemo() async { consumerName: 'first-steps-worker', ), ); + await app.start(); // #endregion first-steps-bootstrap // #region first-steps-enqueue diff --git a/packages/stem/example/docs_snippets/lib/namespaces.dart b/packages/stem/example/docs_snippets/lib/namespaces.dart index 59c6943c..a9547e94 100644 --- a/packages/stem/example/docs_snippets/lib/namespaces.dart +++ b/packages/stem/example/docs_snippets/lib/namespaces.dart @@ -2,6 +2,7 @@ // ignore_for_file: unused_local_variable, unused_import, dead_code import 'package:stem/stem.dart'; +import 'package:stem/memory.dart'; import 'package:stem_redis/stem_redis.dart'; // #region namespaces-broker diff --git a/packages/stem/example/docs_snippets/lib/observability.dart b/packages/stem/example/docs_snippets/lib/observability.dart index a7d8b84e..1ac41c92 100644 --- a/packages/stem/example/docs_snippets/lib/observability.dart +++ b/packages/stem/example/docs_snippets/lib/observability.dart @@ -2,6 +2,7 @@ // ignore_for_file: unused_local_variable, unused_import, dead_code, avoid_print import 'package:stem/stem.dart'; +import 'package:stem/observability.dart'; // #region observability-metrics void configureMetrics() { @@ -10,14 +11,10 @@ void configureMetrics() { // #endregion observability-metrics // #region observability-tracing -Future buildTracedStem( - Iterable> tasks, -) { +Future buildTracedStem(Iterable> tasks) { // Configure OpenTelemetry globally; StemTracer.instance reads from it. final _ = StemTracer.instance; - return StemClient.inMemory( - tasks: tasks, - ); + return StemClient.inMemory(tasks: tasks); } // #endregion observability-tracing @@ -46,7 +43,7 @@ void logTaskStart(Envelope envelope) { configureStemLogging(format: StemLogFormat.pretty); stemLogger.info( 'Task started', - Context({'task': envelope.name, 'id': envelope.id}), + fields: {'task': envelope.name, 'id': envelope.id}, ); } // #endregion observability-logging @@ -79,12 +76,7 @@ Future main() async { final client = await buildTracedStem(tasks); - logTaskStart( - Envelope( - name: traceTaskDefinition.name, - args: const {}, - ), - ); + logTaskStart(Envelope(name: traceTaskDefinition.name, args: const {})); await traceTaskDefinition.enqueue(client); await client.close(); } diff --git a/packages/stem/example/docs_snippets/lib/persistence.dart b/packages/stem/example/docs_snippets/lib/persistence.dart index 2a51f13f..9ebea920 100644 --- a/packages/stem/example/docs_snippets/lib/persistence.dart +++ b/packages/stem/example/docs_snippets/lib/persistence.dart @@ -5,6 +5,7 @@ import 'dart:convert'; import 'dart:io'; import 'package:stem/stem.dart'; +import 'package:stem/memory.dart'; import 'package:stem_postgres/stem_postgres.dart'; import 'package:stem_redis/stem_redis.dart'; import 'package:stem_sqlite/stem_sqlite.dart'; diff --git a/packages/stem/example/docs_snippets/lib/producer.dart b/packages/stem/example/docs_snippets/lib/producer.dart index 9f8054ad..58a631ac 100644 --- a/packages/stem/example/docs_snippets/lib/producer.dart +++ b/packages/stem/example/docs_snippets/lib/producer.dart @@ -21,6 +21,7 @@ Future enqueueInMemory() async { ], ); final app = await client.createApp(); + await app.start(); final taskId = await app.enqueue( 'hello.print', @@ -126,6 +127,7 @@ class GenerateReportTask extends TaskHandler { Future enqueueTyped() async { final client = await StemClient.inMemory(tasks: [GenerateReportTask()]); final app = await client.createApp(); + await app.start(); final result = await GenerateReportTask.definition.enqueueAndWait( app, diff --git a/packages/stem/example/docs_snippets/lib/quick_start.dart b/packages/stem/example/docs_snippets/lib/quick_start.dart index 23301649..568c497e 100644 --- a/packages/stem/example/docs_snippets/lib/quick_start.dart +++ b/packages/stem/example/docs_snippets/lib/quick_start.dart @@ -18,7 +18,7 @@ class ResizeImageTask extends TaskHandler { softTimeLimit: Duration(seconds: 10), hardTimeLimit: Duration(seconds: 20), priority: 7, - rateLimit: '20/m', + rateLimit: const RateLimit.perMinute(20), visibilityTimeout: Duration(seconds: 60), ); @@ -64,6 +64,7 @@ Future main() async { concurrency: 4, ), ); + await app.start(); // #endregion quickstart-bootstrap // #region quickstart-enqueue diff --git a/packages/stem/example/docs_snippets/lib/quick_start_failure.dart b/packages/stem/example/docs_snippets/lib/quick_start_failure.dart index 2c8f2746..0d3cafe3 100644 --- a/packages/stem/example/docs_snippets/lib/quick_start_failure.dart +++ b/packages/stem/example/docs_snippets/lib/quick_start_failure.dart @@ -27,7 +27,10 @@ class EmailReceiptTask extends TaskHandler { // #endregion quickstart-email-failure Future main() async { - final app = await StemApp.inMemory(tasks: [EmailReceiptTask()]); + final app = await StemApp.inMemory( + tasks: [EmailReceiptTask()], + ); + await app.start(); final taskId = await app.enqueue( 'billing.email-receipt', diff --git a/packages/stem/example/docs_snippets/lib/rate_limiting.dart b/packages/stem/example/docs_snippets/lib/rate_limiting.dart index 8d602c13..f9cae69d 100644 --- a/packages/stem/example/docs_snippets/lib/rate_limiting.dart +++ b/packages/stem/example/docs_snippets/lib/rate_limiting.dart @@ -57,7 +57,7 @@ class RateLimitedTask extends TaskHandler { @override TaskOptions get options => const TaskOptions( - rateLimit: '10/s', + rateLimit: const RateLimit.perSecond(10), maxRetries: 3, ); @@ -77,7 +77,7 @@ class GroupRateLimitedTask extends TaskHandler { @override TaskOptions get options => const TaskOptions( - groupRateLimit: '20/m', + groupRateLimit: const RateLimit.perMinute(20), groupRateKeyHeader: 'tenant', groupRateLimiterFailureMode: RateLimiterFailureMode.failClosed, maxRetries: 5, @@ -114,6 +114,9 @@ Future main() async { tasks: [RateLimitedTask()], workerConfig: workerConfig, ); + // #region rate-limit-demo-worker-start + await app.start(); + // #endregion rate-limit-demo-worker-start // #endregion rate-limit-demo-registry // #region rate-limit-demo-stem diff --git a/packages/stem/example/docs_snippets/lib/retry_backoff.dart b/packages/stem/example/docs_snippets/lib/retry_backoff.dart index d72e6783..63fd75ad 100644 --- a/packages/stem/example/docs_snippets/lib/retry_backoff.dart +++ b/packages/stem/example/docs_snippets/lib/retry_backoff.dart @@ -66,6 +66,7 @@ Future main() async { tasks: [FlakyTask()], workerConfig: workerConfig, ); + await app.start(); final taskId = await FlakyTask.definition.enqueue(app); await app.waitForTask(taskId, timeout: const Duration(seconds: 5)); diff --git a/packages/stem/example/docs_snippets/lib/scheduler.dart b/packages/stem/example/docs_snippets/lib/scheduler.dart index 0aa17799..556506aa 100644 --- a/packages/stem/example/docs_snippets/lib/scheduler.dart +++ b/packages/stem/example/docs_snippets/lib/scheduler.dart @@ -5,6 +5,7 @@ import 'dart:async'; import 'dart:io'; import 'package:stem/stem.dart'; +import 'package:stem/memory.dart'; import 'package:stem_postgres/stem_postgres.dart'; import 'package:stem_redis/stem_redis.dart'; diff --git a/packages/stem/example/docs_snippets/lib/signing.dart b/packages/stem/example/docs_snippets/lib/signing.dart index d8b5218a..bf920046 100644 --- a/packages/stem/example/docs_snippets/lib/signing.dart +++ b/packages/stem/example/docs_snippets/lib/signing.dart @@ -5,6 +5,7 @@ import 'dart:async'; import 'package:stem/stem.dart'; +import 'package:stem/memory.dart'; class BillingTask extends TaskHandler { @override diff --git a/packages/stem/example/docs_snippets/lib/tasks.dart b/packages/stem/example/docs_snippets/lib/tasks.dart index bd9caae4..32ee7376 100644 --- a/packages/stem/example/docs_snippets/lib/tasks.dart +++ b/packages/stem/example/docs_snippets/lib/tasks.dart @@ -85,6 +85,7 @@ Future runTypedDefinitionExample() async { tasks: [PublishInvoiceTask()], ); final app = await client.createApp(); + await app.start(); final result = await PublishInvoiceTask.definition.enqueueAndWait( app, @@ -223,6 +224,7 @@ class MyOtherEncoder extends TaskPayloadEncoder { Future main() async { final client = await StemClient.inMemory(tasks: [EmailTask()]); final app = await client.createApp(); + await app.start(); final taskId = await app.enqueue( 'email.send', diff --git a/packages/stem/example/docs_snippets/lib/troubleshooting.dart b/packages/stem/example/docs_snippets/lib/troubleshooting.dart index 3e775ecc..6e27a963 100644 --- a/packages/stem/example/docs_snippets/lib/troubleshooting.dart +++ b/packages/stem/example/docs_snippets/lib/troubleshooting.dart @@ -32,6 +32,7 @@ Future runTroubleshootingDemo() async { concurrency: 1, ), ); + await app.start(); // #endregion troubleshooting-bootstrap // #region troubleshooting-enqueue diff --git a/packages/stem/example/docs_snippets/lib/uniqueness.dart b/packages/stem/example/docs_snippets/lib/uniqueness.dart index 36473579..b963c0f5 100644 --- a/packages/stem/example/docs_snippets/lib/uniqueness.dart +++ b/packages/stem/example/docs_snippets/lib/uniqueness.dart @@ -5,6 +5,7 @@ import 'dart:async'; import 'dart:io'; import 'package:stem/stem.dart'; +import 'package:stem/memory.dart'; import 'package:stem_redis/stem_redis.dart'; class SendDigestTask extends TaskHandler { diff --git a/packages/stem/example/docs_snippets/lib/worker_control.dart b/packages/stem/example/docs_snippets/lib/worker_control.dart index b1ceafc8..9e91fb44 100644 --- a/packages/stem/example/docs_snippets/lib/worker_control.dart +++ b/packages/stem/example/docs_snippets/lib/worker_control.dart @@ -2,6 +2,7 @@ // ignore_for_file: unused_local_variable, unused_import, dead_code, avoid_print import 'package:stem/stem.dart'; +import 'package:stem/memory.dart'; final InMemoryBroker _autoscaleBroker = InMemoryBroker(); final InMemoryResultBackend _autoscaleBackend = InMemoryResultBackend(); diff --git a/packages/stem/example/docs_snippets/lib/workers_programmatic.dart b/packages/stem/example/docs_snippets/lib/workers_programmatic.dart index 8123dcc3..38fc4d2d 100644 --- a/packages/stem/example/docs_snippets/lib/workers_programmatic.dart +++ b/packages/stem/example/docs_snippets/lib/workers_programmatic.dart @@ -6,6 +6,7 @@ import 'dart:async'; import 'dart:io'; import 'package:stem/stem.dart'; +import 'package:stem/memory.dart'; import 'package:stem_redis/stem_redis.dart'; // #region workers-producer-minimal diff --git a/packages/stem/example/docs_snippets/pubspec.yaml b/packages/stem/example/docs_snippets/pubspec.yaml index e8a8c5ff..11bab638 100644 --- a/packages/stem/example/docs_snippets/pubspec.yaml +++ b/packages/stem/example/docs_snippets/pubspec.yaml @@ -1,15 +1,25 @@ name: stem_examples description: Runnable code examples for Stem documentation publish_to: none -resolution: workspace environment: sdk: ">=3.9.2 <4.0.0" dependencies: stem: + path: ../.. stem_redis: + path: ../../../stem_redis stem_postgres: + path: ../../../stem_postgres contextual: ^2.0.0 dev_dependencies: test: ^1.28.0 + +dependency_overrides: + stem: + path: ../.. + stem_redis: + path: ../../../stem_redis + stem_postgres: + path: ../../../stem_postgres diff --git a/packages/stem/example/durable_watchers.dart b/packages/stem/example/durable_watchers.dart index 5098aab5..296e652f 100644 --- a/packages/stem/example/durable_watchers.dart +++ b/packages/stem/example/durable_watchers.dart @@ -35,6 +35,7 @@ Future main() async { final app = await StemWorkflowApp.inMemory( scripts: [shipmentWorkflow], ); + await app.start(); final runId = await shipmentWorkflowRef.start( app, diff --git a/packages/stem/example/ecommerce/README.md b/packages/stem/example/ecommerce/README.md index af8027a7..9f004727 100644 --- a/packages/stem/example/ecommerce/README.md +++ b/packages/stem/example/ecommerce/README.md @@ -82,7 +82,8 @@ Database boot sequence on startup: [`lib/src/database/migrations.dart`](/run/media/kingwill101/disk2/code/code/dart_packages/stem/packages/stem/example/ecommerce/lib/src/database/migrations.dart) - seeds default catalog records if empty -Optional CLI migration command (when your local `ormed_cli` dependency set is compatible): +Optional CLI migration command (install a compatible `ormed_cli` separately when +you need to manage the example database): ```bash dart run ormed_cli:ormed migrate --config ormed.yaml diff --git a/packages/stem/example/ecommerce/lib/src/database/orm_registry.g.dart b/packages/stem/example/ecommerce/lib/src/database/orm_registry.g.dart index 0103746b..2296b040 100644 --- a/packages/stem/example/ecommerce/lib/src/database/orm_registry.g.dart +++ b/packages/stem/example/ecommerce/lib/src/database/orm_registry.g.dart @@ -21,7 +21,8 @@ ModelRegistry buildOrmRegistry() => ModelRegistry() ..registerTypeAlias(_$ormModelDefinitions[1]) ..registerTypeAlias(_$ormModelDefinitions[2]) ..registerTypeAlias(_$ormModelDefinitions[3]) - ..registerTypeAlias(_$ormModelDefinitions[4]); + ..registerTypeAlias(_$ormModelDefinitions[4]) + ; List> get generatedOrmModelDefinitions => List.unmodifiable(_$ormModelDefinitions); @@ -40,7 +41,8 @@ extension GeneratedOrmModels on ModelRegistry { /// Registers factory definitions for all models that have factory support. /// Call this before using [Model.factory()] to ensure definitions are available. -void registerOrmFactories() {} +void registerOrmFactories() { +} /// Combined setup: registers both model registry and factories. /// Returns a ModelRegistry with all generated models registered. @@ -60,14 +62,7 @@ void registerModelScopes({ScopeRegistry? scopeRegistry}) { } /// Bootstraps generated ORM pieces: registry, factories, event handlers, and scopes. -ModelRegistry bootstrapOrm({ - ModelRegistry? registry, - EventBus? bus, - ScopeRegistry? scopes, - bool registerFactories = true, - bool registerEventHandlers = true, - bool registerScopes = true, -}) { +ModelRegistry bootstrapOrm({ModelRegistry? registry, EventBus? bus, ScopeRegistry? scopes, bool registerFactories = true, bool registerEventHandlers = true, bool registerScopes = true}) { final reg = registry ?? buildOrmRegistry(); if (registry != null) { reg.registerGeneratedModels(); diff --git a/packages/stem/example/ecommerce/lib/src/workflows/annotated_defs.stem.g.dart b/packages/stem/example/ecommerce/lib/src/workflows/annotated_defs.stem.g.dart index e8ed64bf..c2d3923f 100644 --- a/packages/stem/example/ecommerce/lib/src/workflows/annotated_defs.stem.g.dart +++ b/packages/stem/example/ecommerce/lib/src/workflows/annotated_defs.stem.g.dart @@ -83,6 +83,29 @@ Object? _stemRequireArg(Map args, String name) { return args[name]; } +TaskInvocationContext _stemTaskInvocationContext( + TaskExecutionContext context, + Map args, +) { + if (context case final TaskInvocationContext value) { + return value; + } + return TaskInvocationContext.local( + id: context.id, + args: args, + headers: context.headers, + meta: context.meta, + attempt: context.attempt, + heartbeat: context.heartbeat, + extendLease: context.extendLease, + progress: (percent, {data}) => context.progress(percent, data: data), + cancellation: context.cancellation, + enqueuer: context, + workflows: context, + workflowEvents: context, + ); +} + Future _stemTaskAdapter0( TaskInvocationContext context, Map args, @@ -113,20 +136,31 @@ abstract final class StemTaskDefinitions { "entityId": args.entityId, "detail": args.detail, }, + decodeArgs: (args) => ( + event: (_stemRequireArg(args, "event") as String), + entityId: (_stemRequireArg(args, "entityId") as String), + detail: (_stemRequireArg(args, "detail") as String), + ), defaultOptions: const TaskOptions(queue: "default"), metadata: const TaskMetadata(), ); } -final List> _stemTasks = >[ - FunctionTaskHandler( - name: "ecommerce.audit.log", - entrypoint: _stemTaskAdapter0, - options: const TaskOptions(queue: "default"), - metadata: const TaskMetadata(), - runInIsolate: false, - ), -]; +final List> _stemTasks = + >[ + StemTaskDefinitions.ecommerceAuditLog.handler( + entrypoint: (context, args) => logAuditEvent( + _stemTaskInvocationContext( + context, + StemTaskDefinitions.ecommerceAuditLog.encodeArgs(args), + ), + args.event, + args.entityId, + args.detail, + ), + executionMode: TaskExecutionMode.inline, + ), + ]; final List _stemWorkflowManifest = [ diff --git a/packages/stem/example/ecommerce/pubspec.yaml b/packages/stem/example/ecommerce/pubspec.yaml index 975c540a..220b3b53 100644 --- a/packages/stem/example/ecommerce/pubspec.yaml +++ b/packages/stem/example/ecommerce/pubspec.yaml @@ -8,7 +8,7 @@ environment: dependencies: ormed: ^0.2.0 - ormed_sqlite: ^0.2.0 + ormed_sqlite: ^0.3.0 path: ^1.9.1 shelf: ^1.4.2 shelf_router: ^1.1.4 @@ -19,7 +19,6 @@ dependencies: dev_dependencies: build_runner: ^2.10.5 - ormed_cli: ^0.2.0 lints: ^6.0.0 server_testing: ^0.3.2 server_testing_shelf: ^0.3.2 diff --git a/packages/stem/example/flutter_stem_example/analysis_options.yaml b/packages/stem/example/flutter_stem_example/analysis_options.yaml index 0d290213..bf8d4218 100644 --- a/packages/stem/example/flutter_stem_example/analysis_options.yaml +++ b/packages/stem/example/flutter_stem_example/analysis_options.yaml @@ -7,6 +7,15 @@ # The following line activates a set of recommended lints for Flutter apps, # packages, and plugins designed to encourage good coding practices. +analyzer: + exclude: + - build/** + - android/** + - ios/** + - web/** + - windows/** + - macos/** + - linux/** include: package:flutter_lints/flutter.yaml linter: diff --git a/packages/stem/example/flutter_stem_example/lib/main.dart b/packages/stem/example/flutter_stem_example/lib/main.dart index 15d3266a..2ce48bcc 100644 --- a/packages/stem/example/flutter_stem_example/lib/main.dart +++ b/packages/stem/example/flutter_stem_example/lib/main.dart @@ -1,5 +1,5 @@ import 'package:flutter/material.dart'; -import 'package:stem/stem.dart'; +import 'package:stem/observability.dart'; import 'src/app.dart'; @@ -8,7 +8,7 @@ export 'src/app.dart'; Future main() async { WidgetsFlutterBinding.ensureInitialized(); configureStemLogging( - level: Level.debug, + level: StemLogLevel.debug, format: StemLogFormat.plain, enableConsole: true, ); diff --git a/packages/stem/example/flutter_stem_example/lib/src/queue_monitor_page.dart b/packages/stem/example/flutter_stem_example/lib/src/queue_monitor_page.dart index 5076942b..5119b65e 100644 --- a/packages/stem/example/flutter_stem_example/lib/src/queue_monitor_page.dart +++ b/packages/stem/example/flutter_stem_example/lib/src/queue_monitor_page.dart @@ -2,6 +2,7 @@ import 'dart:async'; import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; +import 'package:stem/observability.dart' show stemLogger; import 'package:stem/stem.dart'; import 'package:stem_flutter/stem_flutter.dart'; import 'package:stem_flutter_sqlite/stem_flutter_sqlite.dart'; @@ -52,14 +53,12 @@ class _QueueMonitorPageState extends State { ); stemLogger.info( 'Opening producer/runtime stores', - stemLogContext( - component: 'flutter_example', - subsystem: 'bootstrap', - fields: { - 'brokerPath': layout.brokerFile.path, - 'backendPath': layout.backendFile.path, - }, - ), + fields: { + 'component': 'flutter_example', + 'subsystem': 'bootstrap', + 'brokerPath': layout.brokerFile.path, + 'backendPath': layout.backendFile.path, + }, ); final runtime = await StemFlutterSqliteRuntime.open( @@ -130,7 +129,10 @@ class _QueueMonitorPageState extends State { _isBooting = false; }); } catch (error, stackTrace) { - stemLogger.error('Flutter example bootstrap failed: $error', stackTrace); + stemLogger.error( + 'Flutter example bootstrap failed: $error', + stackTrace: stackTrace, + ); if (!mounted) return; setState(() { _bootError = '$error\n$stackTrace'; @@ -173,11 +175,12 @@ class _QueueMonitorPageState extends State { ); stemLogger.info( 'Queued demo task', - stemLogContext( - component: 'flutter_example', - subsystem: 'producer', - fields: {'taskId': taskId, 'label': label}, - ), + fields: { + 'component': 'flutter_example', + 'subsystem': 'producer', + 'taskId': taskId, + 'label': label, + }, ); await _monitor?.refresh(); diff --git a/packages/stem/example/flutter_stem_example/lib/src/worker/worker_isolate.dart b/packages/stem/example/flutter_stem_example/lib/src/worker/worker_isolate.dart index e82ba170..05a91d89 100644 --- a/packages/stem/example/flutter_stem_example/lib/src/worker/worker_isolate.dart +++ b/packages/stem/example/flutter_stem_example/lib/src/worker/worker_isolate.dart @@ -2,6 +2,7 @@ import 'dart:async'; import 'dart:isolate'; import 'package:stem/stem.dart'; +import 'package:stem/observability.dart'; import 'package:stem_flutter/stem_flutter.dart'; import 'package:stem_flutter_sqlite/stem_flutter_sqlite.dart'; @@ -20,7 +21,7 @@ Future workerIsolateMain(Map config) async { try { configureStemLogging( - level: Level.debug, + level: StemLogLevel.debug, format: StemLogFormat.plain, enableConsole: true, ); @@ -38,14 +39,12 @@ Future workerIsolateMain(Map config) async { stores = await StemFlutterSqliteWorkerStores.open(bootstrap); stemLogger.info( 'Worker stores ready', - stemLogContext( - component: 'flutter_example', - subsystem: 'worker', - fields: { - 'brokerPath': bootstrap.brokerPath, - 'backendPath': bootstrap.backendPath, - }, - ), + fields: { + 'component': 'flutter_example', + 'subsystem': 'worker', + 'brokerPath': bootstrap.brokerPath, + 'backendPath': bootstrap.backendPath, + }, ); worker = Worker( @@ -63,14 +62,12 @@ Future workerIsolateMain(Map config) async { eventsSub = worker.events.listen((event) { stemLogger.info( 'Worker event ${event.type.name}', - stemLogContext( - component: 'flutter_example', - subsystem: 'worker_event', - fields: { - 'envelopeId': event.envelopeId, - 'error': event.error?.toString(), - }, - ), + fields: { + 'component': 'flutter_example', + 'subsystem': 'worker_event', + 'envelopeId': event.envelopeId, + 'error': event.error?.toString(), + }, ); if (event.type == WorkerEventType.error || event.type == WorkerEventType.timeout) { @@ -99,7 +96,10 @@ Future workerIsolateMain(Map config) async { } } } catch (error, stackTrace) { - stemLogger.error('Worker isolate bootstrap failed: $error', stackTrace); + stemLogger.error( + 'Worker isolate bootstrap failed: $error', + stackTrace: stackTrace, + ); bootstrap.sendPort.send( StemFlutterWorkerSignal.fatal('$error\n$stackTrace').toMessage(), ); diff --git a/packages/stem/example/flutter_stem_example/pubspec.yaml b/packages/stem/example/flutter_stem_example/pubspec.yaml index 37fd459c..f5bb419b 100644 --- a/packages/stem/example/flutter_stem_example/pubspec.yaml +++ b/packages/stem/example/flutter_stem_example/pubspec.yaml @@ -28,6 +28,7 @@ environment: # the latest version available on pub.dev. To see which dependencies have newer # versions available, run `flutter pub outdated`. dependencies: + contextual: ^2.2.0 flutter: sdk: flutter stem: diff --git a/packages/stem/example/microservice/.certs/ca.crt b/packages/stem/example/microservice/.certs/ca.crt deleted file mode 100644 index e6c4486d..00000000 --- a/packages/stem/example/microservice/.certs/ca.crt +++ /dev/null @@ -1,30 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIFETCCAvmgAwIBAgIUR2xggUz1safbM8xG/oKgAZgnGrYwDQYJKoZIhvcNAQEL -BQAwGDEWMBQGA1UEAwwNc3RlbS5sb2NhbC1jYTAeFw0yNTEwMjAyMjM2NDVaFw0y -NjEwMjAyMjM2NDVaMBgxFjAUBgNVBAMMDXN0ZW0ubG9jYWwtY2EwggIiMA0GCSqG -SIb3DQEBAQUAA4ICDwAwggIKAoICAQDI22+XKRHggfTzUn5F7tiHVXaCRRiNY/l/ -VwXk6BwmEV+/9lQob+DimHDZBG3f9F/+fTxkBRtl9Sr2kfaM62HsbLufiStKjOEI -ZSeAjmUDiiSzDeLcSgjCCFfgiTV/I39UYBguv1TtMmQScTFopxA+QeQb3a3OmBbp -XSP8Tl0DOnEx7MU50k3ondcjzlDJ3KfryI+tBkdNGgrLyZDe85cXnntRzI15tbar -b0ofXB5CRF1DYbm/KOLVAnknxUa+8Sb88NqOpDzqcD2Fuc8OR45gmCD9PyTowkUj -H5Szs97NjKDl8kmTB4Zyq94wbDD3rKqha+fltdn1cUbKvzv7pDvUQ+hvdoTWK4s6 -24cTBrq5oCe2W+kgbm9RutXGUpNtl5RumSpco+iEC9zvoBywlDKpMpb4H/FacKGl -gpk27qYlH9lO0lCfjU/GhqDDAQVDH90s1E2ruEslHs+J+m9yMePGC6GhACRGHkuE -uZ2eKMTM2l6CWVn2yvwdV+kxx87WCziBRjH161z514lvPsRAdk7gOFLa6ab0lAzu -qAhi3v+VBRljHefE0QfOa1NVquP36jAQ2LmgUDkVxJpbq2gBjR7iN4H4HtP9GX8c -HSujKT7290AncHeSS3m7naZsCYczjTGANqML+TGgm2ziH43qYeklonU17oS6hWgI -uGFxJpoaawIDAQABo1MwUTAdBgNVHQ4EFgQUGGgfZtHejXDt+iNQ/DvygE/tR88w -HwYDVR0jBBgwFoAUGGgfZtHejXDt+iNQ/DvygE/tR88wDwYDVR0TAQH/BAUwAwEB -/zANBgkqhkiG9w0BAQsFAAOCAgEAwe6nDCDR7tvYYzo9z9eN3HVXROe7B0DiCQlv -YnL5gyuSp9Zb3D0mhYoz63AtQ7CudqUKWBeQgRKkcNgB/Msmb6YK3++QKXpwoKwG -nYGNnHt10iLJtt2EGCRsa5NBnNBPjgZTScRY93imM361HeSRItGZJ29FQI2QiLCk -mFj+wJ8ng9rjJAaUyTWFWMltU0n682eFG2ELmRb3oy9PS/oJTRNbSJ0koZG6be6G -N/Oo+Oxxlh+6ERRpNDB0oyBueM9okOhTpqE0hI4UlbXK/4kJmwQe103lCZEUcMoW -GVIC373Onj+2u9rdsesbVWI+MOWAtLMie6xK30x5AauyHEyjVMN2ytBiCQ7InTM4 -2kO51TBhiYVyWU9tBcDHVjFwfRK0uO6FnGNPIOB3FzvMUF0hAwVoN91uh00lDNqL -u1ounSLtl2+cqoAeqw+rtf2aVkdJktkoPcVPqMjZqDwO7KJkivLa8TzKZ6LXDSC7 -LFsjuTHmHTqV3m20p84tpLEXeKeL4c5zDj05+Hnxs5u4RAEptQ0/qWyGTwPBIsiL -3PTBSUcdSaj38c1i8Bfe6X7RjnkG+nP1zkRTJxVadEBo714wdW4ZSSvWRwzSzh5g -r229FTq9I02IXjIdqHnYq0ImPLMxMq5snlITt5IaKwzmWykFV4DdyxpmRmK/HGRy -EsrQJ68= ------END CERTIFICATE----- diff --git a/packages/stem/example/microservice/.certs/ca.key b/packages/stem/example/microservice/.certs/ca.key deleted file mode 100644 index 153ec792..00000000 --- a/packages/stem/example/microservice/.certs/ca.key +++ /dev/null @@ -1,52 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDI22+XKRHggfTz -Un5F7tiHVXaCRRiNY/l/VwXk6BwmEV+/9lQob+DimHDZBG3f9F/+fTxkBRtl9Sr2 -kfaM62HsbLufiStKjOEIZSeAjmUDiiSzDeLcSgjCCFfgiTV/I39UYBguv1TtMmQS -cTFopxA+QeQb3a3OmBbpXSP8Tl0DOnEx7MU50k3ondcjzlDJ3KfryI+tBkdNGgrL -yZDe85cXnntRzI15tbarb0ofXB5CRF1DYbm/KOLVAnknxUa+8Sb88NqOpDzqcD2F -uc8OR45gmCD9PyTowkUjH5Szs97NjKDl8kmTB4Zyq94wbDD3rKqha+fltdn1cUbK -vzv7pDvUQ+hvdoTWK4s624cTBrq5oCe2W+kgbm9RutXGUpNtl5RumSpco+iEC9zv -oBywlDKpMpb4H/FacKGlgpk27qYlH9lO0lCfjU/GhqDDAQVDH90s1E2ruEslHs+J -+m9yMePGC6GhACRGHkuEuZ2eKMTM2l6CWVn2yvwdV+kxx87WCziBRjH161z514lv -PsRAdk7gOFLa6ab0lAzuqAhi3v+VBRljHefE0QfOa1NVquP36jAQ2LmgUDkVxJpb -q2gBjR7iN4H4HtP9GX8cHSujKT7290AncHeSS3m7naZsCYczjTGANqML+TGgm2zi -H43qYeklonU17oS6hWgIuGFxJpoaawIDAQABAoICAFPurpDiDqBcbfmHzVB34G+d -HOqO6RWcY1P9D296NtkJhZN34e8L4VOPFAj7utZNDhgQST4ajDps2skycW6tpbvg -K/8cmMSMXjTd4EXuOvPhtDhGYzhej45VYjNn4r0AHmMpzLEsLtb3qa2zM0xiE32Q -lzu1xL2B/IED9Nen/2Ir98U/BaOF6Bx+aLqDqmrQFDOJGdDZwM3NRhmmXy1raQXr -qeiiq7PiZTT6SmUUX2idnMNdgPsKoM8WO36NB2Ue7IXiN9JHe1UUFgFdhdKXwD1J -FO/+hPPmynFE7ySw0zfdvhaYynnwORSOEUj2zPeSZbMSnlL1TOEttsFebDhZhs4M -zMie/YVZCjLSfNWd8ErwUFa0zRk1iSBwgEIoxfKjQkl7p9K9wRJcgh/rVfKrzjD+ -uucKuysBkrRjgZKGWiEgpI7XCX38L767o9S9VaXZ4zVLs+DY5jp0zbLgiKmjkOGX -lc+ufFBFNz8oNvg8r5THLovUMTaMBXUCsEZ4t85rGDZq8D4JtNedQkPWUAuVBhgo -3phn/Y3wc03gUmb0ohDSZosIksp+moG5CH4mJmRAVLAP1Di4wKBXNbyDah+hq1va -xhFUScCFIn4aPnkil8r2SheHr8wfJBF/Y5U+4hW7EZSKFc49JBaLDx8K7Ag6VRmF -UIP8gPMziOprZ2MZri6BAoIBAQDvosoptIulFLDgOeJWYMv9OjNXzNXFm0Uf/dJz -v3zaVIH2BmiOg4sTncqTItRDmB3qv8nbv0e03lOMGFQTkQCg24CSHU5RQKakAdqX -eOs5EwVvGhX/0FK6tniq7clkm3wIIFkZ8p0f12+yGRl0ny/8lGKTJ/wbWeS6Yc3F -jo7m0w8itAjLj4iAUhKuMmD3wm2sWCGmoyO6w6WiIzNYxHV+6fU2sIZJYp262fSo -04jzy7T6B/zWtPTJVP50NXUFgr7baIHJdL5EOfHQiU2vvCJK6TfH+TXvYBABJu4N -ghC96t6YSc5XpbIdUsFmJLJCkFXqhbxGgmbC4NL8XoirPJSLAoIBAQDWkrvBdGYP -CgyaGxp9aAkJ5EQ+dilk2QD6yIXDC4v+4bDxRrR8HDn/Slbl/yuMqjQc3DF8R9sp -twmXBmRJZlL3YDl+Nf/8dXmONFbDFAaK0NEzH5za6DwpWsky+y4Bqa7MOjm01mF3 -+tsYsqLrTQc82D+kuGAnmpVrWzh52ENgPDn3Ts3UAIHKvO6XYEsFm/j1skjgZUIl -AsfgcfiMok9JittDM0CJNNN6GuYKYPO2uXU8MDjOg0b9y012jRne+/oFueORrJnC -tu+DY23AGOe5LHYVDAnY0UGJfjvMIMMwbNSMeOiAMMF2cKG+JlC4+cOP+4DJ27Rk -sYTG205TdO2hAoIBAQCC/JnCyDSYzMGOHzXP/2MhjOGpuYaUWDYlsYmKyX3GnfDX -+VLHq7GmzFaBrMUYGZsoySLeoiThAbHrXxT+CQGt663QsIIxucQc3AKPT4JMFVCg -FUM3LLk75Cf7TrUJ8E+Q5ZWusEYgtJMSZ9PZQnZpevaS94U/shhrpLBkr3L/Sxj3 -4C76jf+QHMynOVR1uWpXjWj/fCc4nF76TIFxoRmUgPAPr9gHXwdWrnKkodQ3Zv8K -uLkE567yKipYfSv1Ei2fIvXml5C1HHNIiPTU/WJFTku2XEuWCA1Xg98VtSXPmgWI -erd2g1Tj4GtkmORzrA+6fNWYOQlQtrG9TzMnXprXAoIBAEp4CHOeiyEjEraVO39U -kPZ+43RuG/WlHqLq3wmPQ1oFSByktlTwkuxYB4qxIZMjKwztGruQUK7MmKZykHGs -8pRqL+nuodYjizlgTtBLbe3VhRViHlUXDnIn0PZBmHj5Q/IbLxXk5VT3mfybWtGh -HSbvwj7PwodcLglhUltvRArtPAzpq6OtPV54bbHZuI/Mzv2uj35ZhjhEG8x4oZNl -bXEPoQNUkUxNKiwRk878UPkUqmxmLAD4LLY+TbccR2NNPjuohSN0rLSYOs31vwxR -WHK6zIyHbg9yqHUoe2Zf5H409xt9huuegDYz5t6Qe6mG/WlGFy93cccUmQlO6YMr -bKECggEBAJ1+EUx7ZErnen+tuRslssTdLprNrtdxIENlZa1WclVa1943jnSYAKBp -VVdT0VF6pqEE3g+HvbNejpVNGkV3mQNnwUFztdhGIDzKYaPPV1AUocnAzwkyqNsn -hYYhaP5FGv6T2wLwuhGn7P+kqDnLcDnmAZAFnQurHN36SISqbsbZmNSXcliWaL8/ -Eqe+SUkpRnGfNwiMtlmeXYDEmOMchsQbwt9kPgt+HNwOYk8iOT6Avcfn0XH1aPsi -01W4xe0SeRsi2qvdNo+pxKhbZFwXWZwo7F7gsNxn7ve3v3Ziq+lFkspciHjbaRNE -lI/Ir2Rfd04BB8Rmc6gQ+/07cI/ZFUE= ------END PRIVATE KEY----- diff --git a/packages/stem/example/microservice/.certs/client.crt b/packages/stem/example/microservice/.certs/client.crt deleted file mode 100644 index 43918adc..00000000 --- a/packages/stem/example/microservice/.certs/client.crt +++ /dev/null @@ -1,30 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIFHjCCAwagAwIBAgIUJ1PYxA55p5ybzxcakPxGkrtnrzUwDQYJKoZIhvcNAQEL -BQAwGDEWMBQGA1UEAwwNc3RlbS5sb2NhbC1jYTAeFw0yNTEwMjAyMjM2NDdaFw0y -NjEwMjAyMjM2NDdaMBYxFDASBgNVBAMMC3N0ZW0tY2xpZW50MIICIjANBgkqhkiG -9w0BAQEFAAOCAg8AMIICCgKCAgEAqy2QgNdS4TIB8BNwdNsYLYaxf15SUFL6LKKJ -oQnUech2UAdNs0RCo+lA9B/DSXnAkJj2qu6IzJM9y7ts10srHyCZkhZMS+xZ3jc9 -db4yTBX7lKYqKiDiNNOszw+wgD3o3AIIq+7J5lpWhC+sorq/qZNJkOKpLq/YmbE3 -/EBAurwBvXkaXXskdmjf9LueS5NG4MKjmGu5kXhzt15uucBtHzAv1L6kZoLUodW2 -KYGU4zdUXn5J2ny6vJklt6fES6ySbmzVKd4Tz3w5+dxmiEwPQ0b8gC4+PYjccxX8 -cmm/DDq8ITFKpXyZhVY3vfK/DR3oZyvrkAyZWrDW1IEOFAsO+grrMjfkFSUbayk6 -mg/G5SVFbvCSy6NzroDhV4J/qWXUBflrH6WepQWdolxMqrhTwQFkg4qmUdrCuqn2 -oRxiBCy0GO3bxznyQKAc0HThRpmlqCTHBj7lT5x0MxiHY5p/nbU+R1is/RArbsQ6 -54XJrtX2c7WtBf7/FJoIhc50fdDm825GeohyCh0klOqnYWWZYHJzCEe+O6szT0sv -iten9MBXkXLSXx+cYDPVN4f20ni2AQ1rwFHHJx9lCBZNaVUYlFnN05Q1m0+Y8V5n -rmaNRS63RAeVllZisZlP9YTbc3DvQ2nHkUVE+a02JgqqcTxe4a2XS5y+X+857eSA -IkIgQycCAwEAAaNiMGAwEwYDVR0lBAwwCgYIKwYBBQUHAwIwCQYDVR0TBAIwADAd -BgNVHQ4EFgQUzFJDXxWw2B4xRqyJMuiHjOxBal8wHwYDVR0jBBgwFoAUGGgfZtHe -jXDt+iNQ/DvygE/tR88wDQYJKoZIhvcNAQELBQADggIBAAwesqezbWIEUNOM+keP -O6kSpm7zITOKiJ33KSoHpNcTRvk+eKTcaVFH1r2U5WtzpEevSfVhIf8mf9TA40Ub -MGVdX+AeWuEIIeLbzasTwt/JSur2PVqLxo3gKGVeoKObEojcEnJA5IRKaIWAYBNN -SXwV5anF8uYc6wx+7mWwYLcnlGTvVSrrMZc5hI58vXbEJqIszg6Iymicd8af2RJA -l9U1wymkUg1RIOccoimaxAyKIFEnynw2TSQc/ZDClCsHSTZCczExuD2ypXF32SlP -0MMG7WU33jPZe/gK4Y5WpppCVTh3ooa/n76WiEJR0fowPLd2jnqg0QMo4alTCmMi -xQKIM22yqNwpKi5NCKDVeJ3HoJHXozPU5Eftuw+iklAG9ZesQDq4I0vSKhsKp+IS -TIu1REAQGBBGQjHbavoEln/LAywDdb2CUHt0wmVOLip0wrMhUgSC4ql9X/kTp92+ -BTcr9kQCkk2+77fLiDxLSU3ZR3VVZ4ZWM87VabPfaDHbrDpl9W/WTPBKOf2GdFG/ -glu198Tdm6nNCDDAFh9W9Pfn+8OBbBLnARTWfHTUtIPijdzljG2D5078kxiMceFS -oY/+bSQmzD3tpHzQducilckB1K4QdBAayJLwTEpy7hj9wZyHFSPdKb51R/2rkthV -jhnZmFYiYlBNBwYtN3WLzWfF ------END CERTIFICATE----- diff --git a/packages/stem/example/microservice/.certs/client.key b/packages/stem/example/microservice/.certs/client.key deleted file mode 100644 index 3415a027..00000000 --- a/packages/stem/example/microservice/.certs/client.key +++ /dev/null @@ -1,52 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQCrLZCA11LhMgHw -E3B02xgthrF/XlJQUvosoomhCdR5yHZQB02zREKj6UD0H8NJecCQmPaq7ojMkz3L -u2zXSysfIJmSFkxL7FneNz11vjJMFfuUpioqIOI006zPD7CAPejcAgir7snmWlaE -L6yiur+pk0mQ4qkur9iZsTf8QEC6vAG9eRpdeyR2aN/0u55Lk0bgwqOYa7mReHO3 -Xm65wG0fMC/UvqRmgtSh1bYpgZTjN1RefknafLq8mSW3p8RLrJJubNUp3hPPfDn5 -3GaITA9DRvyALj49iNxzFfxyab8MOrwhMUqlfJmFVje98r8NHehnK+uQDJlasNbU -gQ4UCw76CusyN+QVJRtrKTqaD8blJUVu8JLLo3OugOFXgn+pZdQF+WsfpZ6lBZ2i -XEyquFPBAWSDiqZR2sK6qfahHGIELLQY7dvHOfJAoBzQdOFGmaWoJMcGPuVPnHQz -GIdjmn+dtT5HWKz9ECtuxDrnhcmu1fZzta0F/v8UmgiFznR90ObzbkZ6iHIKHSSU -6qdhZZlgcnMIR747qzNPSy+K16f0wFeRctJfH5xgM9U3h/bSeLYBDWvAUccnH2UI -Fk1pVRiUWc3TlDWbT5jxXmeuZo1FLrdEB5WWVmKxmU/1hNtzcO9DaceRRUT5rTYm -CqpxPF7hrZdLnL5f7znt5IAiQiBDJwIDAQABAoICAAkG+p9o0O9HrrAlU6FoqoJA -H7KKsVxDZqZy8z93Cr8EvNnklgrrAc6uGZBQ79YQodXzqH0u3i+tC4MSgR9rezyF -gMPNAZIMHAMq4PFe1JKMTZEIdaHKISkjCUfULn0yMUpk9bsczoX44/t4SNToyfxx -5DJ2rvBjh/s29tpf3b9bSduJpu52H4qlEOZ8UdFE1OXmgy/MbTldjlCwc2DADqEo -mTp8IU8MqD+uoGWDoWvyz4lk2eqNTbXDfSiNzugkvjDXns2PRMvaOR59qo9BNO7Z -2q1nLZNw2RtdS0u8sD1qCDYsk3YIYk+Up+o7AoCjv/gCP3ErJkKQ9TYOEPAqqHae -MRH6JgDNSi6KrP5Sglt1nrax+Th6ThHnvhQUgtRjuSGATNIUp0b3toH0RT0rR5Nr -88I6X+mLlHlqQZniNwHKpPPTdFNWtPFAwL92g2nRHJLeCZAMXrfU3n6jBAKhBl37 -3D91MU03RbNn/hX4gXBrSMnenqsE/QgpJHGiHG+8kuO3lCK90IVYygH6K4EOUi/p -8kQ1xKUBy5gHtVybLIqSJMnSp+ngVJx35db8GGfSSNijrk40y7eLsI5OIaM6uFUp -Px3dtPNdb09cTvwySuEb/y1sw3hQH2RKLQSlhM1SArt7XZ/+SJ+e4IBNYglNqAZG -4HgXQY6YEYA17pps3wFlAoIBAQDVZdM+5HDU42buCFlyuf1RCZJXvnMnTrAMhjvO -0PqanVhy8PweUcpXOGt0VGxiqzy+H1LrQNfGyBvy390wpoh10ImwlANU6Ur9h3zi -1419F7vBqflr1OaIDp5gFvGwi8zjNmviNIpPUA4Oi+ZfUkw9AOK2ZZxX7TFU9Pb4 -c4z+wNQsTO5yvNPexmmY7nKNrabnw4h4YYjxIHDIC6uRxDdM/zJpvXmnPDvrTRbX -Dp9UpdxFNnsRoWSdXMeXKKx2NRWShYixVVlU02ebMx9NrEwTa5W99sv/rWeAzpkL -WAEsZ9BxAZAW1L5mr3GRVl/7JShWcVVfbAyX2FDWTdG7mqPjAoIBAQDNWgCnAQc2 -ewNweZ6VhNpM7mXMeaZhafNLzbIxtSiylWjkScq7dNS7TD2G9cdpvOh9jObKBqKx -piFuu7KW8Svqu4bQiJrFUwhyZn92azmMRe2s1++bz/3B5SeTqijtWPunQzODJ4VK -SeCzmQ7ClsWc+w+XKGES04i31BvGlFovd+lTC+4VcQfHSR0MKMNA6Pqm9Gww7Hkq -itOzRO8FzsW1RBpjesDWLKcnI+HJCZZIUgwgferXDBGvEmmODFqtEldd+Hkk3Jko -HVCCdFVTiXp3s7f4cmEjruBhOEEj0VTAiz1WZAHneF/BU3jRmGiIWI36vU16WD/8 -v7+wOioMOW7tAoIBAQCS6EeWS3O1y1sn0U2q4omWiU1a5Hyn+c7OwHZZ+ZocaOfk -MSyPvuTaSLgI3LnE73egtzXBSOGDMm2zNRQ0EuFsogS4MB1fDkAW63g5PgVksbsL -JvXTmzydO9sh8YmvbX6+iM+aaWsyEL2sm/kbdC+R65Mn2Cnaiirk/sMq5XmSQ7ON -T+TzzjTZb6RwqO7pxDtb4EuJ/DNnNFsQDKCvd/JOKn1NOCiQzbG+WlK3+JQNlSVf -7JKq333Bq4an9FNOCVtxhEZHNqeJ8wOnX3lUYqH42J68qTNlL75KeW/41DvykwF3 -WokUfp8ZV/yJ/nmTDtCHOfx49Yxx2HRGJEqXqWzbAoIBAQDGpWr+IqN8k1EHa875 -qpOWWgOF4qQNlfVx/eCD71AYOSpHtA52ehUTvjPCq10lL0XjtfrHsdWObjvnJ5Ei -LNij8JYAgjTd7227ZOWmgyEfTcIWMqkqG/DIsUkwsil3OF3wNrTNCd0JuR+x/Lzn -hfxR2BgDzF7jXFfkJD5eQcJgKuo9mMvOWBIDOPntej0vPHDEMqu6Q/cOLU+PGn7y -KkzdBa1VUDjctkNYPIC8/hYvjZcHk2cqchVlOzV64P6Sb6yuTZ+7Bd7ujDUGjUzB -S7t5shOkNdk8MH4D7ewtnuEShV9ng8THyo0usfWtI3bzdl6aEKezU01KzZ679fgY -7TIdAoIBAD2v2Yf8XjVkwnEDjp4lPwdCYj9IoGVxx7x6m9jTiJRFQiC7WOX2Ogrz -oHqWkgab9cy3KiIIcqP5p9zlEXOg2uD9clNiLMF0SnFBxRxCFo+bwGSlmo/QhPAa -mM+lfBvBoeVRhvQhojOe70j7vBPI07JmNC4hrmWjucxIz5V6ihieu05ZjC5Dz2zq -oNNo5hJtBcTLlTZpbQynf6QodqcVe2nknBuVVmw38G/0vPBBI98/K2MlSEO/6mx/ -CMvIhEfClLYy+QerXenH3lDvk3aSMue3ChbMjl445oeekUJaggEJpYaUsoOUv3Kd -tT1c6mVMGpszu6BDxo5mypc/O/qQPKE= ------END PRIVATE KEY----- diff --git a/packages/stem/example/microservice/.certs/server.crt b/packages/stem/example/microservice/.certs/server.crt deleted file mode 100644 index bb4ae13b..00000000 --- a/packages/stem/example/microservice/.certs/server.crt +++ /dev/null @@ -1,30 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIFNDCCAxygAwIBAgIUJ1PYxA55p5ybzxcakPxGkrtnrzQwDQYJKoZIhvcNAQEL -BQAwGDEWMBQGA1UEAwwNc3RlbS5sb2NhbC1jYTAeFw0yNTEwMjAyMjM2NDZaFw0y -NjEwMjAyMjM2NDZaMBUxEzARBgNVBAMMCnN0ZW0ubG9jYWwwggIiMA0GCSqGSIb3 -DQEBAQUAA4ICDwAwggIKAoICAQCW8DNqTmPZeGFkxuLt71P4Z3eSndpn+fsV6aqs -xoettAoioKtC0anMND8RJsLmiPVAxHYqOvep1WG5x3ncn8nqhguLHUl3koBIY8aI -zrv+NQ2CqLr/DzlygLz9XeiPMJtyHmCecveDVHlry3f2Z2Nh0C64Hi2QZMHJuNgO -p1vB/OfC+A5hIgux7a8/nF/CfNtWsybl0MIID88jJvTJvbFnQB7v7krDS9vKUSfg -N1f28KGOSnolNsPtFOkpT9O+gGoJxCYIanhkZnIhu0YHIHdKl0bW5aM3DERTKEOG -lQbpipw7L3HywYjUTkeyGVeOy0W7xrEfc5GBwkmIXlIkmsmLHO4togludB1LpZBH -64dupAj2yEbSW3PbwxEir08P+6cwo1PVs47RopHtoGDf2nxps87rk2Sh4PJrR532 -X1y/Sy3SxU5OaWX8KOLdKX7Gnk5yj+LQjGSn/I+13tRd3MdkoUZzoBvuMjTFecvy -FYELmysg+7Jc0o+ryRF3gA34t51tanZzypLAIBgEtaDrKcyJTIHvCy3tgx/wJyE0 -uM533cxuRzwZoN0NL1vnIYMQ22QEV/ivGYTu17gkXCs8xsyyBC2O7IiCxF66WKMV -XzilkJQ0bvY65wvClVNG/6EID/Uxac7ZgRBrdq646H9umRq8N38j+d1XL3LR5ZWo -SO24twIDAQABo3kwdzAVBgNVHREEDjAMggpzdGVtLmxvY2FsMBMGA1UdJQQMMAoG -CCsGAQUFBwMBMAkGA1UdEwQCMAAwHQYDVR0OBBYEFGlW3LCIXGph2h6S7/KxRMR+ -bBxpMB8GA1UdIwQYMBaAFBhoH2bR3o1w7fojUPw78oBP7UfPMA0GCSqGSIb3DQEB -CwUAA4ICAQB/dvM5drew75zlcDNQVPc516iNVDXWH+p69lU+PHIB2A6WlNSn7aQa -AAorqPxVZRHL9h6LkqCjvprahJlu9cKRVlI95LMUmPnOHSP9VGyAIsjv19hVW9+R -T6g8Uh4eo7UotpgBYItL1IQPvxwq7+RekoLaHo2j/nkOmia34qo0fJ8kz8rJ7nRV -nRl2pCxhiP9iyezz5+73j3DBEKknRCK3SEGcxfokrDrUvG+luaLujtj+uj3rO4mS -1SvXIao/ani9ac8w144n7mAcVXkdxKAQp2MZ6jh+wktG1YdOjND7FBwa5NxGBYww -tpJP7si8jOVLgGMzhHZ0BJiy7OQsHe0iEpo/vf1hJnPhX71mJDROmw63fwQ0tO3H -didQaGl8feDmhyC17JRyQAJViL+SafC/hff7pCJZnk513YJcqRq7CPTySUIixdbI -CT13eJbyC/WKG63i6QiaArsqilP7ZUUOueOiGJL6C5DCl6NVmpesDYrLJ848WlTY -EUtJozPTyI8bUPBGwGRthCadByNPV11xIGuXf4hUK5Yon6E+nXO9xHT+bmnbpqNs -5AmnbeEexeWSQBemfWAweNfPOswO72lYqoLM/BraQqxhgm4iUUlXGRPPTn93CdPg -HcF7Z538OdWUrKm8c6WQcaz64W5u38FtGRdyxE3+WwlSFqCG659Fow== ------END CERTIFICATE----- diff --git a/packages/stem/example/microservice/.certs/server.key b/packages/stem/example/microservice/.certs/server.key deleted file mode 100644 index ce32c603..00000000 --- a/packages/stem/example/microservice/.certs/server.key +++ /dev/null @@ -1,52 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQCW8DNqTmPZeGFk -xuLt71P4Z3eSndpn+fsV6aqsxoettAoioKtC0anMND8RJsLmiPVAxHYqOvep1WG5 -x3ncn8nqhguLHUl3koBIY8aIzrv+NQ2CqLr/DzlygLz9XeiPMJtyHmCecveDVHlr -y3f2Z2Nh0C64Hi2QZMHJuNgOp1vB/OfC+A5hIgux7a8/nF/CfNtWsybl0MIID88j -JvTJvbFnQB7v7krDS9vKUSfgN1f28KGOSnolNsPtFOkpT9O+gGoJxCYIanhkZnIh -u0YHIHdKl0bW5aM3DERTKEOGlQbpipw7L3HywYjUTkeyGVeOy0W7xrEfc5GBwkmI -XlIkmsmLHO4togludB1LpZBH64dupAj2yEbSW3PbwxEir08P+6cwo1PVs47RopHt -oGDf2nxps87rk2Sh4PJrR532X1y/Sy3SxU5OaWX8KOLdKX7Gnk5yj+LQjGSn/I+1 -3tRd3MdkoUZzoBvuMjTFecvyFYELmysg+7Jc0o+ryRF3gA34t51tanZzypLAIBgE -taDrKcyJTIHvCy3tgx/wJyE0uM533cxuRzwZoN0NL1vnIYMQ22QEV/ivGYTu17gk -XCs8xsyyBC2O7IiCxF66WKMVXzilkJQ0bvY65wvClVNG/6EID/Uxac7ZgRBrdq64 -6H9umRq8N38j+d1XL3LR5ZWoSO24twIDAQABAoICAEPP/wmP1oRp9TfVsza7A7ri -oXIZwXpdcVVS+f9N4r9D2BnenZMTp3m7L2qBQ6dXePh0S7KFc6e12oYg/LsGfy0C -tET4D33NMB1V1mi4mPWI2F0PeZQGPwUWmdIrUsSiTfRjeX02ocU7OqIXc/ocUSYs -5hCKXmY1HmuGxGsw/aSV/3o1ACDbru1toAKwOsWiWtkE/EDcHZCJqcJdSjUpKH7+ -6+dX4/cCKekntQajVr33Nl47T81QYY4uiuKonrVD9GAW5A975kMVU8mb3cZt66pz -dLwvx7h/pmLGADCC2+wo/jAJDOTH1rWrxcJaaGAbK2VFy2YaAhGOPIMr/TO85zcM -Ky9SEeR+z9KftKPqVSgT1XWdvHaqXmuuezyc/kIOcH+JgIj2UjZt8FplzVi/55sf -k6J+mKMR0pn9BNl36OAwnb5iCnRuQV9qyjkDgyPZXeX5VltYZqIQfd8KecMy0cnI -jsotKCuO52bs318ckKv58HI+LNSObaxHB7iub147V4E0J9KF2CIvZHBQzm7yeZtU -jaDJGk9pJgvBH4Fi2IEig/6f90dc3u7sszWALY3u3qzGT9Qch7mggJ8nvBFhdSE8 -X4mEBcAXJxnz63sQBz/NTq4U4UiUggidFjvjvuy1ksNQEC2UYLjeJ6JBDOkRde7w -T7RlH7gh2kSAqWSlQ4SBAoIBAQDM+/90m3+X2989Td/RVz/P3fIDZ/t65R77YUE/ -G/Ou/9H/oP5uHV8aumPqF3u52Mz3x3mzYvIOGbdMCzJbjD1jrP8jWT1P6De9YzWY -uiwwUoyXbr4GvJEdgrWjhYfzhv9P96yMj8XQUMH64eT0dWWguiY5heQj2faKdT0+ -00NqyOY2B6UooN7h8ZGuQtFPz0Oya9e78Cxm2zU83gmIx/AeXHSicc+5faSP8vcP -+1CXupWqAukRL9QMUX8d04RcqiTxSj3IWVxMGPjKG1Ui90hH40Cvl/CwGfm2VYYt -t6w2GdqGH/0nGvcgxEG8fwvqQVALbd1drHVZpElDlLjcQQkxAoIBAQC8gM8VdPh8 -22hOvSQe1tFNbNcupY6Dudgbi8EXxPFF62DHPbAEWJwE8tXtKxy3tg2PoObQS2bE -yB2bP6jhrU+09ENgOEVdU8acCMkTiA980aYf/jREdzR5kcl0F985/0lND1Gqq/kp -fXAR5oELWJKDxogSsUyXLjl/rgBmgsMeP1xuVD+ToMcPJHgzKe+xXSxwT0n98Drf -XBfQZOGlju3mIqdt2/8YY7kRL0HEXcphh9/xSOgKzZdd73bzeP48PblVPMi2J0E0 -xpBR3Xy8W2E1gcxzzS8SgmGfiBdUg0ggq02u9tYw3bmfd3x8qJ9ZjD84N32WNUm8 -4ouZJ1GUUeZnAoIBAHMIcAFWSjJEd3EFxLbMCclzOh9jOSNsdQiJOX5bB1OLyKN2 -C5Jr9TUgqHV0HEDdQ5lBXjCnNENpdw27stThTIolSUDjEZ9IKA+mHyvIPogXq0JI -KWfKZn1zHs/XXcXyWlDcIv72qAIRb+qE3mG5iAC8tjrNwovEA7ELvgUFte21NEvZ -2nuEW8/cKu7qHGB3Zfw4vOaBD8ng9m1kBWG3Ii9Hn0SSLgpgUfteU/uqAz5CvT6d -R4IXnx4Slg185+jAqt6l1Soby8oAsY1p7UskDnIZO/HOiImq6hHCyPva+U+bDqD6 -TqylW04OFP3dJiid4nenffTVlT1rDppYWa1w6eECggEBAK7rID6a8lJ6Dsqx/KiZ -UlN0YiEpA5v8girdfmqeF/13s7Bu77y4tN5qsFSKIvXee6sB9eznaw/DWqXLZt0z -w7j4yOSqksNNkAkOpff309dRe8RIURwPOaRtwfB2jm/1EoHtayzitCzp92rE6I4s -vJiHS5F88MOA4141iq2VPZX92bmPeAd3s/+7KcbgbUm6WhhFmzWHDDpj7L02P6Uv -glaBZbgt4wMw1nS5019yYPAr84Sk0J3ltUY3hbvcVJ518FrGMsHEn19nulgdHThz -2zqZnaz/g2S7k/dDg9Kql+2oyEdJSezCcxMk/2F135hq6ecyptkRzcD2v+94at0l -OpUCggEBAMQDOr52ydKWWhVDTe15JFMKgOJP4dUKTeyVG0tQERUfbt93gorHEBFK -p9Qvx5+bprPruInbT76o7t8E5gCkX2Kh/UWHy18WLaYbc4Pvd+GjOarJumHi8sr9 -Lh1wboxxke7DnBx176er88BSdCeQoPIq+Fh3auESX0nUpKpC7XIS2d9U76ZUXR3w -nXBH1jA9sNuPApdl6OfVyVYNcUAEOFfzbJUxtZKGKiz15EvUc5jKGkVDoRVtZRYP -AQqGCMuStY1kpXGWyVJDkiBmN7IAkgs+yWGDdkLdCbsGpaLhskJ/m7vfl2AFuTFR -YL4YblWSjkesDxQB7IxJ1ZEUBuS8IzA= ------END PRIVATE KEY----- diff --git a/packages/stem/example/microservice/certs/ca.crt b/packages/stem/example/microservice/certs/ca.crt deleted file mode 100644 index d7c2fdb4..00000000 --- a/packages/stem/example/microservice/certs/ca.crt +++ /dev/null @@ -1,29 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIFBzCCAu+gAwIBAgIUPtcBAqCQDbK5U0DvZtKRS6SaNlMwDQYJKoZIhvcNAQEL -BQAwEzERMA8GA1UEAwwIcmVkaXMtY2EwHhcNMjUxMDIwMjI0OTU0WhcNMjYxMDIw -MjI0OTU0WjATMREwDwYDVQQDDAhyZWRpcy1jYTCCAiIwDQYJKoZIhvcNAQEBBQAD -ggIPADCCAgoCggIBAIgEyCTgP05wrly4El6c4ZeCM5K31OWOl3KYTYMHYaASIgcv -oQwScXji7jRIOuPhDRQAUhgUCRGzhebG+z+5hZtekNxH9ym5t0tn/3WXKCyKPZoO -3nmPd+NCb5dBdnMNKCfo0drcgOCY57+6vep7goRRTyE+9IlSiSomU5Xyzxj6vlYG -5g6nFjFXOW60oTrRsyoFMgXJ/TOg5LlS1NaSMTzOS6O0I1SZkZG09DZjAh3JBf71 -H1cbZGI5IOp3SVVjHVsTt1k3EZkZTaRJAsuc1O9aBriYUwfD7gv8GQG4PgFhTF4H -IDFSeFRuAz8iZVF97helLNM3zWVSf0lWAKDdobZp15UoUy/LlFfb5mfcdRWGBHVN -Di4Mutzxt/hrftjtNNJJi/6znJckF3M2uDCLP9Zmtt8r7L4oLLVyTNig+lVhNvwY -VMBVFrruNl0d5qzSZqy5KNv8yePNTeLGMjdjq1DpxbM3sSx9fUWi7rBMY+8eDNYU -ZV8XkTtB/QJF0zFeS+VeWnsUJIEA1sfHiCIfOPR5weiW30AIUOQ/1C7osD0EZLNa -jZjTCuBPjAIsA2lrRAqaMO64EiUAMfqk45HVbTBHxiEU+P0USwmnpyRfucefMCIh -SGGo9/GLg2YA38LqqyLW1bnadwXOMcRhKLACKr+apm+l8QjF+HhMjhjpLUbVAgMB -AAGjUzBRMB0GA1UdDgQWBBSd29W3JnBNKTJOOnRW/GQc9Por9TAfBgNVHSMEGDAW -gBSd29W3JnBNKTJOOnRW/GQc9Por9TAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3 -DQEBCwUAA4ICAQBG1QEjIxzCmclR/60nWEPxbsG2+8Xm7XoCYMlyYmSDMty5VlBB -XIMdVVq6oE0JI2s8baGd76ykYscJceNPQgvbkLZtr25N9ezMb5UxS2copxhp/Kr6 -3ws1KuDWnBXc7Xg0N4IFYwijMSD+wPJubOVjATyhsE/SBsT39aSOjcAmnxfW5HH+ -VuWr7bh5ZmC+lT/wRLjDme3Z6rvuGkal1l+gBvU3wEw/zHMdGfb+C8NfSXJHWLx2 -sbvhYTcJ/sZqQc/2oUbbQqIk32IRIUrASsJXau7qKxr5scv2SFqKeRxuabiTLtRV -6MeiAJJ0IoLuojR/fdbL0LG/fkYlWelHeym1H4FHW/+n3pr1yu4L2GiPyR/aC8u7 -/VUZ0BxYDzcHHPX47Thp9bhEct/7YvOdOXilOOHX0Af+GO5G9rOQ9eBQz8RK7Vt6 -IZFUb5TAzELXFCXBDS+bV5uyMEkJWpIZWmewMmauIBSOqE9DlUNHqGrQX9Z5zQJo -dbgYY4FHlPsv/uqjJz0WUTnlyNWjeqKS7jYvIKksXHJ03mmW29d6yvxfPTBrdoYV -L4W7o1lTb77L4zEf77yPaOjAuj11koEBzybAyeoJBzwOLBZ9d5toPDo23/lYrmp3 -7TlWamJ0o4LWKj2t1CZ66iX+o9fsjkRRz7fEDvOmbCq7eW1Vkuv6l1KOhQ== ------END CERTIFICATE----- diff --git a/packages/stem/example/microservice/certs/ca.key b/packages/stem/example/microservice/certs/ca.key deleted file mode 100644 index 31c2828d..00000000 --- a/packages/stem/example/microservice/certs/ca.key +++ /dev/null @@ -1,52 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQCIBMgk4D9OcK5c -uBJenOGXgjOSt9TljpdymE2DB2GgEiIHL6EMEnF44u40SDrj4Q0UAFIYFAkRs4Xm -xvs/uYWbXpDcR/cpubdLZ/91lygsij2aDt55j3fjQm+XQXZzDSgn6NHa3IDgmOe/ -ur3qe4KEUU8hPvSJUokqJlOV8s8Y+r5WBuYOpxYxVzlutKE60bMqBTIFyf0zoOS5 -UtTWkjE8zkujtCNUmZGRtPQ2YwIdyQX+9R9XG2RiOSDqd0lVYx1bE7dZNxGZGU2k -SQLLnNTvWga4mFMHw+4L/BkBuD4BYUxeByAxUnhUbgM/ImVRfe4XpSzTN81lUn9J -VgCg3aG2adeVKFMvy5RX2+Zn3HUVhgR1TQ4uDLrc8bf4a37Y7TTSSYv+s5yXJBdz -Nrgwiz/WZrbfK+y+KCy1ckzYoPpVYTb8GFTAVRa67jZdHeas0masuSjb/MnjzU3i -xjI3Y6tQ6cWzN7EsfX1Fou6wTGPvHgzWFGVfF5E7Qf0CRdMxXkvlXlp7FCSBANbH -x4giHzj0ecHolt9ACFDkP9Qu6LA9BGSzWo2Y0wrgT4wCLANpa0QKmjDuuBIlADH6 -pOOR1W0wR8YhFPj9FEsJp6ckX7nHnzAiIUhhqPfxi4NmAN/C6qsi1tW52ncFzjHE -YSiwAiq/mqZvpfEIxfh4TI4Y6S1G1QIDAQABAoICAAlSLeasW7I+pw2RYKg3/ylL -Nx5hYiHGuIAAyg5bXw2MMzruVjk1+gLd0ZM8IEBVqmLZ6xudeIn3FTXoIgzEeXsu -SxDeuU7pnD264fxMhZMtza8PwJ+wck0Hfw8CDFcHNQlXxf/F92qiF9Q2p2/uPawJ -yA7Vb1u0uqkXM3MAZBWduJWRHKAEQ7v8XUzF7JGTur/RAilNkFWC0QHBLcw0Jv3N -fkyjhNNLQHjtdK8rGu3wb2yAyZXGJFuAd44RwButnTP3VoEdig4a4uGfTjrRTnHw -56qBPRljDB5T94u7tOYSepxWnSu8GiULspsXj+gP0D/cZhq7BXnhd3m5VCAx/Umh -r0SQ6Oc/LI0whWz/4cG+iAlGtToNKxvFOyxVyiOVQkXcjYstQZ3VCLhgAWWMRdEN -dtq9ESsWU38LpAuSxug957yzmiQMSuzI8dwfgchNTmVlJy+OP++3WE8vNP0vteYW -zF7BkpFUX0Ubl3rWKcUbncx88fjRp0XVSMcq0C1ztyEtwEQVf24FiCnaaPFrWpZh -OrlLkGO8ROXVIW69xdg649cmgQeIawaI9k4cX6/esCrRXikofzFG1004Td2ulna8 -sY8VThHGpUwEzYqPAd+/EOms6c+YmPeLTe3s+HOUYn7F8/gdCEiS+jB3kA6HW182 -SFEhJ3xVs1mW/wIJVrXJAoIBAQC84Q6Nt3RuCnWHKA5i/GIr9Ugs322sgbjdSty6 -7S+BoxahzzE7d0pcBkWgs5+p8mBPw2qyCgZ8nsKlfB7PFFGAB495hYWj6iCJ57xw -mdYwz10iww/eU4WpGC8XaVyPeFMR5aoVyTSBaWDVHo6tzkC6xopCSCi0+9QgnCax -7bMVaiC+knbjxx6rULanDHmT+iSjjIidym9a7mi/g7grgPLuhneGlqJX2y513Jm9 -KMEP2NbFVDxbLc7s2FeNU2uYN360zRPTFhuDuMDI1Ks5UFbf+ejA2/cG43RwJLT1 -LkCWUafYBRwdAQe7oapUPXuX/AEvWrvjox9ZAR/b0g3FR+VNAoIBAQC4Wtbozp25 -iqR0xaKTeTTX8jOpibKGkRuWDmyhNCcddNrP2TR2y1APXmDD+qUC1j4OGh1KobPs -LijPnN2gkIQdrzTyvoJ5toYsUEbeX+bz0uxffGt/80GGVQ0c6p/926DxWj/PWo/t -L1/iMOCFHPDL8d2hvaceOnnnjGQRc6yMPANENzyBhm3Yi86QRUAXE0I2Pb5GjiZ1 -88sWowTv9+0bV0xt3AIi5Ym7MsTum1VuaZnwVCW4iSCQLgDh7dBNTfBctsfnRLS2 -BbCTJhfkhE/Tt/mcazDB+nzfeNLEmaCRnNyjQZIu0J/DKc91bO+K38JURlqRKE40 -Kua9bzepowOpAoIBAA8tPlAg15gn+v2bOFy3zuOdV+lTIqrj2YjehRH4EQMnqZ/i -2B/hG/Z9AodBJ6kuMpL2LtKTf6Xv09AcQxV8mnfhI6QuGGjtXSP3nOqnUdCmOOOc -LOA+cdT+fnoxGyDkGDit7ZbGjhN/9inpxKkuk4DuhxmAVtUEDy13npxZbzRfTLZD -ao4u58X4PFT76aD8Toldt3SGWaBC2+PXn6rPN7X21A/adrz8faU2zpumdKzjTY6Z -kCf7bE8XqyFN/K1EeEfq2pbxhdB1vfflLsTyMajZmdw1UIobKLYCHRA+YS4y7Xus -HXAvLwHioZ8XOaEivK2jasmFx/UM5BRzsmuRtVkCggEANf7NMTTK93wG/PihzGXU -t2cGAKMG+ejoVoP1ByYaSsNeD0CD4ENeOJIg72WrDPCpDkJRz/kalJNUerJsK0oI -mBUoAe0QlAuEstf0Wc9m+oUNnE35wmkCCRFCHPUPVTANrIMNa3MP8p1MjRhZukMm -t+UoT+YXW4AvEkWJTZ8+q3gsUL+I5jr0U1dncdQGmHuYigB9iXa7BxMXqZKnsI1C -Eh74kLi+m2/1bjHJgkVWuVcOslhDEBCBzBjdQ07/EpVAUDdsx2zVLPbvp7mhh9nE -87LP1iODDV2LU3y05sZEREHYUayz3yhVkrjdPXxhkKBG754HirpuZVMnb9ZyTCol -uQKCAQABD/Ly6nkLZ4AnO3NNQS5OIGhEb6fzIoYdAks+44yd2LO1hrJb0WbPmtC7 -MTMHwULPvhVlsmNutSIPnSpY5SrSdoZgo8kdWn1hVMiRZbNl99u4k1dzAiLkmuLg -E/g5qTGgUWuX2L790WACP/8OjpmcRLMJ9Qx4PbG59XJpOd240drhdSGPVetov/0T -dzaPEwziB2vg1J297XhRs44p7HBGIwU0PF1lt9BDo7bfbE+rsjEE2Ys1904d5f/e -mpUwelNnIuQ7QbsFlyOCCKNIkZmjHwX5vXB2VsnNO3KbEZlrXQUvZDZ8TUErnRd2 -JgX/5khmEixDo9J8LopaMyYxPl2A ------END PRIVATE KEY----- diff --git a/packages/stem/example/microservice/certs/client.crt b/packages/stem/example/microservice/certs/client.crt deleted file mode 100644 index 524a1c85..00000000 --- a/packages/stem/example/microservice/certs/client.crt +++ /dev/null @@ -1,30 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIFGTCCAwGgAwIBAgIUKQJMnXGqR0sgtrJxHlRpDjMXDfQwDQYJKoZIhvcNAQEL -BQAwEzERMA8GA1UEAwwIcmVkaXMtY2EwHhcNMjUxMDIwMjI0OTU0WhcNMjYxMDIw -MjI0OTU0WjAWMRQwEgYDVQQDDAtzdGVtLWNsaWVudDCCAiIwDQYJKoZIhvcNAQEB -BQADggIPADCCAgoCggIBAJWvR6Pdvjsb4QcmhYQkWLpUptQeK25lbxsg5Toot494 -3opdvmOQSM7dQJ9qyWSZRdF7ThA2cGGz5Lav0NLviXtML4TDU61dEupdJTzBJG5P -qntPeZTn/aTzg2uS0BgkkCTtMHrZR6WFpKFuf4nIvq07sWrR1IBJuu7Um84fU03G -YHrCCryraBPWh085RUXMs4IW81jzmHfxbeTlwVAAIuFWihXaNCppd03mJ6V3LclH -Oyl7pksYoYULtK6Y3yHF7njJ8phnqcQGeR0N5xFSFmwqXrIavjWg92TcevU4YTN3 -lO5CXAVHDNdSeXb31PNIPGY8YmjVHyLkspEXb5GHeQtwNh+q5VkQ6vJ0XwIfTglY -/ikAEg38erBFPxX3sVSk/0CRjZxfPXWio1vxp32uZ4eftn+Lg4bCzG+TXnEVlU03 -Y7UecYtcQUe9iBnCCtNiHVuzphxnmRtyoGdPBrXZgdbNNepLAX9HFkr6jxoRiGR9 -eecAhuHcjssJflC4BKwG0aC/GrBwuj71b9b+oGd9iiN7V1y9pD2ioQyXU+7wGcrV -ER3A4zJRpAUs0K+YW6dgrLcDqABWt/TD3rKiaGiUOZ8zzoXvuvRwtspX6+W3Im/T -LONEdRCoOaMGgD7NdG2p5jS3ogmQvKEstGF6N35UR5NYeXEqJNAHb5tfr02Zm46f -AgMBAAGjYjBgMBMGA1UdJQQMMAoGCCsGAQUFBwMCMAkGA1UdEwQCMAAwHQYDVR0O -BBYEFNsSFWMqMyACeIWavOD6mISQxtdZMB8GA1UdIwQYMBaAFJ3b1bcmcE0pMk46 -dFb8ZBz0+iv1MA0GCSqGSIb3DQEBCwUAA4ICAQAJby+iC0+zIn/yWjXqWn+fJ6DY -DtZLtP0/lamuz2/rJFUkEvl79nwtj2tP8JpciioPiegqaGwWEMcpIuYbeVVHJdij -FuKLSKv+Qyqbyuw93NyUsz6+VazNtvv8lsWCL5AqURxEsSRhRY3Ty2dhPL9NPSk7 -XRkqG5pT+zCxKCtsQ5kjQWYlWf+/wx+Bd254GUqAy5WuUdPNq7zS69ZzdHBv0kuf -PFlugU/eHIGMHvQMAk3eIQMkep9pruhdUjX/d8KJNJ/l+5IhxUTBURnJ3O1HZqjf -IMwal1f/A2EXYVi2NVQj5175aLEfPN+SF0jpZlv93x/+FdQf+86I8yH4KFdjNRxQ -ZVhiaS2tWz4pjX0Vxvm/WMDYnDIo5d9RgXWwUvoSFVcP/ULKvcx1fXV3Pm4Iwmbu -dyJSdv39lOSntLreSFi6qfKt4wVOMn5L+HS9tkMWm6i7eBi8h/RBOGhhpP2lNolX -3jIjhyJKQA3tso+cB1vpKXOklKm2EuKdfiVLDfr/3rwh8hAxLsqTvJxXPeGZCd+y -CzKaUzu1QDR4Zkt1wjWgZfDjlp+44x5lIRuCMdyGqL+kG4XKT3C8Mev/X1Twzk7y -lUh3izAt6vh3v5cAQH+ROSw6hi3oJgWY0ih62aedorFLF2riqDCgqa/I1fSVuRJV -lAG8xEXUsmbaSDYeMQ== ------END CERTIFICATE----- diff --git a/packages/stem/example/microservice/certs/client.key b/packages/stem/example/microservice/certs/client.key deleted file mode 100644 index abbc650f..00000000 --- a/packages/stem/example/microservice/certs/client.key +++ /dev/null @@ -1,52 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCVr0ej3b47G+EH -JoWEJFi6VKbUHituZW8bIOU6KLePeN6KXb5jkEjO3UCfaslkmUXRe04QNnBhs+S2 -r9DS74l7TC+Ew1OtXRLqXSU8wSRuT6p7T3mU5/2k84NrktAYJJAk7TB62UelhaSh -bn+JyL6tO7Fq0dSASbru1JvOH1NNxmB6wgq8q2gT1odPOUVFzLOCFvNY85h38W3k -5cFQACLhVooV2jQqaXdN5ieldy3JRzspe6ZLGKGFC7SumN8hxe54yfKYZ6nEBnkd -DecRUhZsKl6yGr41oPdk3Hr1OGEzd5TuQlwFRwzXUnl299TzSDxmPGJo1R8i5LKR -F2+Rh3kLcDYfquVZEOrydF8CH04JWP4pABIN/HqwRT8V97FUpP9AkY2cXz11oqNb -8ad9rmeHn7Z/i4OGwsxvk15xFZVNN2O1HnGLXEFHvYgZwgrTYh1bs6YcZ5kbcqBn -Twa12YHWzTXqSwF/RxZK+o8aEYhkfXnnAIbh3I7LCX5QuASsBtGgvxqwcLo+9W/W -/qBnfYoje1dcvaQ9oqEMl1Pu8BnK1REdwOMyUaQFLNCvmFunYKy3A6gAVrf0w96y -omholDmfM86F77r0cLbKV+vltyJv0yzjRHUQqDmjBoA+zXRtqeY0t6IJkLyhLLRh -ejd+VEeTWHlxKiTQB2+bX69NmZuOnwIDAQABAoICAA9j+t8/9QfZ2XzlOpQ4MoFS -I+cqhPewaUimTLt6pUkUufkLI4DUPJA8kqDxl2yHD/U2kvO6zTpWh107XaLaq02o -qzgg51ErgT7kveftygOnoFVWeh9T7TIRMcGVUl0xITsSwknXXnUH9UH9/RIbOHPS -b+8WTR21twlIhg6fDO8xxwjIO9djXmuvwrDkKBnBrvFgz7ulLFek1NJUx1gk83Es -9zzA910nA/PYBtqOEwCfmNculg4hpbXq2rSbFlJAKkF0LkP6YKu9qKVSoSAYSyLr -BlqUwlr8RoOx4dil5+SlVoRbAWAvioP3QHL5j4VGN6OhcNLrmux5DF07xiZ2Af2B -Q8h9fFG6htbbQu2xigBVuQW47DzGrvBHBZG78N7NkHW5At7kJ+d2XmIoGBK5EXgo -SHRQOeWRt4eZJXewtLNoKrwRQWKFCZdQQ7lm/AA7PbGa/8vBrRaaA8iG+6Y5uw1k -5pW/nwywYEYXNY8lcnqxPyN/oR9egbxPl0fVdJZpT9DsGfqsOdW2tKlSi4Fpcmsc -pglBsrrQNrqQAzmojcGDwMrahc7nM/Z2zsGMViK8VuecnmyaErJzwEXVUAqLEGz6 -2KutJruEtajDCIcixh9ZmxH0pfoE5ymP4CQ/zb/rj4E0TpCKqaA17TIGb7Cewiwk -yIScvNofHHm/I5AR4UWtAoIBAQDP2kv3gJqDXhnc4Cdij94lt+0Oq9+3/AJpSI3O -srMWXeENXsTskxSHcsw9ag3Lds470k+Cb3iMOauWQhkHDV3rRnn7m4uaYHsYJoFp -GCgUl/CxeI8hlW5c4fYpbvxoF6rOOtDanhOtCcBFIQSs1un6yl3iZDXXcxtGnQLm -bs55OydWzReegDbaZOTgyJI3ahHYY8iQMUYehKfZYBAqa47UQt5QFk9Q802N0mGl -X8i/fPR4/eXgM7Pq5TR/eR+y/InNkQoCUVwvzqdyukRZFXeHWccl21JHHg0ZQeYa -i9Gs2iy93DWm/EH/tW71JuLmEHAAmBzMCscs3KJjZJM+8bm7AoIBAQC4W5u69f5T -epiS3mvwU6XTWndcx//FH367u9h40/oxGzx4l5+eoOVq0u6zVqabTTkMVnDcEgfi -J4GJUA4yQ+1dZQFkAr3YtM/XvIT23BgxnrAHunMPnh3WV4YRz9Qi0g10DMJlBJCb -g1cBC3JC4He6hPUw4RTGNe0Q1VvY5s9A4EMfNnz8OZ22FlxjOHbGJCaRG/drUT/t -qpT//QnXzi6St1D/1DjEI2eaPHZxlKpRRJSl5wNTphoeORX7Wmx9+GA2LSDVI9GG -0mNDXJdiJNLFkZe2vJf+jZjQBV/wKykS7rYYD2IpE1RFPMxG9gL0/r6hulYMZ3cC -U+Aiobf2gc5tAoIBAQC6nBuuFzkiDl8b7eeOVuo0ZQAhNvBBh7FGgwxF8ixVPW3r -ihU07Qv2W3vA+7Fs5bpc/QHDcn8BLlm+KpO7wCV9MSYdXlYlANAjK5buGvEJJCzF -uR6QIVAXdguCwZE9h1ojk6ylSl0R8vo7VlyTLNSfBAbsVdn9F+v6JsdOyVb4gsJb -7QBynlwhEYpnwct8ku94noWX55UzmxfzUorQ+wYdzfc4Yf9OB1KurJ5cUjWyreXz -psg21VSH5MaQV9e843x5IemhkRg6F8hAebkZXeuVimYAt/sTvIg4D/EeDj/ud2bK -AqXsyPgcMvFuij4WHdXgwrDABEE0JtZodAi67103AoIBAHNe+O0ehsNNJHA8+0WR -w2Op6KrKinidBh2MJ6Mt3ROgHM7G7GBEBm48hJ78qhTkVcAv1D8UlZRk7r60t72j -Hsdkr2McCBy2wp8a3gbyo71htn2Po/PxGnEJouBAtFv6IesvFPRM0AZSQj9oH2BU -wviZviuRoAOOucXgD7OEdcz+y2KcIclicR0Dj1uUeVf0RaJn4wVo+l3Mc6RjgSZp -ZGHykKEY+VFdarduAR2BULi8911/XWiauRuS0/vRkbWfmu4sfbCE0tnRjccZCa+k -pbZ4L35vNZtal3gO9QQCrpinhJM4BVD5RLJzos8Bo8YYKcpiJvlqQmLVhJ2HoO2G -PF0CggEATf0/cPPGNqe2/DLpev3bRgkIHT35zkjfGQmIrYqZ5yPxNvI3lOqX3fPW -B0zUySxgZyfinmVUzPNArBpKRCB6GxRDqkjk5+SrN59ReAb+lNQgVbHnpVnZWGkc -3aYic0aMhUR+6nSbNPfc5i4sfAlTh7PvX2fyEvkiW06A0avyZZhqK3tQAqfXs6ll -Z5QGqG5QPithvK3W2axCzoS/xbObulFnlJhC9XCYB2t7x1ytcR5H7soXf0B6m7He -ay8CkAbHjnlkRnIWLM75bmA1inlMPQuHcmjUE19BbKEXIGgrlc0NsHIznHSZBSAR -4V6nCEhErI0mUtt2mtXYDOcDb5C2Cw== ------END PRIVATE KEY----- diff --git a/packages/stem/example/microservice/certs/server.crt b/packages/stem/example/microservice/certs/server.crt deleted file mode 100644 index af57e537..00000000 --- a/packages/stem/example/microservice/certs/server.crt +++ /dev/null @@ -1,30 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIFMDCCAxigAwIBAgIUKQJMnXGqR0sgtrJxHlRpDjMXDfMwDQYJKoZIhvcNAQEL -BQAwEzERMA8GA1UEAwwIcmVkaXMtY2EwHhcNMjUxMDIwMjI0OTU0WhcNMjYxMDIw -MjI0OTU0WjAQMQ4wDAYDVQQDDAVyZWRpczCCAiIwDQYJKoZIhvcNAQEBBQADggIP -ADCCAgoCggIBANN+YsvIKcRdHIAQnYBALgkYKEbu0rX8zcnEyznwhXT/NWAJlKas -mjK8tprEl8R1Pz9Ci81SwYd+/PEUD9j2iSahAQK/RpC9W60VWTioxfNBX3kzQPBR -eUZfluQMCVlt8m2sUfoQp1aHqJ8UrpAomsdVWBIEKPCaJxthBof3OME+ko9r8Pqb -tgbQNfPT4LmnVnh/Pw/aCZLOn5O5syCBYlobcV/yVQNLFag1SmYDS3Q27W00GpIV -5xORzLSIj5fcg9Igu0LKCS/eI6U6SITsiCVruDo7nwj2M7EA2c24fVcLIPWURiMx -ryLbBJS62CX2ajlWFRN+qocS2fGpszKxjc/e0DByficJ/Ajmm6vMIVoWrD5kHwOF -Kms/AuT88n54cEEBSJxNdhFHNWMFS9sE/T9cabbIyHjDhD63vF03RIIoxNderbeZ -Fr4JzR/EmsI1QQ1vlIOjTwkKCz0KzWFUx4RmHtTGWGf8TxgLIDlqZmScGC2j92HL -og1HOm3IAlzZCdPpipxopAIxVpf4mfNT47VnAsjQ0Sm1A3q299jj+PX2Tyrusno5 -7mMBQWDPBPGOOfd5g55LBn7yYGTcTXOIdn1CQmpvpjD/JiOlCqVoBL0mFCBq3vua -JqdvKc9gczHRLEfUOZwMSmfZjzO2CF1nkzdUA95yFlk+Rub/K+MyqUTNAgMBAAGj -fzB9MBsGA1UdEQQUMBKCBXJlZGlzgglsb2NhbGhvc3QwEwYDVR0lBAwwCgYIKwYB -BQUHAwEwCQYDVR0TBAIwADAdBgNVHQ4EFgQUUo1ejmK7qneQRvUI4x29mxzaJJ0w -HwYDVR0jBBgwFoAUndvVtyZwTSkyTjp0VvxkHPT6K/UwDQYJKoZIhvcNAQELBQAD -ggIBAE1LPXaDtvuTX++wDdwARHxRtZvtk0F5gElK3OyBBepVAeUo40fO4BRAADb+ -PYXQahSyn2jolvP1ONuXuxIA4Wiopon53p/3w/F+xkeAtPOT6yf5eV4gQtkUvgbh -Nl8K4zgOE3BwWmYHqp5XCv18wBSGrzW4k9+Xxrtnoq2f3Zf9lHsUjZlJUyKB6cRu -/4LWwF+nGLDpLtKFMksE/0Hki8DWTWnu9+VTLycRmdNN8jC+CNuA/0cyyB/dqlLh -0YY7rAyDJpAMQXhCtF+KEgF4tshKFc3ZJIdaWnr7zg9QGinuq4ooOnVcL4s1yI22 -+V3UiV5n7a5umQJlxlc7filfY1IG5oJbhTDKTOw1b2Z3cu6PaRG5m6udleCLUOOp -k9/fWkbjg5XBtb4yiWvUEPVNzqB9MrpfKJiDko3YhbnV7qAAWJ+4buc9+5sstt8P -mRmid+HkxPgM1PiDhBrKBpcg5NenstTidT1CI2Gv27fuso9xrXe8nRR8gCNrlvx6 -y57uC8aqP0mfuSynaFYiI8+fzvklPFjDmKW6C8jzZUzbwge1DaHAuz2sg1Nq1r95 -UiiuSHOLkAlT7rBgUivEMXlCg9o0oLGd+fyK3cCHc+zeHuQXLMdXgJ+1UmhubTGR -u2tIg/KlFLSlMh+K1FlIA74tKP04/eZ/DH+j73S9VbcNjofx ------END CERTIFICATE----- diff --git a/packages/stem/example/microservice/certs/server.key b/packages/stem/example/microservice/certs/server.key deleted file mode 100644 index cbae2c52..00000000 --- a/packages/stem/example/microservice/certs/server.key +++ /dev/null @@ -1,52 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQDTfmLLyCnEXRyA -EJ2AQC4JGChG7tK1/M3JxMs58IV0/zVgCZSmrJoyvLaaxJfEdT8/QovNUsGHfvzx -FA/Y9okmoQECv0aQvVutFVk4qMXzQV95M0DwUXlGX5bkDAlZbfJtrFH6EKdWh6if -FK6QKJrHVVgSBCjwmicbYQaH9zjBPpKPa/D6m7YG0DXz0+C5p1Z4fz8P2gmSzp+T -ubMggWJaG3Ff8lUDSxWoNUpmA0t0Nu1tNBqSFecTkcy0iI+X3IPSILtCygkv3iOl -OkiE7Igla7g6O58I9jOxANnNuH1XCyD1lEYjMa8i2wSUutgl9mo5VhUTfqqHEtnx -qbMysY3P3tAwcn4nCfwI5purzCFaFqw+ZB8DhSprPwLk/PJ+eHBBAUicTXYRRzVj -BUvbBP0/XGm2yMh4w4Q+t7xdN0SCKMTXXq23mRa+Cc0fxJrCNUENb5SDo08JCgs9 -Cs1hVMeEZh7Uxlhn/E8YCyA5amZknBgto/dhy6INRzptyAJc2QnT6YqcaKQCMVaX -+JnzU+O1ZwLI0NEptQN6tvfY4/j19k8q7rJ6Oe5jAUFgzwTxjjn3eYOeSwZ+8mBk -3E1ziHZ9QkJqb6Yw/yYjpQqlaAS9JhQgat77mianbynPYHMx0SxH1DmcDEpn2Y8z -tghdZ5M3VAPechZZPkbm/yvjMqlEzQIDAQABAoICABNL30C/kQwBANYwbeZjNLAQ -u7AJH0GKgM4tfetZ5htQI7MQ/LaeXe1OKY0CbeR3XB9t92py6ARwUiy+y1lc2ROk -sf5prEsNc91dasgQIgt5Rfg9dGnYzSteElrZPEskDG/0PoCF33a2JuzoyqqNgd25 -DDGMnp2hOvs1BY8HP+7lwiKC0zhSV6/UdG+8P69AtNOGrJ7KIoB6CLZGRbEsDOiy -w0vhRN9pjN7Nie231D0Uu+HwVJJKnEw9S8OXCB7veDmGUf+chm0PJApxwGccW6mq -Y+MwSGjdacb3EGhEMm9sLQQrFKEmadfQyuC1EX6ZNoTntUGTl4R+dFMnzBlOeOU5 -ckfYmST938Pa+xs5RMTN6E+hZT3hBXdbSQeYCg6888Uv+GJHObvFZZjPa0edojSm -D0wu4ULB0WASr+YCSVnncAtvhm7C8kjF+tzBsVkhchdET8w19P05+yiYhSL01AyI -nMOK3rt2LQuyaZeKEexffmCD2JJNKsVi4GYIomZxTqVGwQAuEbDmWQWZReaBotPT -DAZQnAKh6/EgUW0RMF3Hw4dq9/FY+98G1nHzTCBbnBQnevFYSDm1N8+LR6TGMCaT -6nYaIlAxmppcwMKA2g0m/eoPmQ9cF0XLKJRU+XlawIJWRPUGNGTBSYcl/XO04Tch -F6HkebOssy/8F0evuKPxAoIBAQDup2Dxp7SdjMZ7Bw2moArL0nWciOD+MQKD51KP -cA9AGZN+eIqUP+bsExZoBjKoSQd7G32c4RzAKGMyzDipoqqDOwBS5nCQjjC8idX3 -yKwZQt7in/3o7duMT9tiTOExihT13cuFZQ9RtZ/pbp0QbuUFzzaV8XvgooOBKAmJ -X1NHbLwcf7wyl9B8T2Y/WOH+6KlfIwHNwmFkPMtzOMXiQ59lJghpbGto7EmVETCN -VDzosEebZGpIkD2d/0QFjBSpLbidW+UUTgjCw2cR1P6xRJ//pjZ7dycqb3rMe2c9 -UGfnVo1jOtkmbK0Zo4eSO+JAhn4GzbhiNid4NH6QzHifuYwlAoIBAQDi3aPRXoYo -1+fnNF2Riz09OzQaU/SpzyhvvYncuyZUoziqBsCLov8AlxsDziTStD+/DripLURg -g50uVCU692VYWVElQf0OyDoP5/VI88mnfUjMkotC7ETjwpeYSY0XDzmCOQYD/p0F -bUU1GNsxo3r8+4mYDFtHiQeYOnfk983AD9l4N9qJEUTSS0pfDB4WseccJwuipoXJ -gWPLslehgrEW5SPei2w3TOSgFCe0fWi6qzK+iw5z+jn9VwNhNFIvdGeQ4j0mETk2 -+oKMWrHiHhwI/hsFCteIq3hPb9uEDSH4T73KT9CnxPw+6b+j7wBUU+ZkUb4FnjaY -KTKq5h2IMqGJAoIBABNdlapGV4upTrnQ+rSYrAeaDl0jZk56H6AgdzyYdURYTvWm -Rmh9PvnNGrZaOmYZdvfwZSMDyVd2b4ZBuB9bycZYVVBkzqGMSDwkWy5+JCjwmjF9 -lhb+piAeosi5QFTKxaSl2Mv8RV/2AwHoLeqVdRQhmmZUcB7sUQt7+joheBqFokYV -v2nDk/FiaAOf7MbIjmXf61eM/GYr4L56zuPrZNXKXkhfud+LvPTRrkTpnv+qvkTH -wzAcNEPL4PMqSMrpoN90A4TASEm1nXl+9SPhU7JlBGrEYnM5FJffM6NwEN5e0Z5J -ujE6BFneQTQGk/d0cmmO6/VcO5uvds29KdScM9kCggEANN+HEILBLb+0EpSHwiX7 -bcvghspOWU0olXJNMR6E/nDUAfx035ZrbA3LhAWFW34a5UJfEz6zkvLZrTR7QNTr -z71xRgetHMbZkV90MT5wHxL55oO45S3/mD1/60QpF9i6hHkZx9CNc/hVitxyKY16 -Wn+dP8yRcYikNnxZ+Qkv8LVA793ElvFpYsGdYa2HTa7EuDD5azz4LVB0m8LC3aeB -8l4REArae8D1/8DyyAP6ZpcDypjKE4mb/klLRlpVVS4B/ShdLAQTm5ai2mdA0fxG -eGoQjgpK1VVxQsJFHO+anS6ogwA7Fv6380mzGlfXcc/kADl02gZXnvrrDh13gIic -UQKCAQBODS5bTyKA8sxT+YdL1iiMyyydo4r10WSg3ys6HVhJK+IRTxMEb3wZfPS8 -lDjxGYcra+1vtef5hrKmYCsaTIbZtQ05VM68VP+aBH6U4Mvh02SMnkfjVx15lrXz -H8RkEe7janCnv2FOo6m9AHiVkyMVj4dKN/z/jwwdTLjGT+F/jlJ819DXGc4zs8VL -+mAgCRUdSw/3Hgzxu5vfRFmv+6DPvuu9Kehloc7I+GgtNuInPr6LFH3w1U08XrHR -lrfx4+PRPqJibpVzsIeFOqE8SXRnYh3Mc5ncz8xxeP+TQ991je/VIgom9mtttbxb -pnmw3CS2914DktHUbIVNsnogaI7R ------END PRIVATE KEY----- diff --git a/packages/stem/example/microservice/worker/bin/worker.dart b/packages/stem/example/microservice/worker/bin/worker.dart index ad46ef6f..56f7cc50 100644 --- a/packages/stem/example/microservice/worker/bin/worker.dart +++ b/packages/stem/example/microservice/worker/bin/worker.dart @@ -2,6 +2,7 @@ import 'dart:async'; import 'dart:io'; import 'package:stem/stem.dart'; +import 'package:stem/memory.dart'; import 'package:stem_redis/stem_redis.dart'; const _taskSpecs = <_WorkerTaskSpec>[ @@ -66,12 +67,15 @@ Future main(List args) async { config.brokerUrl, tls: config.tls, ); - final backend = config.resultBackendUrl != null - ? await RedisResultBackend.connect( - config.resultBackendUrl!, - tls: config.tls, - ) - : InMemoryResultBackend(); + final ResultBackend backend; + if (config.resultBackendUrl != null) { + backend = await RedisResultBackend.connect( + config.resultBackendUrl!, + tls: config.tls, + ); + } else { + backend = InMemoryResultBackend(); + } // #region signing-worker-signer final signer = PayloadSigner.maybe(config.signing); // #endregion signing-worker-signer diff --git a/packages/stem/example/monolith_service/bin/service.dart b/packages/stem/example/monolith_service/bin/service.dart index 2a4b4f77..6160e29c 100644 --- a/packages/stem/example/monolith_service/bin/service.dart +++ b/packages/stem/example/monolith_service/bin/service.dart @@ -6,6 +6,7 @@ import 'package:shelf/shelf.dart'; import 'package:shelf/shelf_io.dart'; import 'package:shelf_router/shelf_router.dart'; import 'package:stem/stem.dart'; +import 'package:stem/memory.dart'; Future main(List args) async { final port = int.tryParse(Platform.environment['PORT'] ?? '8080') ?? 8080; diff --git a/packages/stem/example/persistent_sleep.dart b/packages/stem/example/persistent_sleep.dart index 340a4b06..fad25bef 100644 --- a/packages/stem/example/persistent_sleep.dart +++ b/packages/stem/example/persistent_sleep.dart @@ -24,6 +24,7 @@ Future main() async { final app = await StemWorkflowApp.inMemory( flows: [sleepLoop], ); + await app.start(); final runId = await sleepLoop.start(app); await app.executeRun(runId); diff --git a/packages/stem/example/postgres_tls/README.md b/packages/stem/example/postgres_tls/README.md index 099b058f..37c4e6db 100644 --- a/packages/stem/example/postgres_tls/README.md +++ b/packages/stem/example/postgres_tls/README.md @@ -20,7 +20,7 @@ can be reused across brokers and backends. ```bash export STEM_BROKER_URL=redis://127.0.0.1:${REDIS_PORT:-6379} export STEM_RESULT_BACKEND_URL=postgresql://postgres:postgres@127.0.0.1:${POSTGRES_PORT:-5432}/stem_test - export STEM_TLS_CA_CERT=../../../stem_cli/docker/testing/certs/postgres-root.crt + export STEM_TLS_CA_CERT=../../../stem_cli/docker/testing/postgres/certs/root.crt ``` 3. Compile binaries: ```bash diff --git a/packages/stem/example/postgres_tls/Taskfile.yml b/packages/stem/example/postgres_tls/Taskfile.yml index e2789e02..251bd678 100644 --- a/packages/stem/example/postgres_tls/Taskfile.yml +++ b/packages/stem/example/postgres_tls/Taskfile.yml @@ -73,7 +73,7 @@ tasks: export POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-postgres}" export STEM_BROKER_URL="${STEM_BROKER_URL:-redis://127.0.0.1:${REDIS_PORT}}" export STEM_RESULT_BACKEND_URL="${STEM_RESULT_BACKEND_URL:-postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@127.0.0.1:${POSTGRES_PORT}/stem_test}" - export STEM_TLS_CA_CERT="${STEM_TLS_CA_CERT:-$root/../../../stem_cli/docker/testing/certs/postgres-root.crt}" + export STEM_TLS_CA_CERT="${STEM_TLS_CA_CERT:-$root/../../../stem_cli/docker/testing/postgres/certs/root.crt}" "$root/build/worker/bundle/bin/"* ' @@ -101,7 +101,7 @@ tasks: export POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-postgres}" export STEM_BROKER_URL="${STEM_BROKER_URL:-redis://127.0.0.1:${REDIS_PORT}}" export STEM_RESULT_BACKEND_URL="${STEM_RESULT_BACKEND_URL:-postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@127.0.0.1:${POSTGRES_PORT}/stem_test}" - export STEM_TLS_CA_CERT="${STEM_TLS_CA_CERT:-$root/../../../stem_cli/docker/testing/certs/postgres-root.crt}" + export STEM_TLS_CA_CERT="${STEM_TLS_CA_CERT:-$root/../../../stem_cli/docker/testing/postgres/certs/root.crt}" "$root/build/enqueue/bundle/bin/"* ' diff --git a/packages/stem/example/quality_gates/README.md b/packages/stem/example/quality_gates/README.md deleted file mode 100644 index 614d0c9f..00000000 --- a/packages/stem/example/quality_gates/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# Quality Gate Runner - -This example provides a `just`-based command runner for common quality gates -(formatting, analysis, unit tests, chaos tests, performance tests, and coverage). - -## Usage - -```bash -cd example/quality_gates -# or from repo root: -# cd packages/stem/example/quality_gates - -# Fast checks -just quick - -# Full gate set (format + analyze + unit + chaos + perf + coverage) -just quality - -# Smoke-build the example binaries (optional) -just examples-smoke -``` - -Coverage output is written to `packages/stem/coverage/lcov.info`. diff --git a/packages/stem/example/quality_gates/justfile b/packages/stem/example/quality_gates/justfile deleted file mode 100644 index 29971a82..00000000 --- a/packages/stem/example/quality_gates/justfile +++ /dev/null @@ -1,48 +0,0 @@ -import "../Justfile.common" - -ROOT := justfile_directory() -REPO_ROOT := env_var_or_default("REPO_ROOT", ROOT + "/../../../../") - -COMPOSE_FILE := env_var_or_default("COMPOSE_FILE", "") -DEPS_SERVICES := env_var_or_default("DEPS_SERVICES", "") -BUILD_DIR := env_var_or_default("BUILD_DIR", "build") -ENV_FILE := env_var_or_default("ENV_FILE", ".env") -SESSION := env_var_or_default("SESSION", "stem-quality-gates") -PROJECT_NAME := env_var_or_default("PROJECT_NAME", SESSION) -COMPOSE_ENV := "" - - - -build: - @echo "No build artifacts required." - -format: - cd "{{REPO_ROOT}}"; dart format --output=none --set-exit-if-changed . - -analyze: - cd "{{REPO_ROOT}}"; dart analyze - -unit: - cd "{{REPO_ROOT}}/packages/stem"; dart test --fail-fast - -chaos: - cd "{{REPO_ROOT}}"; dart test packages/stem_redis/test --tags chaos --fail-fast - -perf: - cd "{{REPO_ROOT}}"; dart test packages/stem/test/performance --fail-fast - -coverage: - cd "{{REPO_ROOT}}/packages/stem"; dart test --coverage=coverage --fail-fast - cd "{{REPO_ROOT}}/packages/stem"; dart run coverage:format_coverage --in=coverage --out=coverage/lcov.info --lcov --report-on=lib - -quick: format analyze unit - -quality: format analyze unit chaos perf coverage - -examples-smoke: - cd "{{REPO_ROOT}}/packages/stem/example/autoscaling_demo"; just build - cd "{{REPO_ROOT}}/packages/stem/example/scheduler_observability"; just build - cd "{{REPO_ROOT}}/packages/stem/example/signing_key_rotation"; just build - cd "{{REPO_ROOT}}/packages/stem/example/ops_health_suite"; just build - cd "{{REPO_ROOT}}/packages/stem/example/progress_heartbeat"; just build - cd "{{REPO_ROOT}}/packages/stem/example/worker_control_lab"; just build diff --git a/packages/stem/example/rate_limit_delay/README.md b/packages/stem/example/rate_limit_delay/README.md index 055721ea..05955ba8 100644 --- a/packages/stem/example/rate_limit_delay/README.md +++ b/packages/stem/example/rate_limit_delay/README.md @@ -2,7 +2,7 @@ This example exercises Stem’s rate limiting, delayed delivery, and priority clamping features using Redis. A burst of tasks is enqueued with different -priorities and optional `notBefore` timestamps. A custom Redis-backed rate +priorities and optional `notBefore` timestamps. The shipped Redis-backed rate limiter enforces a global `3/s` token bucket; denied tasks are rescheduled with backoff, and priority values are clamped to the queue’s `[1,5]` range. @@ -10,8 +10,8 @@ backoff, and priority values are clamped to the queue’s `[1,5]` range. - **Redis** – shared broker, result backend, and rate limiter store. - **Producer** – enqueues jobs with mixed delays and priority overrides. -- **Worker** – processes the `throttled` queue with a fixed-window rate limiter, - logging when work is deferred. +- **Worker** – processes the `throttled` queue with a distributed token bucket, + rescheduling work when the bucket is empty. ## Quick Start (Docker Compose) @@ -24,9 +24,7 @@ You should see logs similar to: ``` rate-limit-delay-producer-1 | [producer] job=1 priority=9 applied=5 delay=0s id=... -rate-limit-delay-worker-1 | [rate-limiter][granted] key=demo.throttled.render:global tokens=3 window=1000ms -> available immediately rate-limit-delay-worker-1 | [worker][start] job=1 attempt=0 requestedPriority=9 appliedPriority=5 rateLimited=false ... -rate-limit-delay-worker-1 | [rate-limiter][denied] key=demo.throttled.render:global tokens=3 window=1000ms -> retry in 872ms rate-limit-delay-worker-1 | [signal][retry] task=demo.throttled.render retry=0 next=... ``` @@ -61,8 +59,8 @@ rate-limit-delay-worker-1 | [signal][retry] task=demo.throttled.render retry= ## What to Observe -- **Rate limiting:** the custom Redis fixed-window limiter logs whether tokens - were granted or denied, with `retryAfter` durations surfaced in the worker. +- **Rate limiting:** the Redis token bucket shares permits across worker + processes, with `retryAfter` durations surfaced in the worker. - **Delayed delivery:** half the jobs include a `notBefore` timestamp—watch the worker start times versus the scheduled time in the log output. - **Priority clamping:** tasks request priority 9, but the routing config clamps diff --git a/packages/stem/example/rate_limit_delay/lib/rate_limiter.dart b/packages/stem/example/rate_limit_delay/lib/rate_limiter.dart deleted file mode 100644 index 08b48ea0..00000000 --- a/packages/stem/example/rate_limit_delay/lib/rate_limiter.dart +++ /dev/null @@ -1,147 +0,0 @@ -import 'dart:async'; -import 'dart:io'; - -import 'package:redis/redis.dart'; -import 'package:stem/stem.dart'; - -/// Simple fixed-window rate limiter backed by Redis. -/// -/// This is intentionally lightweight for demo purposes. Production deployments -/// should use a more robust implementation with Lua scripts cached server-side -/// and better error handling. -// #region rate-limit-redis-limiter -class RedisFixedWindowRateLimiter implements RateLimiter { - RedisFixedWindowRateLimiter._( - this._connection, - this._command, { - required this.namespace, - }); - - final RedisConnection _connection; - final Command _command; - final String namespace; - bool _closed = false; - - static const _script = ''' -local key = KEYS[1] -local limit = tonumber(ARGV[1]) -local interval = tonumber(ARGV[2]) - -local current = redis.call('GET', key) -if not current then - redis.call('SET', key, 1, 'PX', interval) - return {1, interval} -end - -current = tonumber(current) -if current < limit then - redis.call('INCR', key) - local ttl = redis.call('PTTL', key) - if ttl < 0 then ttl = interval end - return {1, ttl} -end - -local ttl = redis.call('PTTL', key) -if ttl < 0 then ttl = interval end -return {0, ttl} -'''; - - static Future connect( - String uri, { - String namespace = 'stem-demo', - }) async { - final parsed = Uri.parse(uri); - final host = parsed.host.isNotEmpty ? parsed.host : 'localhost'; - final port = parsed.hasPort ? parsed.port : 6379; - final connection = RedisConnection(); - final scheme = parsed.scheme.isEmpty ? 'redis' : parsed.scheme; - - if (scheme == 'rediss') { - throw UnsupportedError( - 'TLS connections are not implemented for the rate limiter demo. ' - 'Use redis:// URLs or extend the example.', - ); - } - - final command = await connection.connect(host, port); - - if (parsed.userInfo.isNotEmpty) { - final parts = parsed.userInfo.split(':'); - final password = parts.length == 2 ? parts[1] : parts[0]; - await command.send_object(['AUTH', password]); - } - - if (parsed.pathSegments.isNotEmpty) { - final db = int.tryParse(parsed.pathSegments.first); - if (db != null) { - await command.send_object(['SELECT', db]); - } - } - - final resolvedNamespace = parsed.queryParameters['ns'] ?? namespace.trim(); - - return RedisFixedWindowRateLimiter._( - connection, - command, - namespace: resolvedNamespace.isEmpty ? 'stem-demo' : resolvedNamespace, - ); - } - - String _keyFor(String key) => '$namespace:rate:$key'; - - @override - Future acquire( - String key, { - int tokens = 1, - Duration? interval, - Map? meta, - }) async { - final window = interval ?? const Duration(seconds: 1); - final response = await _command.send_object([ - 'EVAL', - _script, - 1, - _keyFor(key), - tokens, - window.inMilliseconds, - ]); - - if (response is! List || response.length != 2) { - throw StateError( - 'Unexpected response from rate limiter script: $response', - ); - } - - final allowed = (response[0] as num).toInt() == 1; - final ttlMs = (response[1] as num).toInt(); - final remainingMs = ttlMs < 0 ? window.inMilliseconds : ttlMs; - final retryAfter = allowed ? null : Duration(milliseconds: remainingMs); - - final decision = RateLimitDecision( - allowed: allowed, - retryAfter: retryAfter, - meta: { - 'windowMs': window.inMilliseconds, - 'remainingMs': remainingMs, - if (meta != null) ...meta, - }, - ); - - final status = allowed ? 'granted' : 'denied'; - final retryText = retryAfter == null - ? 'available immediately' - : 'retry in ${retryAfter.inMilliseconds}ms'; - stdout.writeln( - '[rate-limiter][$status] key=$key tokens=$tokens window=${window.inMilliseconds}ms -> $retryText', - ); - - return decision; - } - - Future close() async { - if (_closed) return; - _closed = true; - await _connection.close(); - } -} -// #endregion rate-limit-redis-limiter diff --git a/packages/stem/example/rate_limit_delay/lib/shared.dart b/packages/stem/example/rate_limit_delay/lib/shared.dart index 849b838f..7c769bae 100644 --- a/packages/stem/example/rate_limit_delay/lib/shared.dart +++ b/packages/stem/example/rate_limit_delay/lib/shared.dart @@ -5,8 +5,6 @@ import 'dart:io'; import 'package:stem/stem.dart'; import 'package:stem_redis/stem_redis.dart'; -import 'rate_limiter.dart'; - const _taskName = 'demo.throttled.render'; List> buildTasks() { @@ -18,7 +16,7 @@ List> buildTasks() { queue: 'throttled', maxRetries: 0, visibilityTimeout: Duration(seconds: 60), - rateLimit: '3/s', + rateLimit: const RateLimit.perSecond(3), ), entrypoint: _renderEntrypoint, ), @@ -46,8 +44,8 @@ Future connectBackend(String uri) => RedisResultBackend.connect(uri); // #region rate-limit-redis-connector -Future connectRateLimiter(String uri) => - RedisFixedWindowRateLimiter.connect(uri); +Future connectRateLimiter(String uri) => + RedisRateLimiter.connect(uri); // #endregion rate-limit-redis-connector List attachSignalLogging() { diff --git a/packages/stem/example/retry_task/bin/worker.dart b/packages/stem/example/retry_task/bin/worker.dart index c42c3a71..f3ad8715 100644 --- a/packages/stem/example/retry_task/bin/worker.dart +++ b/packages/stem/example/retry_task/bin/worker.dart @@ -2,6 +2,7 @@ import 'dart:async'; import 'dart:io'; import 'package:stem/stem.dart'; +import 'package:stem/memory.dart'; import 'package:stem_redis/stem_redis.dart'; import '../lib/shared.dart'; diff --git a/packages/stem/example/routing_parity/bin/worker.dart b/packages/stem/example/routing_parity/bin/worker.dart index c01b3f71..522e1721 100644 --- a/packages/stem/example/routing_parity/bin/worker.dart +++ b/packages/stem/example/routing_parity/bin/worker.dart @@ -1,6 +1,7 @@ import 'dart:io'; import 'package:stem/stem.dart'; +import 'package:stem/memory.dart'; import 'package:stem_redis/stem_redis.dart'; import 'package:stem_routing_parity_example/routing_demo.dart'; diff --git a/packages/stem/example/signals_demo/bin/worker.dart b/packages/stem/example/signals_demo/bin/worker.dart index 338efd49..bdc97cc8 100644 --- a/packages/stem/example/signals_demo/bin/worker.dart +++ b/packages/stem/example/signals_demo/bin/worker.dart @@ -2,6 +2,7 @@ import 'dart:async'; import 'dart:io'; import 'package:stem/stem.dart'; +import 'package:stem/memory.dart'; import 'package:stem_redis/stem_redis.dart'; import 'package:stem_signals_demo/shared.dart'; diff --git a/packages/stem/example/stem_example.dart b/packages/stem/example/stem_example.dart index a5d464e8..1e937d05 100644 --- a/packages/stem/example/stem_example.dart +++ b/packages/stem/example/stem_example.dart @@ -21,7 +21,7 @@ class HelloTask implements TaskHandler { TaskOptions get options => const TaskOptions( queue: 'default', maxRetries: 3, - rateLimit: '10/s', + rateLimit: const RateLimit.perSecond(10), visibilityTimeout: Duration(seconds: 60), ); // #endregion getting-started-task-options diff --git a/packages/stem/example/task_context_mixed/lib/shared.dart b/packages/stem/example/task_context_mixed/lib/shared.dart index aff34176..4dfe0696 100644 --- a/packages/stem/example/task_context_mixed/lib/shared.dart +++ b/packages/stem/example/task_context_mixed/lib/shared.dart @@ -168,7 +168,7 @@ class InlineCoordinatorTask extends TaskHandler { TaskOptions get options => const TaskOptions( queue: mixedQueue, maxRetries: 1, - rateLimit: '30/m', + rateLimit: const RateLimit.perMinute(30), priority: 3, retryPolicy: TaskRetryPolicy( backoff: true, diff --git a/packages/stem/example/task_usage_patterns.dart b/packages/stem/example/task_usage_patterns.dart index 18a4d382..c14ddc2f 100644 --- a/packages/stem/example/task_usage_patterns.dart +++ b/packages/stem/example/task_usage_patterns.dart @@ -95,6 +95,7 @@ Future main() async { tasks: tasks, workerConfig: const StemWorkerConfig(consumerName: 'example-worker'), ); + await app.start(); await ParentTask.definition.enqueue(app); await invocationParentDefinition.enqueue(app); diff --git a/packages/stem/example/unique_tasks/unique_task_example.dart b/packages/stem/example/unique_tasks/unique_task_example.dart index d440bb12..7f7a990a 100644 --- a/packages/stem/example/unique_tasks/unique_task_example.dart +++ b/packages/stem/example/unique_tasks/unique_task_example.dart @@ -2,6 +2,7 @@ import 'dart:async'; import 'dart:io'; import 'package:stem/stem.dart'; +import 'package:stem/memory.dart'; import 'package:stem_sqlite/stem_sqlite.dart'; class SendDigestTask implements TaskHandler { diff --git a/packages/stem/example/workflows/basic_in_memory.dart b/packages/stem/example/workflows/basic_in_memory.dart index 08477098..8726b649 100644 --- a/packages/stem/example/workflows/basic_in_memory.dart +++ b/packages/stem/example/workflows/basic_in_memory.dart @@ -14,6 +14,7 @@ Future main() async { final app = await StemWorkflowApp.inMemory( flows: [basicHello], ); + await app.start(); final runId = await basicHello.start(app); final result = await basicHello.waitFor(app, runId); diff --git a/packages/stem/example/workflows/cancellation_policy.dart b/packages/stem/example/workflows/cancellation_policy.dart index 10476274..7a3a1eab 100644 --- a/packages/stem/example/workflows/cancellation_policy.dart +++ b/packages/stem/example/workflows/cancellation_policy.dart @@ -28,6 +28,7 @@ Future main() async { final app = await StemWorkflowApp.inMemory( flows: [reportsGenerate], ); + await app.start(); final runId = await reportsGenerate.start( app, diff --git a/packages/stem/example/workflows/runtime_metadata_views.dart b/packages/stem/example/workflows/runtime_metadata_views.dart index f729da42..cf8dcb63 100644 --- a/packages/stem/example/workflows/runtime_metadata_views.dart +++ b/packages/stem/example/workflows/runtime_metadata_views.dart @@ -5,6 +5,7 @@ import 'dart:convert'; import 'package:stem/stem.dart'; +import 'package:stem/memory.dart'; Future main() async { final app = await StemApp.create( diff --git a/packages/stem/example/workflows/sleep_and_event.dart b/packages/stem/example/workflows/sleep_and_event.dart index 7fd9fae7..e0d1b40b 100644 --- a/packages/stem/example/workflows/sleep_and_event.dart +++ b/packages/stem/example/workflows/sleep_and_event.dart @@ -29,6 +29,7 @@ Future main() async { final app = await StemWorkflowApp.inMemory( flows: [sleepAndEvent], ); + await app.start(); final runId = await sleepAndEvent.start(app); diff --git a/packages/stem/example/workflows/versioned_rewind.dart b/packages/stem/example/workflows/versioned_rewind.dart index 174a7115..ae81e0a1 100644 --- a/packages/stem/example/workflows/versioned_rewind.dart +++ b/packages/stem/example/workflows/versioned_rewind.dart @@ -17,6 +17,7 @@ Future main() async { final app = await StemWorkflowApp.inMemory( flows: [versionedWorkflow], ); + await app.start(); final runId = await versionedWorkflow.start(app); await app.executeRun(runId); diff --git a/packages/stem/lib/advanced.dart b/packages/stem/lib/advanced.dart new file mode 100644 index 00000000..f962d014 --- /dev/null +++ b/packages/stem/lib/advanced.dart @@ -0,0 +1,39 @@ +/// Low-level and compatibility APIs for Stem integrations. +/// +/// Prefer `package:stem/stable.dart` for application code. This entrypoint +/// exposes the full historical surface for custom transports, instrumentation, +/// signals, and framework integrations that need implementation details. +library; + +export 'src/backend/encoding_result_backend.dart'; +export 'src/control/control_messages.dart'; +export 'src/control/file_revoke_store.dart'; +export 'src/control/revoke_store.dart'; +export 'src/core/chord_metadata.dart'; +export 'src/core/chord_policy.dart'; +export 'src/core/clock.dart' hide FakeStemClock; +export 'src/core/config.dart'; +export 'src/core/contracts.dart'; +export 'src/core/encoder_keys.dart'; +export 'src/core/envelope.dart'; +export 'src/core/function_task_handler.dart'; +export 'src/core/queue_events.dart'; +export 'src/core/stem_event.dart'; +export 'src/core/task_payload_encoder.dart'; +export 'src/core/unique_task_coordinator.dart'; +export 'src/observability/config.dart'; +export 'src/observability/heartbeat.dart'; +export 'src/observability/heartbeat_transport.dart'; +export 'src/observability/metrics.dart'; +export 'src/observability/snapshots.dart'; +export 'src/observability/tracing.dart'; +export 'src/routing/routing_registry.dart'; +export 'src/scheduler/beat.dart'; +export 'src/scheduler/schedule_calculator.dart'; +export 'src/security/tls.dart'; +export 'src/signals/emitter.dart'; +export 'src/signals/middleware.dart'; +export 'src/signals/payloads.dart'; +export 'src/signals/signal.dart'; +export 'src/signals/stem_signals.dart'; +export 'stable.dart'; diff --git a/packages/stem/lib/memory.dart b/packages/stem/lib/memory.dart new file mode 100644 index 00000000..c50ff644 --- /dev/null +++ b/packages/stem/lib/memory.dart @@ -0,0 +1,8 @@ +/// Optional in-memory adapters for local development and tests. +/// +/// Production applications should import a concrete adapter package instead. +/// This library is intentionally separate from the primary `stem.dart` API so +/// adapter usage is explicit in new code. +library; + +export 'src/memory.dart'; diff --git a/packages/stem/lib/observability.dart b/packages/stem/lib/observability.dart new file mode 100644 index 00000000..fbbe5979 --- /dev/null +++ b/packages/stem/lib/observability.dart @@ -0,0 +1,15 @@ +/// Optional observability integrations for Stem. +/// +/// This library is intentionally separate from `package:stem/stem.dart`. +/// Logging is exposed through Stem-owned configuration types; the underlying +/// logging package remains an implementation detail. +library; + +export 'src/observability/config.dart'; +export 'src/observability/heartbeat.dart'; +export 'src/observability/heartbeat_transport.dart'; +export 'src/observability/logging_api.dart'; +export 'src/observability/logging_types.dart'; +export 'src/observability/metrics.dart'; +export 'src/observability/snapshots.dart'; +export 'src/observability/tracing.dart'; diff --git a/packages/stem/lib/src/backend/encoding_result_backend.dart b/packages/stem/lib/src/backend/encoding_result_backend.dart index 0d5d19bd..583f1f83 100644 --- a/packages/stem/lib/src/backend/encoding_result_backend.dart +++ b/packages/stem/lib/src/backend/encoding_result_backend.dart @@ -17,7 +17,8 @@ ResultBackend withTaskPayloadEncoder( } /// Result backend decorator that applies [TaskPayloadEncoder] semantics. -class EncodingResultBackend implements ResultBackend { +class EncodingResultBackend + implements ResultBackend, AtomicTerminalResultBackend { /// Creates an encoding wrapper around the provided backend. EncodingResultBackend(this._inner, this.registry); @@ -29,6 +30,11 @@ class EncodingResultBackend implements ResultBackend { /// The wrapped backend used for persistence. ResultBackend get inner => _inner; + @override + bool get supportsAtomicTerminalWrites => + _inner is AtomicTerminalResultBackend && + (_inner as AtomicTerminalResultBackend).supportsAtomicTerminalWrites; + @override Future set( String taskId, @@ -53,6 +59,33 @@ class EncodingResultBackend implements ResultBackend { ); } + @override + Future setTerminalIfAbsent( + TaskStatus status, { + Duration? ttl, + }) async { + final encoded = _encodeStatus(status); + if (_inner is AtomicTerminalResultBackend && + (_inner as AtomicTerminalResultBackend).supportsAtomicTerminalWrites) { + final atomic = _inner as AtomicTerminalResultBackend; + return atomic.setTerminalIfAbsent(encoded, ttl: ttl); + } + + // Keep custom ResultBackend implementations source compatible. This path + // preserves the old behavior but is intentionally not advertised as an + // atomic cross-process write. + await set( + status.id, + status.state, + payload: status.payload, + error: status.error, + attempt: status.attempt, + meta: status.meta, + ttl: ttl, + ); + return true; + } + @override /// Fetches a task status and decodes its payload if needed. Future get(String taskId) async { diff --git a/packages/stem/lib/src/bootstrap/factories.dart b/packages/stem/lib/src/bootstrap/factories.dart index 259d969f..112b8862 100644 --- a/packages/stem/lib/src/bootstrap/factories.dart +++ b/packages/stem/lib/src/bootstrap/factories.dart @@ -1,13 +1,7 @@ import 'package:stem/src/control/revoke_store.dart'; import 'package:stem/src/core/contracts.dart'; import 'package:stem/src/core/unique_task_coordinator.dart'; -import 'package:stem/src/observability/config.dart'; -import 'package:stem/src/observability/heartbeat_transport.dart'; -import 'package:stem/src/security/signing.dart'; -import 'package:stem/src/worker/worker_config.dart'; -import 'package:stem/src/workflow/core/event_bus.dart'; -import 'package:stem/src/workflow/core/workflow_store.dart'; -import 'package:stem_memory/stem_memory.dart' +import 'package:stem/src/memory.dart' show InMemoryBroker, InMemoryEventBus, @@ -16,6 +10,12 @@ import 'package:stem_memory/stem_memory.dart' InMemoryRevokeStore, InMemoryScheduleStore, InMemoryWorkflowStore; +import 'package:stem/src/observability/config.dart'; +import 'package:stem/src/observability/heartbeat_transport.dart'; +import 'package:stem/src/security/signing.dart'; +import 'package:stem/src/worker/worker_config.dart'; +import 'package:stem/src/workflow/core/event_bus.dart'; +import 'package:stem/src/workflow/core/workflow_store.dart'; /// Wrapper for constructing and disposing Stem resources lazily. class StemResourceFactory { @@ -41,8 +41,8 @@ class StemResourceFactory { } } -/// Factory for building [Broker] instances. -class StemBrokerFactory extends StemResourceFactory { +/// Factory for building [QueueBroker] instances. +class StemBrokerFactory extends StemResourceFactory { /// Creates a broker factory from create/dispose hooks. StemBrokerFactory({required super.create, super.dispose}); diff --git a/packages/stem/lib/src/bootstrap/stem_app.dart b/packages/stem/lib/src/bootstrap/stem_app.dart index cdec188c..8a0d443a 100644 --- a/packages/stem/lib/src/bootstrap/stem_app.dart +++ b/packages/stem/lib/src/bootstrap/stem_app.dart @@ -13,14 +13,31 @@ import 'package:stem/src/core/stem.dart'; import 'package:stem/src/core/task_payload_encoder.dart'; import 'package:stem/src/core/task_result.dart'; import 'package:stem/src/core/unique_task_coordinator.dart'; +import 'package:stem/src/memory.dart' show InMemoryRevokeStore; import 'package:stem/src/routing/routing_config.dart'; import 'package:stem/src/routing/routing_registry.dart'; import 'package:stem/src/security/signing.dart'; import 'package:stem/src/worker/worker.dart'; -import 'package:stem_memory/stem_memory.dart' show InMemoryRevokeStore; + +/// Worker lifecycle surface for a managed application runtime. +abstract interface class StemWorkerHost { + /// Whether the managed worker is currently running. + bool get isStarted; + + /// Starts task consumption explicitly. + Future start(); + + /// Stops task consumption and disposes managed resources. + Future shutdown(); +} /// Convenience bootstrap for setting up a Stem runtime with sensible defaults. -abstract interface class StemTaskApp implements TaskResultCaller {} +/// +/// The app combines the producer, observer, and worker-host roles for +/// applications that intentionally own all three. Narrower processes should +/// depend on [StemProducer] or [StemObserver] instead. +abstract interface class StemTaskApp + implements TaskResultCaller, StemWorkerHost {} /// Convenience bootstrap for setting up a Stem runtime with sensible defaults. class StemApp implements StemTaskApp { @@ -31,15 +48,13 @@ class StemApp implements StemTaskApp { required this.backend, required this.stem, required this.worker, - required this.allowWorkerAutoStart, required List Function()> disposers, }) : _disposers = disposers { - canvas = _ManagedCanvas( + canvas = Canvas( broker: broker, backend: backend, registry: registry, encoderRegistry: stem.payloadEncoders, - onBeforeDispatch: _maybeAutoStart, ); } @@ -50,7 +65,7 @@ class StemApp implements StemTaskApp { final StemModule? module; /// Active broker instance used by the helper. - final Broker broker; + final QueueBroker broker; /// Optional result backend used by the helper. final ResultBackend backend; @@ -61,9 +76,6 @@ class StemApp implements StemTaskApp { /// Worker managed by the helper. final Worker worker; - /// Whether shortcut operations may lazily start the managed worker. - final bool allowWorkerAutoStart; - /// Canvas facade used for chains, groups, and chords. late final Canvas canvas; @@ -73,15 +85,9 @@ class StemApp implements StemTaskApp { Future? _startFuture; /// Whether the managed worker has been started. + @override bool get isStarted => _started; - Future _maybeAutoStart() { - if (_started || !allowWorkerAutoStart) { - return Future.value(); - } - return start(); - } - /// Registers an additional task handler with the underlying registry. void register(TaskHandler handler) => registry.register(handler); @@ -117,7 +123,6 @@ class StemApp implements StemTaskApp { Map meta = const {}, TaskEnqueueOptions? enqueueOptions, }) async { - await _maybeAutoStart(); return stem.enqueue( name, args: args, @@ -140,7 +145,6 @@ class StemApp implements StemTaskApp { Map meta = const {}, TaskEnqueueOptions? enqueueOptions, }) async { - await _maybeAutoStart(); return stem.enqueueValue( name, value, @@ -158,19 +162,16 @@ class StemApp implements StemTaskApp { TaskCall call, { TaskEnqueueOptions? enqueueOptions, }) async { - await _maybeAutoStart(); return stem.enqueueCall(call, enqueueOptions: enqueueOptions); } @override Future getTaskStatus(String taskId) async { - await _maybeAutoStart(); return stem.getTaskStatus(taskId); } @override Future getGroupStatus(String groupId) async { - await _maybeAutoStart(); return stem.getGroupStatus(groupId); } @@ -183,7 +184,6 @@ class StemApp implements StemTaskApp { TResult Function(Map payload, int version)? decodeVersionedJson, }) async { - await _maybeAutoStart(); return stem.waitForTask( taskId, timeout: timeout, @@ -204,6 +204,7 @@ class StemApp implements StemTaskApp { } /// Starts the managed worker if it is not already running. + @override Future start() async { if (_started) return; final existing = _startFuture; @@ -227,6 +228,7 @@ class StemApp implements StemTaskApp { } /// Shuts down the worker and disposes any managed resources. + @override Future shutdown() async { for (final disposer in _disposers) { await disposer(); @@ -257,7 +259,6 @@ class StemApp implements StemTaskApp { TaskPayloadEncoder resultEncoder = const JsonTaskPayloadEncoder(), TaskPayloadEncoder argsEncoder = const JsonTaskPayloadEncoder(), Iterable additionalEncoders = const [], - bool allowWorkerAutoStart = true, }) async { final effectiveModule = StemModule.combine( module: module, @@ -364,7 +365,6 @@ class StemApp implements StemTaskApp { backend: encodedBackend, stem: stem, worker: worker, - allowWorkerAutoStart: allowWorkerAutoStart, disposers: disposers, ); } @@ -379,7 +379,6 @@ class StemApp implements StemTaskApp { TaskPayloadEncoder resultEncoder = const JsonTaskPayloadEncoder(), TaskPayloadEncoder argsEncoder = const JsonTaskPayloadEncoder(), Iterable additionalEncoders = const [], - bool allowWorkerAutoStart = true, }) { return StemApp.create( module: module, @@ -392,7 +391,6 @@ class StemApp implements StemTaskApp { resultEncoder: resultEncoder, argsEncoder: argsEncoder, additionalEncoders: additionalEncoders, - allowWorkerAutoStart: allowWorkerAutoStart, ); } @@ -424,7 +422,6 @@ class StemApp implements StemTaskApp { TaskPayloadEncoder argsEncoder = const JsonTaskPayloadEncoder(), Iterable additionalEncoders = const [], StemStack? stack, - bool allowWorkerAutoStart = true, }) async { final needsUniqueLockStore = uniqueTasks && @@ -495,7 +492,6 @@ class StemApp implements StemTaskApp { resultEncoder: resultEncoder, argsEncoder: argsEncoder, additionalEncoders: additionalEncoders, - allowWorkerAutoStart: allowWorkerAutoStart, ); // Dispose auto-provisioned lock/revoke stores after worker shutdown and @@ -524,7 +520,6 @@ class StemApp implements StemTaskApp { Iterable modules = const [], Iterable> tasks = const [], StemWorkerConfig workerConfig = const StemWorkerConfig(), - bool allowWorkerAutoStart = true, }) async { final effectiveModule = StemModule.combine(module: module, modules: modules) ?? client.module; @@ -581,7 +576,6 @@ class StemApp implements StemTaskApp { backend: client.backend, stem: client.stem, worker: worker, - allowWorkerAutoStart: allowWorkerAutoStart, disposers: [ () async { await worker.shutdown(); @@ -590,63 +584,3 @@ class StemApp implements StemTaskApp { ); } } - -class _ManagedCanvas extends Canvas { - _ManagedCanvas({ - required super.broker, - required super.backend, - required super.registry, - required super.encoderRegistry, - required Future Function() onBeforeDispatch, - }) : _onBeforeDispatch = onBeforeDispatch; - - final Future Function() _onBeforeDispatch; - - @override - Future send(TaskSignature signature) async { - await _onBeforeDispatch(); - return super.send(signature); - } - - @override - Future> group( - List> signatures, { - String? groupId, - }) async { - await _onBeforeDispatch(); - return super.group(signatures, groupId: groupId); - } - - @override - Future submitBatch( - List> signatures, { - String? batchId, - Duration? ttl, - }) async { - await _onBeforeDispatch(); - return super.submitBatch(signatures, batchId: batchId, ttl: ttl); - } - - @override - Future> chain( - List> signatures, { - void Function(int index, TaskStatus status, T? value)? onStepCompleted, - }) async { - await _onBeforeDispatch(); - return super.chain(signatures, onStepCompleted: onStepCompleted); - } - - @override - Future> chord({ - required List> body, - required TaskSignature callback, - Duration pollInterval = const Duration(milliseconds: 100), - }) async { - await _onBeforeDispatch(); - return super.chord( - body: body, - callback: callback, - pollInterval: pollInterval, - ); - } -} diff --git a/packages/stem/lib/src/bootstrap/stem_client.dart b/packages/stem/lib/src/bootstrap/stem_client.dart index c94e0e83..cb41bb8a 100644 --- a/packages/stem/lib/src/bootstrap/stem_client.dart +++ b/packages/stem/lib/src/bootstrap/stem_client.dart @@ -181,7 +181,7 @@ abstract class StemClient implements TaskResultCaller { } /// Underlying broker used by the client. - Broker get broker; + QueueBroker get broker; /// Result backend used by workers and producers. ResultBackend get backend; @@ -380,7 +380,6 @@ abstract class StemClient implements TaskResultCaller { Duration pollInterval = const Duration(milliseconds: 500), Duration leaseExtension = const Duration(seconds: 30), WorkflowIntrospectionSink? introspectionSink, - bool allowWorkerAutoStart = true, }) { final effectiveModule = StemModule.combine(module: module, modules: modules) ?? this.module; @@ -398,7 +397,6 @@ abstract class StemClient implements TaskResultCaller { pollInterval: pollInterval, leaseExtension: leaseExtension, introspectionSink: introspectionSink, - allowWorkerAutoStart: allowWorkerAutoStart, ); } @@ -408,7 +406,6 @@ abstract class StemClient implements TaskResultCaller { Iterable modules = const [], Iterable> tasks = const [], StemWorkerConfig? workerConfig, - bool allowWorkerAutoStart = true, }) { final effectiveModule = StemModule.combine(module: module, modules: modules) ?? this.module; @@ -417,7 +414,6 @@ abstract class StemClient implements TaskResultCaller { module: effectiveModule, tasks: tasks, workerConfig: workerConfig ?? defaultWorkerConfig, - allowWorkerAutoStart: allowWorkerAutoStart, ); } @@ -517,7 +513,7 @@ class _DefaultStemClient extends StemClient { } @override - final Broker broker; + final QueueBroker broker; @override final ResultBackend backend; diff --git a/packages/stem/lib/src/bootstrap/workflow_app.dart b/packages/stem/lib/src/bootstrap/workflow_app.dart index f0ef4966..660ec768 100644 --- a/packages/stem/lib/src/bootstrap/workflow_app.dart +++ b/packages/stem/lib/src/bootstrap/workflow_app.dart @@ -49,7 +49,6 @@ class StemWorkflowApp required this.runtime, required this.store, required this.eventBus, - required this.allowWorkerAutoStart, required this.ownsStemApp, required Future Function() disposeStore, required Future Function() disposeBus, @@ -68,9 +67,6 @@ class StemWorkflowApp /// Event bus used to deliver workflow events. final EventBus eventBus; - /// Whether shortcut operations may lazily start the managed worker. - final bool allowWorkerAutoStart; - /// Whether this wrapper owns the provided [app] and may shut it down. final bool ownsStemApp; @@ -81,6 +77,7 @@ class StemWorkflowApp Future? _runtimeStartFuture; /// Whether both the runtime and managed worker have been started. + @override bool get isStarted => isRuntimeStarted && isWorkerStarted; /// Whether the workflow runtime has been started. @@ -91,9 +88,6 @@ class StemWorkflowApp Future _ensureReadyForWorkflowStart() async { await startRuntime(); - if (allowWorkerAutoStart) { - await startWorker(); - } } /// Starts the workflow runtime and the underlying Stem worker. @@ -106,6 +100,7 @@ class StemWorkflowApp /// final app = await StemWorkflowApp.inMemory(); /// await app.start(); /// ``` + @override Future start() async { await startRuntime(); await startWorker(); @@ -221,13 +216,14 @@ class StemWorkflowApp /// Schedules a workflow run. /// - /// Lazily starts the runtime on the first invocation so simple examples do - /// not need to call [start] manually. The managed worker is only auto-started - /// when [allowWorkerAutoStart] is `true`. + /// Lazily starts the workflow runtime on the first invocation. The managed + /// worker is never started implicitly; call [start] explicitly when this + /// process should consume work. /// /// Example: /// ```dart /// final app = await StemWorkflowApp.inMemory(); + /// await app.start(); /// final runId = await app.startWorkflow( /// 'exampleWorkflow', /// params: {'key': 'value'}, @@ -663,6 +659,7 @@ class StemWorkflowApp /// await app.shutdown(); /// print('App shutdown complete.'); /// ``` + @override Future shutdown() async { await runtime.dispose(); if (ownsStemApp) { @@ -716,7 +713,6 @@ class StemWorkflowApp TaskPayloadEncoder resultEncoder = const JsonTaskPayloadEncoder(), TaskPayloadEncoder argsEncoder = const JsonTaskPayloadEncoder(), Iterable additionalEncoders = const [], - bool allowWorkerAutoStart = true, bool ownsStemApp = false, }) async { final effectiveModule = @@ -742,13 +738,11 @@ class StemWorkflowApp resultEncoder: resultEncoder, argsEncoder: argsEncoder, additionalEncoders: additionalEncoders, - allowWorkerAutoStart: allowWorkerAutoStart, ); if (stemApp != null) { _validateReusableStemApp( appInstance, resolvedWorkerConfig, - allowWorkerAutoStart: allowWorkerAutoStart, ); } @@ -791,7 +785,6 @@ class StemWorkflowApp runtime: runtime, store: store, eventBus: eventBus, - allowWorkerAutoStart: allowWorkerAutoStart, ownsStemApp: stemApp == null || ownsStemApp, disposeStore: () async => storeFactoryInstance.dispose(store), disposeBus: () async => busFactory.dispose(eventBus), @@ -830,7 +823,6 @@ class StemWorkflowApp TaskPayloadEncoder resultEncoder = const JsonTaskPayloadEncoder(), TaskPayloadEncoder argsEncoder = const JsonTaskPayloadEncoder(), Iterable additionalEncoders = const [], - bool allowWorkerAutoStart = true, }) { return StemWorkflowApp.create( module: module, @@ -854,7 +846,6 @@ class StemWorkflowApp resultEncoder: resultEncoder, argsEncoder: argsEncoder, additionalEncoders: additionalEncoders, - allowWorkerAutoStart: allowWorkerAutoStart, ); } @@ -893,7 +884,6 @@ class StemWorkflowApp TaskPayloadEncoder resultEncoder = const JsonTaskPayloadEncoder(), TaskPayloadEncoder argsEncoder = const JsonTaskPayloadEncoder(), Iterable additionalEncoders = const [], - bool allowWorkerAutoStart = true, }) async { final resolvedWorkerConfig = _resolveWorkflowWorkerConfig( workerConfig, @@ -925,7 +915,6 @@ class StemWorkflowApp resultEncoder: resultEncoder, argsEncoder: argsEncoder, additionalEncoders: additionalEncoders, - allowWorkerAutoStart: allowWorkerAutoStart, ); try { @@ -946,7 +935,6 @@ class StemWorkflowApp leaseExtension: leaseExtension, workflowRegistry: workflowRegistry, introspectionSink: introspectionSink, - allowWorkerAutoStart: allowWorkerAutoStart, ownsStemApp: true, ); } on Object catch (error, stackTrace) { @@ -983,7 +971,6 @@ class StemWorkflowApp Duration pollInterval = const Duration(milliseconds: 500), Duration leaseExtension = const Duration(seconds: 30), WorkflowIntrospectionSink? introspectionSink, - bool allowWorkerAutoStart = true, }) async { final effectiveModule = StemModule.combine(module: module, modules: modules) ?? client.module; @@ -997,7 +984,6 @@ class StemWorkflowApp final appInstance = await StemApp.fromClient( client, workerConfig: resolvedWorkerConfig, - allowWorkerAutoStart: allowWorkerAutoStart, ); return StemWorkflowApp.create( module: effectiveModule, @@ -1014,7 +1000,6 @@ class StemWorkflowApp leaseExtension: leaseExtension, workflowRegistry: client.workflowRegistry, introspectionSink: introspectionSink, - allowWorkerAutoStart: allowWorkerAutoStart, ownsStemApp: true, ); } @@ -1043,7 +1028,6 @@ extension StemAppWorkflowExtension on StemApp { Duration leaseExtension = const Duration(seconds: 30), WorkflowRegistry? workflowRegistry, WorkflowIntrospectionSink? introspectionSink, - bool allowWorkerAutoStart = true, }) { return StemWorkflowApp.create( module: @@ -1062,25 +1046,14 @@ extension StemAppWorkflowExtension on StemApp { leaseExtension: leaseExtension, workflowRegistry: workflowRegistry, introspectionSink: introspectionSink, - allowWorkerAutoStart: allowWorkerAutoStart, ); } } void _validateReusableStemApp( StemApp app, - StemWorkerConfig workerConfig, { - required bool allowWorkerAutoStart, -}) { - if (app.allowWorkerAutoStart != allowWorkerAutoStart) { - throw StateError( - 'StemWorkflowApp.create(stemApp: ...) requires the reused StemApp ' - 'to use the same allowWorkerAutoStart setting. Create the StemApp with ' - 'allowWorkerAutoStart: $allowWorkerAutoStart or omit stemApp so the ' - 'workflow app can create a matching shortcut wrapper.', - ); - } - + StemWorkerConfig workerConfig, +) { final requiredQueues = workerConfig.subscription?.resolveQueues( workerConfig.queue, diff --git a/packages/stem/lib/src/canvas/canvas.dart b/packages/stem/lib/src/canvas/canvas.dart index 6ee5c1e4..8c1a7ce4 100644 --- a/packages/stem/lib/src/canvas/canvas.dart +++ b/packages/stem/lib/src/canvas/canvas.dart @@ -2,12 +2,14 @@ import 'dart:async'; import 'package:stem/src/backend/encoding_result_backend.dart'; import 'package:stem/src/core/chord_metadata.dart'; +import 'package:stem/src/core/chord_policy.dart'; import 'package:stem/src/core/clock.dart'; import 'package:stem/src/core/contracts.dart'; import 'package:stem/src/core/encoder_keys.dart'; import 'package:stem/src/core/envelope.dart'; import 'package:stem/src/core/task_payload_encoder.dart'; import 'package:stem/src/core/task_result.dart'; +import 'package:stem/src/observability/tracing.dart'; import 'package:uuid/uuid.dart'; /// Describes a task to schedule along with optional decoder metadata. @@ -104,6 +106,73 @@ class TaskChainResult { bool get isCompleted => finalStatus?.state == TaskState.succeeded; } +/// A fluent, heterogeneous typed Canvas chain. +/// +/// Each call to [then] is type-checked against the previous task's result: +/// +/// ```dart +/// final pipeline = canvas +/// .typedChain(download, request) +/// .then(resize) +/// .then(upload); +/// final result = await pipeline.run(); +/// ``` +/// +/// The existing [Canvas.chain] API remains available for raw signatures and +/// migration compatibility. This builder decodes each terminal result before +/// encoding it as the next task's typed arguments. +class TypedCanvasChain { + const TypedCanvasChain._(this._canvas, this._steps); + + final Canvas _canvas; + final List<_TypedCanvasChainStep> _steps; + + /// Appends a task whose arguments accept the current result type. + TypedCanvasChain then( + TaskDefinition definition, { + Map headers = const {}, + TaskOptions? options, + DateTime? notBefore, + Map? meta, + TNext Function(Object? payload)? decode, + }) { + return TypedCanvasChain._( + _canvas, + [ + ..._steps, + _TypedCanvasChainStep( + build: (input) => definition + .toSignature( + input as TCurrent, + headers: headers, + options: options, + notBefore: notBefore, + meta: meta, + decode: decode, + ) + .call(), + decode: decode ?? definition.decode, + ), + ], + ); + } + + /// Runs the chain and returns the final task's typed result. + Future> run({ + void Function(int index, TaskStatus status, Object? value)? onStepCompleted, + }) => _canvas._runTypedChain( + _steps, + onStepCompleted: onStepCompleted, + ); +} + +class _TypedCanvasChainStep { + const _TypedCanvasChainStep({required this.build, required this.decode}); + + final Envelope Function(Object? previousValue) build; + final Object? Function(Object? payload) decode; +} + /// Handle returned by [Canvas.group] providing a stream of typed results. class GroupDispatch { /// Creates a group dispatch handle. @@ -140,6 +209,8 @@ class ChordResult { required this.chordId, required this.callbackTaskId, required this.values, + this.policy = const ChordPolicy.allOrFail(), + this.failures = const [], }); /// Identifier for the chord group. @@ -150,6 +221,12 @@ class ChordResult { /// Results from the chord body tasks. final List values; + + /// Policy used to decide whether the callback could run. + final ChordPolicy policy; + + /// Terminal body statuses that were not successful. + final List failures; } /// Lifecycle states for first-class batch submissions. @@ -286,7 +363,7 @@ class Canvas { } /// The message broker used to publish task envelopes. - final Broker broker; + final QueueBroker broker; /// Registry of payload encoders used by the canvas. final TaskPayloadEncoderRegistry payloadEncoders; @@ -337,27 +414,46 @@ class Canvas { ); } final taskIds = []; - for (final signature in signatures) { - final raw = signature(); - final grouped = raw.copyWith( - headers: {...raw.headers, 'stem-group-id': id}, - meta: {...raw.meta, 'groupId': id}, - ); - final (envelope, resultEncoder) = _prepareEnvelope(grouped); - taskIds.add(envelope.id); - await broker.publish(envelope); - final queuedMeta = _withResultEncoderMeta({ - ...envelope.meta, - 'queue': envelope.queue, - 'groupId': id, - }, resultEncoder); - await backend.set( - envelope.id, - TaskState.queued, - attempt: envelope.attempt, - meta: queuedMeta, - ); - } + await StemTracer.instance.trace( + 'stem.canvas.group', + () async { + final compositionContext = StemTracer.instance.ambientContextOrNull(); + for (final signature in signatures) { + final raw = signature(); + final traceHeaders = Map.from(raw.headers); + StemTracer.instance.injectTraceContext( + traceHeaders, + context: compositionContext, + ); + StemTracer.instance.injectTraceLink( + traceHeaders, + context: compositionContext, + ); + final grouped = raw.copyWith( + headers: {...traceHeaders, 'stem-group-id': id}, + meta: {...raw.meta, 'groupId': id}, + ); + final (envelope, resultEncoder) = _prepareEnvelope(grouped); + taskIds.add(envelope.id); + await broker.publish(envelope); + final queuedMeta = _withResultEncoderMeta({ + ...envelope.meta, + 'queue': envelope.queue, + 'groupId': id, + }, resultEncoder); + await backend.set( + envelope.id, + TaskState.queued, + attempt: envelope.attempt, + meta: queuedMeta, + ); + } + }, + attributes: { + 'stem.canvas.group.id': id, + 'stem.canvas.group.task_count': signatures.length, + }, + ); final controller = StreamController>.broadcast(); if (taskIds.isEmpty) { @@ -497,6 +593,39 @@ class Canvas { return _buildBatchStatus(status); } + /// Starts a fluent heterogeneous typed chain. + /// + /// Unlike [chain], subsequent tasks are allowed to have different argument + /// and result types. The compiler checks each transition when + /// [TypedCanvasChain.then] is called. + TypedCanvasChain typedChain( + TaskDefinition definition, + TArgs args, { + Map headers = const {}, + TaskOptions? options, + DateTime? notBefore, + Map? meta, + TResult Function(Object? payload)? decode, + }) { + final signature = definition.toSignature( + args, + headers: headers, + options: options, + notBefore: notBefore, + meta: meta, + decode: decode, + ); + return TypedCanvasChain._( + this, + [ + _TypedCanvasChainStep( + build: (_) => signature(), + decode: signature.decodePayload, + ), + ], + ); + } + Future _withBatchSubmissionLock( String batchId, Future Function() operation, @@ -606,6 +735,107 @@ class Canvas { return completer.future; } + Future> _runTypedChain( + List<_TypedCanvasChainStep> steps, { + void Function(int index, TaskStatus status, Object? value)? onStepCompleted, + }) async { + if (steps.isEmpty) { + throw ArgumentError('Typed chain requires at least one task'); + } + + final chainId = _generateId('chain'); + final completer = Completer>(); + + Future runStep(int index, Object? previousValue) async { + final step = steps[index]; + final raw = step.build(previousValue); + final meta = { + ...raw.meta, + 'chainId': chainId, + 'chainIndex': index, + 'queue': raw.queue, + 'chainPrevResult': ?previousValue, + }; + final headers = { + ...raw.headers, + 'stem-chain-id': chainId, + 'stem-chain-index': '$index', + }; + final envelope = raw.copyWith(headers: headers, meta: meta); + final (preparedEnvelope, resultEncoder) = _prepareEnvelope(envelope); + + late StreamSubscription subscription; + subscription = backend + .watch(preparedEnvelope.id) + .listen( + (status) async { + if (status.state == TaskState.succeeded) { + try { + final decoded = step.decode(status.payload); + onStepCompleted?.call(index, status, decoded); + await subscription.cancel(); + if (index + 1 < steps.length) { + await runStep(index + 1, decoded); + } else if (!completer.isCompleted) { + completer.complete( + TaskChainResult( + chainId: chainId, + finalTaskId: preparedEnvelope.id, + finalStatus: status, + value: decoded as TFinal?, + ), + ); + } + } on Object catch (error, stackTrace) { + await subscription.cancel(); + if (!completer.isCompleted) { + completer.completeError(error, stackTrace); + } + } + } else if (status.state == TaskState.failed || + status.state == TaskState.cancelled) { + await subscription.cancel(); + if (!completer.isCompleted) { + completer.completeError( + StateError('Chain $chainId failed at step $index'), + ); + } + } + }, + onError: (Object error, StackTrace stackTrace) { + if (!completer.isCompleted) { + completer.completeError(error, stackTrace); + } + }, + ); + + await broker.publish(preparedEnvelope); + final queuedMeta = _withResultEncoderMeta( + preparedEnvelope.meta, + resultEncoder, + ); + await backend.set( + preparedEnvelope.id, + TaskState.queued, + attempt: preparedEnvelope.attempt, + meta: queuedMeta, + ); + } + + unawaited( + () async { + try { + await runStep(0, null); + } on Object catch (error, stackTrace) { + if (!completer.isCompleted) { + completer.completeError(error, stackTrace); + } + } + }(), + ); + return completer.future; + } + /// Coordinates a chord: a group of tasks followed by a callback. /// /// Publishes [body] as a group and waits until every task in the group @@ -617,20 +847,37 @@ class Canvas { Future> chord({ required List> body, required TaskSignature callback, + ChordPolicy policy = const ChordPolicy.allOrFail(), Duration pollInterval = const Duration(milliseconds: 100), }) async { - final handle = await _startChord(body: body, callback: callback); + final handle = await _startChord( + body: body, + callback: callback, + policy: policy, + ); final values = await _awaitChordValues( handle.chordId, body, handle.bodyTaskIds, + policy, pollInterval, ); final callbackTaskId = await handle.callbackFuture; + final groupStatus = await backend.getGroup(handle.chordId); return ChordResult( chordId: handle.chordId, callbackTaskId: callbackTaskId, values: values, + policy: policy, + failures: groupStatus == null + ? const [] + : groupStatus.results.values + .where( + (status) => + status.state == TaskState.failed || + status.state == TaskState.cancelled, + ) + .toList(growable: false), ); } @@ -638,6 +885,7 @@ class Canvas { Future _monitorChord( String chordId, String callbackId, + ChordPolicy policy, Completer completer, ) async { while (true) { @@ -646,12 +894,11 @@ class Canvas { await Future.delayed(const Duration(milliseconds: 100)); continue; } - final hasFailure = status.results.values.any( - (s) => s.state == TaskState.failed || s.state == TaskState.cancelled, - ); - if (hasFailure) { + if (policy.shouldFail(status)) { if (!completer.isCompleted) { - completer.completeError('Chord $chordId failed due to task failure'); + completer.completeError( + StateError('Chord $chordId failed under ${policy.kind.name}'), + ); } return; } @@ -674,6 +921,7 @@ class Canvas { Future<_ChordHandle> _startChord({ required List> body, required TaskSignature callback, + required ChordPolicy policy, }) async { if (body.isEmpty) { throw ArgumentError('Chord body must have at least one task'); @@ -686,7 +934,10 @@ class Canvas { GroupDescriptor( id: chordId, expected: body.length, - meta: {ChordMetadata.callbackEnvelope: callbackEnvelope.toJson()}, + meta: { + ChordMetadata.callbackEnvelope: callbackEnvelope.toJson(), + ChordMetadata.policy: policy.toJson(), + }, ), ); final bodyDispatch = await group(body, groupId: chordId); @@ -694,14 +945,16 @@ class Canvas { await bodyDispatch.dispose(); final completer = Completer(); unawaited( - _monitorChord(chordId, callbackEnvelope.id, completer).catchError(( - Object error, - StackTrace stack, - ) { - if (!completer.isCompleted) { - completer.completeError(error, stack); - } - }), + _monitorChord(chordId, callbackEnvelope.id, policy, completer).catchError( + ( + Object error, + StackTrace stack, + ) { + if (!completer.isCompleted) { + completer.completeError(error, stack); + } + }, + ), ); return _ChordHandle( chordId: chordId, @@ -714,6 +967,7 @@ class Canvas { String chordId, List> body, List taskIds, + ChordPolicy policy, Duration pollInterval, ) async { while (true) { @@ -722,16 +976,9 @@ class Canvas { await Future.delayed(pollInterval); continue; } - final failures = status.results.entries - .where( - (entry) => - entry.value.state == TaskState.failed || - entry.value.state == TaskState.cancelled, - ) - .toList(); - if (failures.isNotEmpty) { + if (policy.shouldFail(status)) { throw StateError( - 'Chord $chordId failed due to task ${failures.first.key}', + 'Chord $chordId failed under ${policy.kind.name}', ); } final values = []; diff --git a/packages/stem/lib/src/control/control_messages.dart b/packages/stem/lib/src/control/control_messages.dart index 80b33eef..bc9b31e2 100644 --- a/packages/stem/lib/src/control/control_messages.dart +++ b/packages/stem/lib/src/control/control_messages.dart @@ -282,7 +282,7 @@ extension ControlCommandEnvelope on ControlCommandMessage { name: ControlEnvelopeTypes.command, queue: queue, args: toMap(), - headers: {'stem-control': '1', if (headers != null) ...headers}, + headers: {'stem-control': '1', ...?headers}, ); } } @@ -295,7 +295,7 @@ extension ControlReplyEnvelope on ControlReplyMessage { name: ControlEnvelopeTypes.reply, queue: queue, args: toMap(), - headers: {'stem-control-reply': '1', if (headers != null) ...headers}, + headers: {'stem-control-reply': '1', ...?headers}, ); } } diff --git a/packages/stem/lib/src/core/chord_metadata.dart b/packages/stem/lib/src/core/chord_metadata.dart index ab2e0017..b19ca060 100644 --- a/packages/stem/lib/src/core/chord_metadata.dart +++ b/packages/stem/lib/src/core/chord_metadata.dart @@ -1,3 +1,5 @@ +import 'package:stem/src/core/chord_policy.dart'; + /// Metadata keys used for chord coordination. class ChordMetadata { const ChordMetadata._(); @@ -10,4 +12,10 @@ class ChordMetadata { /// Identifier of the callback task associated with the chord. static const String callbackTaskId = 'stem.chord.callbackTaskId'; + + /// Serialized [ChordPolicy] stored on the group descriptor. + static const String policy = 'stem.chord.policy'; + + /// Failure summaries passed to callbacks for non-failing policies. + static const String failures = 'stem.chord.failures'; } diff --git a/packages/stem/lib/src/core/chord_policy.dart b/packages/stem/lib/src/core/chord_policy.dart new file mode 100644 index 00000000..c35641ed --- /dev/null +++ b/packages/stem/lib/src/core/chord_policy.dart @@ -0,0 +1,104 @@ +import 'package:meta/meta.dart'; +import 'package:stem/src/core/contracts.dart'; + +/// Determines when a chord callback may run. +enum ChordPolicyKind { + /// Require every body task to succeed. + allOrFail, + + /// Wait for every body task to reach a terminal state and collect failures. + collectTerminalResults, + + /// Wait for every body task and require at least [ChordPolicy.minSuccessful] + /// successful results. + allowPartial, +} + +/// Failure policy for a Canvas chord. +@immutable +class ChordPolicy { + /// Requires every body task to succeed. This is the default policy. + const ChordPolicy.allOrFail() + : kind = ChordPolicyKind.allOrFail, + minSuccessful = null; + + /// Runs the callback after all body tasks are terminal, including failures. + const ChordPolicy.collectTerminalResults() + : kind = ChordPolicyKind.collectTerminalResults, + minSuccessful = null; + + /// Runs the callback after all body tasks are terminal when at least + /// [minSuccessful] tasks succeeded. + const ChordPolicy.allowPartial({required int minSuccessful}) + : assert(minSuccessful > 0, 'minSuccessful must be positive'), + kind = ChordPolicyKind.allowPartial, + minSuccessful = minSuccessful; + + /// Reconstructs a policy persisted in group metadata. + factory ChordPolicy.fromJson(Object? value) { + if (value is! Map) return const ChordPolicy.allOrFail(); + final kind = switch (value['kind']?.toString()) { + 'collectTerminalResults' => ChordPolicyKind.collectTerminalResults, + 'allowPartial' => ChordPolicyKind.allowPartial, + _ => ChordPolicyKind.allOrFail, + }; + if (kind == ChordPolicyKind.allowPartial) { + final minimum = (value['minSuccessful'] as num?)?.toInt(); + if (minimum != null && minimum > 0) { + return ChordPolicy.allowPartial(minSuccessful: minimum); + } + return const ChordPolicy.allOrFail(); + } + return switch (kind) { + ChordPolicyKind.collectTerminalResults => + const ChordPolicy.collectTerminalResults(), + ChordPolicyKind.allOrFail => const ChordPolicy.allOrFail(), + ChordPolicyKind.allowPartial => const ChordPolicy.allOrFail(), + }; + } + + /// The policy behavior. + final ChordPolicyKind kind; + + /// Minimum successful body tasks for [ChordPolicyKind.allowPartial]. + final int? minSuccessful; + + /// Serializes this policy for durable group metadata. + Map toJson() => { + 'kind': kind.name, + if (minSuccessful != null) 'minSuccessful': minSuccessful, + }; + + /// Whether this status is eligible to dispatch the callback. + bool shouldDispatch(GroupStatus status) { + if (!status.isComplete) return false; + final successful = status.results.values + .where((result) => result.state == TaskState.succeeded) + .length; + return switch (kind) { + ChordPolicyKind.allOrFail => successful == status.expected, + ChordPolicyKind.collectTerminalResults => true, + ChordPolicyKind.allowPartial => successful >= minSuccessful!, + }; + } + + /// Whether the chord has reached a terminal failure under this policy. + bool shouldFail(GroupStatus status) { + final hasFailure = status.results.values.any( + (result) => + result.state == TaskState.failed || + result.state == TaskState.cancelled, + ); + if (kind == ChordPolicyKind.allOrFail && hasFailure) return true; + return status.isComplete && !shouldDispatch(status); + } + + @override + bool operator ==(Object other) => + other is ChordPolicy && + other.kind == kind && + other.minSuccessful == minSuccessful; + + @override + int get hashCode => Object.hash(kind, minSuccessful); +} diff --git a/packages/stem/lib/src/core/contracts.dart b/packages/stem/lib/src/core/contracts.dart index f50c69ba..0e525a4f 100644 --- a/packages/stem/lib/src/core/contracts.dart +++ b/packages/stem/lib/src/core/contracts.dart @@ -33,6 +33,7 @@ library; import 'dart:async'; import 'dart:collection'; +import 'package:meta/meta.dart'; import 'package:stem/src/core/envelope.dart'; import 'package:stem/src/core/payload_codec.dart'; import 'package:stem/src/core/payload_map.dart'; @@ -92,9 +93,67 @@ class RoutingSubscription { } } -/// Abstract broker interface implemented by queue adapters (Redis, SQS, etc). -/// Since: 0.1.0 -abstract class Broker { +/// Delivery guarantee advertised by a broker adapter. +enum BrokerDeliveryGuarantee { + /// A delivery may be observed more than once after crashes or lease loss. + atLeastOnce, + + /// A delivery is removed before handler execution and may be lost on crash. + atMostOnce, + + /// The adapter does not declare a delivery guarantee. + unknown, +} + +/// Runtime-declared capabilities of a broker adapter. +/// +/// This is intentionally additive to the [Broker] contract. Existing +/// adapters can continue implementing the legacy getters while callers use a +/// single snapshot when deciding whether to expose optional operations. +class BrokerCapabilities { + /// Creates a broker capability snapshot. + const BrokerCapabilities({ + required this.supportsDelayedDelivery, + required this.supportsPriorityOrdering, + this.deliveryGuarantee = BrokerDeliveryGuarantee.unknown, + this.supportsBroadcastFanout = false, + this.supportsQueueInspection = false, + this.supportsLeaseExtension = false, + this.supportsDeadLettering = false, + this.supportsDeadLetterReplay = false, + }); + + /// Whether the adapter supports broker-native delayed delivery. + final bool supportsDelayedDelivery; + + /// Delivery guarantee callers can rely on during worker crashes. + final BrokerDeliveryGuarantee deliveryGuarantee; + + /// Whether priority ordering is part of the adapter's delivery contract. + final bool supportsPriorityOrdering; + + /// Whether one published broadcast message is delivered to each subscriber. + final bool supportsBroadcastFanout; + + /// Whether pending and in-flight queue counts are available. + final bool supportsQueueInspection; + + /// Whether an active delivery lease can be extended. + final bool supportsLeaseExtension; + + /// Whether failed deliveries can be retained in a dead-letter store. + final bool supportsDeadLettering; + + /// Whether dead-letter entries can be replayed into an active queue. + final bool supportsDeadLetterReplay; +} + +/// Core queue operations required to publish and consume task deliveries. +/// +/// This is the narrow transport surface for new adapter integrations. [Broker] +/// remains the compatibility facade while optional operational capabilities are +/// migrated to the interfaces below. +abstract interface class QueueBroker { /// Publishes the given [envelope] using [routing] metadata when provided. /// /// When [routing] is omitted, brokers MUST fall back to [Envelope.queue] and @@ -121,6 +180,65 @@ abstract class Broker { /// If [requeue] is true, the message is requeued for retry. Future nack(Delivery delivery, {bool requeue = true}); + /// Releases resources held by the transport. + Future close(); +} + +/// Optional broker capability for extending active delivery leases. +// ignore: one_member_abstracts +abstract interface class LeaseBroker { + /// Extends the lease for the [delivery] by [by]. + Future extendLease(Delivery delivery, Duration by); +} + +/// Optional broker capability for queue-depth inspection. +abstract interface class InspectableBroker { + /// Returns the number of pending messages for [queue], if supported. + Future pendingCount(String queue); + + /// Returns the number of in-flight messages for [queue], if supported. + Future inflightCount(String queue); +} + +/// Optional broker capability for dead-letter inspection and replay. +abstract interface class DeadLetterBroker { + /// Moves a delivery to the dead-letter store. + Future deadLetter( + Delivery delivery, { + String? reason, + Map? meta, + }); + + /// Lists dead letter queue entries for [queue]. + Future listDeadLetters( + String queue, { + int limit = 50, + int offset = 0, + }); + + /// Retrieves a single dead letter entry by envelope [id]. + Future getDeadLetter(String queue, String id); + + /// Replays dead letter entries back onto the active queue. + Future replayDeadLetters( + String queue, { + int limit = 50, + DateTime? since, + Duration? delay, + bool dryRun = false, + }); + + /// Removes dead letter entries from [queue]. + Future purgeDeadLetters(String queue, {DateTime? since, int? limit}); +} + +/// Abstract broker compatibility facade implemented by queue adapters. +/// +/// New adapters should implement [QueueBroker] plus only the optional +/// capability interfaces they actually support. Existing adapters may +/// continue implementing this broader contract during the migration period. +/// Since: 0.1.0 +abstract class Broker implements QueueBroker { /// Sends the [delivery] to the dead letter queue. /// /// [reason] provides the reason for dead lettering, and [meta] additional @@ -129,13 +247,15 @@ abstract class Broker { Delivery delivery, { String? reason, Map? meta, - }); + }) => throw UnsupportedError('Dead-lettering is not supported.'); /// Removes all messages from the [queue]. - Future purge(String queue); + Future purge(String queue) => + throw UnsupportedError('Queue purging is not supported.'); /// Extends the lease for the [delivery] by the [by] duration. - Future extendLease(Delivery delivery, Duration by); + Future extendLease(Delivery delivery, Duration by) => + throw UnsupportedError('Lease extension is not supported.'); /// Returns the number of pending messages for [queue], if supported. Future pendingCount(String queue) async => null; @@ -156,11 +276,12 @@ abstract class Broker { String queue, { int limit = 50, int offset = 0, - }); + }) => throw UnsupportedError('Dead-letter inspection is not supported.'); /// Retrieves a single dead letter entry by envelope [id], or `null` if not /// found. - Future getDeadLetter(String queue, String id); + Future getDeadLetter(String queue, String id) => + throw UnsupportedError('Dead-letter inspection is not supported.'); /// Replays at most [limit] dead letter entries back onto the active queue. /// @@ -175,19 +296,216 @@ abstract class Broker { DateTime? since, Duration? delay, bool dryRun = false, - }); + }) => throw UnsupportedError('Dead-letter replay is not supported.'); /// Removes dead letter entries from [queue]. /// /// When [since] is provided, only entries with `deadAt` greater than or equal /// to the timestamp must be removed. When [limit] is set, at most that many /// entries are purged. Returns the number of entries removed. - Future purgeDeadLetters(String queue, {DateTime? since, int? limit}); + Future purgeDeadLetters( + String queue, { + DateTime? since, + int? limit, + }) => throw UnsupportedError('Dead-letter purging is not supported.'); /// Releases any resources held by the broker. + @override Future close() async {} } +/// Optional provider interface for adapters with capability declarations that +/// are more precise than the legacy [Broker] getters. +abstract interface class BrokerCapabilitiesProvider { + /// Returns the adapter's optional-operation capabilities. + BrokerCapabilities get capabilities; +} + +/// Resolves capabilities without making [Broker] implementations add a new +/// required member. External adapters therefore remain source-compatible and +/// still receive a useful snapshot from the legacy getters. +extension BrokerCapabilitiesExtension on Broker { + /// Returns the adapter's optional-operation capabilities. + BrokerCapabilities get capabilities { + if (this is BrokerCapabilitiesProvider) { + return (this as BrokerCapabilitiesProvider).capabilities; + } + return BrokerCapabilities( + supportsDelayedDelivery: supportsDelayed, + supportsPriorityOrdering: supportsPriority, + ); + } +} + +/// Resolves capabilities for code that accepts the narrow [QueueBroker] type. +/// +/// Queue-only adapters without a capability provider report only the +/// operations guaranteed by [QueueBroker]. This avoids making optional +/// operations appear available merely because the caller received a transport +/// through the narrow interface. +extension QueueBrokerCapabilitiesExtension on QueueBroker { + /// Returns the adapter's optional-operation capabilities. + BrokerCapabilities get capabilities { + if (this is BrokerCapabilitiesProvider) { + return (this as BrokerCapabilitiesProvider).capabilities; + } + if (this is Broker) return (this as Broker).capabilities; + return const BrokerCapabilities( + supportsDelayedDelivery: false, + supportsPriorityOrdering: false, + ); + } +} + +/// Optional operations for code that accepts the narrow [QueueBroker] type. +/// +/// The extension first uses an explicit capability interface, then falls back +/// to the legacy [Broker] facade. This preserves existing adapters while +/// allowing new queue-only adapters to omit unsupported operations entirely. +extension QueueBrokerOptionalOperations on QueueBroker { + /// Extends a delivery lease when the adapter supports leases. + Future extendLease(Delivery delivery, Duration by) { + if (this is LeaseBroker) { + return (this as LeaseBroker).extendLease(delivery, by); + } + if (this is Broker) { + return (this as Broker).extendLease(delivery, by); + } + throw UnsupportedError('Lease extension is not supported.'); + } + + /// Sends a delivery to the dead-letter store when supported. + Future deadLetter( + Delivery delivery, { + String? reason, + Map? meta, + }) { + if (this is DeadLetterBroker) { + return (this as DeadLetterBroker).deadLetter( + delivery, + reason: reason, + meta: meta, + ); + } + if (this is Broker) { + return (this as Broker).deadLetter( + delivery, + reason: reason, + meta: meta, + ); + } + throw UnsupportedError('Dead-lettering is not supported.'); + } + + /// Removes all messages from a queue when supported. + Future purge(String queue) { + if (this is Broker) return (this as Broker).purge(queue); + throw UnsupportedError('Queue purging is not supported.'); + } + + /// Returns a pending queue count when supported. + Future pendingCount(String queue) { + if (this is InspectableBroker) { + return (this as InspectableBroker).pendingCount(queue); + } + if (this is Broker) return (this as Broker).pendingCount(queue); + return Future.value(); + } + + /// Returns an in-flight queue count when supported. + Future inflightCount(String queue) { + if (this is InspectableBroker) { + return (this as InspectableBroker).inflightCount(queue); + } + if (this is Broker) return (this as Broker).inflightCount(queue); + return Future.value(); + } + + /// Lists dead-letter entries when the adapter supports inspection. + Future listDeadLetters( + String queue, { + int limit = 50, + int offset = 0, + }) { + if (this is DeadLetterBroker) { + return (this as DeadLetterBroker).listDeadLetters( + queue, + limit: limit, + offset: offset, + ); + } + if (this is Broker) { + return (this as Broker).listDeadLetters( + queue, + limit: limit, + offset: offset, + ); + } + throw UnsupportedError('Dead-letter inspection is not supported.'); + } + + /// Retrieves one dead-letter entry when supported. + Future getDeadLetter(String queue, String id) { + if (this is DeadLetterBroker) { + return (this as DeadLetterBroker).getDeadLetter(queue, id); + } + if (this is Broker) return (this as Broker).getDeadLetter(queue, id); + throw UnsupportedError('Dead-letter inspection is not supported.'); + } + + /// Replays dead-letter entries when supported. + Future replayDeadLetters( + String queue, { + int limit = 50, + DateTime? since, + Duration? delay, + bool dryRun = false, + }) { + if (this is DeadLetterBroker) { + return (this as DeadLetterBroker).replayDeadLetters( + queue, + limit: limit, + since: since, + delay: delay, + dryRun: dryRun, + ); + } + if (this is Broker) { + return (this as Broker).replayDeadLetters( + queue, + limit: limit, + since: since, + delay: delay, + dryRun: dryRun, + ); + } + throw UnsupportedError('Dead-letter replay is not supported.'); + } + + /// Purges dead-letter entries when supported. + Future purgeDeadLetters( + String queue, { + DateTime? since, + int? limit, + }) { + if (this is DeadLetterBroker) { + return (this as DeadLetterBroker).purgeDeadLetters( + queue, + since: since, + limit: limit, + ); + } + if (this is Broker) { + return (this as Broker).purgeDeadLetters( + queue, + since: since, + limit: limit, + ); + } + throw UnsupportedError('Dead-letter purging is not supported.'); + } +} + /// Logical task status across enqueue, running, success, failure states. enum TaskState { /// Task is queued and awaiting execution. @@ -904,6 +1222,34 @@ abstract class ResultBackend { Future close() async {} } +/// Optional atomic terminal-state arbitration for result backends. +/// +/// Workers can receive the same delivery more than once across processes. A +/// backend implementing this capability must persist a [TaskStatus] only when +/// the current record is absent or non-terminal, and must do that check and +/// write atomically. It returns `true` for the writer that won the terminal +/// state and `false` when another terminal state was already persisted. +/// +/// Implementations should normally be used after the producer has created the +/// task's initial queued record. A backend may return `false` when a task +/// record is absent if its storage cannot atomically create-and-arbitrate a +/// missing row. +/// +/// This is deliberately separate from [ResultBackend] so existing custom +/// backends remain source compatible. Workers fall back to [ResultBackend.set] +/// for backends that do not advertise this capability; that fallback is not a +/// cross-process first-writer-wins guarantee. +abstract interface class AtomicTerminalResultBackend { + /// Whether this backend provides a cross-process atomic terminal write. + bool get supportsAtomicTerminalWrites; + + /// Attempts to persist [status] as the task's terminal state. + Future setTerminalIfAbsent( + TaskStatus status, { + Duration? ttl, + }); +} + /// Schedule entry persisted by a Beat-like scheduler. class ScheduleEntry { /// Creates a schedule entry for a recurring task. @@ -1332,8 +1678,8 @@ class TaskOptions { maxRetries: (json['maxRetries'] as num?)?.toInt() ?? 0, softTimeLimit: _durationFromJson(json['softTimeLimitMs']), hardTimeLimit: _durationFromJson(json['hardTimeLimitMs']), - rateLimit: json['rateLimit'] as String?, - groupRateLimit: json['groupRateLimit'] as String?, + rateLimit: RateLimit.parse(json['rateLimit']), + groupRateLimit: RateLimit.parse(json['groupRateLimit']), groupRateKey: json['groupRateKey'] as String?, groupRateKeyHeader: json['groupRateKeyHeader'] as String? ?? 'tenant', groupRateLimiterFailureMode: failureMode, @@ -1359,11 +1705,11 @@ class TaskOptions { final Duration? hardTimeLimit; /// The rate limit for tasks with these options. - final String? rateLimit; + final RateLimit? rateLimit; /// Group-scoped rate limit shared by tasks that resolve to /// the same group key. - final String? groupRateLimit; + final RateLimit? groupRateLimit; /// Optional static group key used for group-scoped rate limiting. final String? groupRateKey; @@ -1398,8 +1744,8 @@ class TaskOptions { int? maxRetries, Duration? softTimeLimit, Duration? hardTimeLimit, - String? rateLimit, - String? groupRateLimit, + RateLimit? rateLimit, + RateLimit? groupRateLimit, String? groupRateKey, String? groupRateKeyHeader, RateLimiterFailureMode? groupRateLimiterFailureMode, @@ -1436,8 +1782,8 @@ class TaskOptions { 'maxRetries': maxRetries, 'softTimeLimitMs': softTimeLimit?.inMilliseconds, 'hardTimeLimitMs': hardTimeLimit?.inMilliseconds, - 'rateLimit': rateLimit, - 'groupRateLimit': groupRateLimit, + 'rateLimit': rateLimit?.toString(), + 'groupRateLimit': groupRateLimit?.toString(), 'groupRateKey': groupRateKey, 'groupRateKeyHeader': groupRateKeyHeader, 'groupRateLimiterFailureMode': groupRateLimiterFailureMode.name, @@ -2252,6 +2598,9 @@ abstract interface class TaskExecutionContext /// Metadata for the task invocation. Map get meta; + /// Cooperative cancellation state for this invocation. + TaskCancellationToken get cancellation; + /// Notify the worker that the task is still running. void heartbeat(); @@ -2283,6 +2632,54 @@ abstract interface class TaskExecutionContext }); } +/// Cooperative cancellation state exposed to task code. +/// +/// Cancellation is advisory for inline handlers: task code must call +/// [throwIfCancelled] at safe points. Isolate-backed handlers can still be +/// terminated by the worker's hard timeout or hard shutdown path. +class TaskCancellationToken { + /// Creates a token whose state is optionally resolved by [isCancelled]. + TaskCancellationToken({bool Function()? isCancelled}) + : _isCancelled = isCancelled, + _state = _CancellationState(); + + /// Creates a token that is never cancelled. + const TaskCancellationToken.none() : _isCancelled = null, _state = null; + + final bool Function()? _isCancelled; + final _CancellationState? _state; + + /// Whether cancellation has been requested. + bool get isCancellationRequested => + (_state?.cancelled ?? false) || (_isCancelled?.call() ?? false); + + /// Marks this token as cancelled. + void cancel() { + final state = _state; + if (state != null) state.cancelled = true; + } + + /// Throws when cancellation has been requested. + void throwIfCancelled() { + if (isCancellationRequested) { + throw const TaskCancellationException(); + } + } +} + +class _CancellationState { + bool cancelled = false; +} + +/// Thrown when a task cooperatively observes cancellation. +class TaskCancellationException implements Exception { + /// Creates a cancellation exception. + const TaskCancellationException(); + + @override + String toString() => 'Task execution was cancelled'; +} + /// Shared task-progress helpers for execution contexts. extension TaskExecutionContextProgressX on TaskExecutionContext { /// Report progress with a JSON-serializable DTO payload. @@ -2334,6 +2731,7 @@ class TaskContext implements TaskExecutionContext { Map? data, }) progress, + this.cancellation = const TaskCancellationToken.none(), this.args = const {}, this.enqueuer, this.workflows, @@ -2360,6 +2758,9 @@ class TaskContext implements TaskExecutionContext { /// Metadata for the task. @override final Map meta; + + @override + final TaskCancellationToken cancellation; final void Function() _heartbeat; final Future Function(Duration) _extendLease; final Future Function( @@ -2411,7 +2812,7 @@ class TaskContext implements TaskExecutionContext { ..addAll(headers); final scopeMeta = TaskEnqueueScope.currentMeta(); final mergedMeta = { - if (scopeMeta != null) ...scopeMeta, + ...?scopeMeta, ...this.meta, ...meta, }; @@ -2474,7 +2875,7 @@ class TaskContext implements TaskExecutionContext { ..addAll(call.headers); final scopeMeta = TaskEnqueueScope.currentMeta(); final mergedMeta = { - if (scopeMeta != null) ...scopeMeta, + ...?scopeMeta, ...meta, ...call.meta, }; @@ -2621,6 +3022,31 @@ class TaskContext implements TaskExecutionContext { } } +/// Selects the isolate in which a task handler executes. +enum TaskExecutionMode { + /// Runs the handler in the worker's coordinator isolate. + /// + /// A hard time limit stops the worker from awaiting the handler, but Dart + /// cannot forcibly cancel arbitrary inline asynchronous work. Inline code + /// should therefore observe [TaskExecutionContext.cancellation] at safe + /// points when it needs cooperative cancellation. + inline, + + /// Runs the handler through the worker's managed isolate pool. + /// + /// The handler must provide a top-level [TaskEntrypoint]. The worker can + /// terminate the execution isolate when a hard time limit or hard shutdown + /// is reached. + isolate, +} + +/// Optional provider for handlers that explicitly declare their execution +/// mode. +abstract interface class TaskExecutionModeProvider { + /// Declares where the handler executes. + TaskExecutionMode get executionMode; +} + /// Runtime task handler. /// Since: 0.1.0 abstract class TaskHandler { @@ -2637,10 +3063,95 @@ abstract class TaskHandler { Future call(TaskContext context, Map args); /// Optional entrypoint that allows this task to execute inside an isolate - /// worker. When `null`, the handler runs in the coordinator isolate. + /// worker. When `null`, the default execution mode is + /// [TaskExecutionMode.inline]. TaskEntrypoint? get isolateEntrypoint => null; } +/// Resolves the execution mode for a legacy or explicitly-declared handler. +extension TaskHandlerExecutionModeX on TaskHandler { + /// Returns the explicit mode when the handler provides one, otherwise + /// preserves the historical entrypoint-based behavior. + /// + /// Declaring [TaskExecutionMode.isolate] without an entrypoint is rejected + /// by the worker with a descriptive error. + TaskExecutionMode get executionMode { + final provider = this; + if (provider is TaskExecutionModeProvider) { + return (provider as TaskExecutionModeProvider).executionMode; + } + return isolateEntrypoint == null + ? TaskExecutionMode.inline + : TaskExecutionMode.isolate; + } +} + +/// Typed task handler for the recommended manual registration path. +/// +/// The runtime still lowers task arguments to the durable map transport, but +/// application code receives [TArgs] and returns [TResult]. Generated task +/// adapters can use the same contract while keeping transport decoding out of +/// user code. +class TypedTaskHandler + implements TaskHandler, TaskExecutionModeProvider { + /// Creates a typed task handler backed by [definition]. + const TypedTaskHandler({ + required this.definition, + required this.entrypoint, + this.argsDecoder, + this.isolateEntrypoint, + TaskExecutionMode? executionMode, + }) : _executionMode = executionMode; + + /// Typed task definition used for naming, options, metadata, and results. + final TaskDefinition definition; + + /// Typed application handler. + final Future Function(TaskExecutionContext context, TArgs args) + entrypoint; + + /// Optional decoder for definitions that only provide an encoder. + final TaskArgsDecoder? argsDecoder; + + @override + String get name => definition.name; + + @override + TaskOptions get options => definition.defaultOptions; + + @override + TaskMetadata get metadata => definition.metadata; + + /// Optional top-level adapter used when the task is dispatched to an + /// execution isolate. Generated handlers provide this adapter so the + /// durable map transport is decoded inside the child isolate. + @override + final TaskEntrypoint? isolateEntrypoint; + + final TaskExecutionMode? _executionMode; + + /// Explicit execution mode, or the mode inferred from + /// [isolateEntrypoint] when omitted. + @override + TaskExecutionMode get executionMode => + _executionMode ?? + (isolateEntrypoint == null + ? TaskExecutionMode.inline + : TaskExecutionMode.isolate); + + @override + Future call(TaskContext context, Map args) async { + final decoder = argsDecoder ?? definition.decodeArgs; + if (decoder == null) { + throw StateError( + 'Task definition "${definition.name}" does not provide an argument ' + 'decoder for TypedTaskHandler.', + ); + } + return entrypoint(context, decoder(args)); + } +} + /// Registry mapping task names to handler implementations. abstract class TaskRegistry { /// Registers the [handler] with this registry. @@ -2729,6 +3240,9 @@ class TaskMetadata { /// Encodes strongly typed task arguments into a JSON-ready map. typedef TaskArgsEncoder = Map Function(TArgs args); +/// Decodes persisted task arguments into the typed handler input. +typedef TaskArgsDecoder = TArgs Function(Map args); + /// Builds metadata for a task invocation using its arguments. typedef TaskMetaBuilder = Map Function(TArgs args); @@ -2760,6 +3274,7 @@ class TaskDefinition { const TaskDefinition({ required this.name, required TaskArgsEncoder encodeArgs, + this.decodeArgs, TaskMetaBuilder? encodeMeta, this.defaultOptions = const TaskOptions(), this.metadata = const TaskMetadata(), @@ -2779,6 +3294,7 @@ class TaskDefinition { return TaskDefinition( name: name, encodeArgs: (args) => _encodeCodecArgs(name, argsCodec, args), + decodeArgs: (args) => argsCodec.decode(args), encodeMeta: encodeMeta, defaultOptions: defaultOptions, metadata: _metadataWithResultCodec(name, metadata, resultCodec), @@ -2790,6 +3306,9 @@ class TaskDefinition { /// `toJson()`. factory TaskDefinition.json({ required String name, + TArgs Function(Map payload)? decodeArgsJson, + TArgs Function(Map payload, int version)? + decodeArgsVersionedJson, TaskMetaBuilder? encodeMeta, TaskOptions defaultOptions = const TaskOptions(), TaskMetadata metadata = const TaskMetadata(), @@ -2804,6 +3323,23 @@ class TaskDefinition { decodeResultJson == null || decodeResultVersionedJson == null, 'Specify either decodeResultJson or decodeResultVersionedJson, not both.', ); + assert( + decodeArgsJson == null || decodeArgsVersionedJson == null, + 'Specify either decodeArgsJson or decodeArgsVersionedJson, not both.', + ); + final argsCodec = decodeArgsVersionedJson != null + ? PayloadCodec.versionedJson( + version: defaultDecodeVersion ?? 1, + decode: decodeArgsVersionedJson, + defaultDecodeVersion: defaultDecodeVersion, + typeName: argsTypeName ?? '$TArgs', + ) + : (decodeArgsJson == null + ? null + : PayloadCodec.json( + decode: decodeArgsJson, + typeName: argsTypeName ?? '$TArgs', + )); final resultCodec = decodeResultVersionedJson != null ? PayloadCodec.versionedJson( version: defaultDecodeVersion ?? 1, @@ -2820,6 +3356,7 @@ class TaskDefinition { return TaskDefinition( name: name, encodeArgs: (args) => _encodeJsonArgs(args, argsTypeName ?? '$TArgs'), + decodeArgs: argsCodec?.decode, encodeMeta: encodeMeta, defaultOptions: defaultOptions, metadata: _metadataWithResultCodec(name, metadata, resultCodec), @@ -2832,6 +3369,9 @@ class TaskDefinition { factory TaskDefinition.versionedJson({ required String name, required int version, + TArgs Function(Map payload)? decodeArgsJson, + TArgs Function(Map payload, int version)? + decodeArgsVersionedJson, TaskMetaBuilder? encodeMeta, TaskOptions defaultOptions = const TaskOptions(), TaskMetadata metadata = const TaskMetadata(), @@ -2846,6 +3386,23 @@ class TaskDefinition { decodeResultJson == null || decodeResultVersionedJson == null, 'Specify either decodeResultJson or decodeResultVersionedJson, not both.', ); + assert( + decodeArgsJson == null || decodeArgsVersionedJson == null, + 'Specify either decodeArgsJson or decodeArgsVersionedJson, not both.', + ); + final argsCodec = decodeArgsVersionedJson != null + ? PayloadCodec.versionedJson( + version: version, + decode: decodeArgsVersionedJson, + defaultDecodeVersion: defaultDecodeVersion, + typeName: argsTypeName ?? '$TArgs', + ) + : (decodeArgsJson == null + ? null + : PayloadCodec.json( + decode: decodeArgsJson, + typeName: argsTypeName ?? '$TArgs', + )); final resultCodec = decodeResultVersionedJson != null ? PayloadCodec.versionedJson( version: version, @@ -2866,6 +3423,7 @@ class TaskDefinition { version: version, typeName: argsTypeName ?? '$TArgs', ), + decodeArgs: argsCodec?.decode, encodeMeta: encodeMeta, defaultOptions: defaultOptions, metadata: _metadataWithResultCodec(name, metadata, resultCodec), @@ -3134,6 +3692,9 @@ class TaskDefinition { /// Optional decoder for converting persisted payloads into a typed result. final TaskResultDecoder? decodeResult; + /// Optional decoder used by typed handler adapters. + final TaskArgsDecoder? decodeArgs; + final TaskArgsEncoder _encodeArgs; final TaskMetaBuilder? _encodeMeta; @@ -3215,6 +3776,27 @@ class TaskDefinition { /// Encodes arguments into a JSON-ready map. Map encodeArgs(TArgs args) => _encodeArgs(args); + /// Creates a typed inline handler for this definition. + /// + /// When [executionMode] is omitted, the mode is inferred from + /// [isolateEntrypoint]. Set it explicitly when the execution guarantee is + /// part of the task's contract. + TypedTaskHandler handler({ + required Future Function(TaskExecutionContext context, TArgs args) + entrypoint, + TaskArgsDecoder? argsDecoder, + TaskEntrypoint? isolateEntrypoint, + TaskExecutionMode? executionMode, + }) { + return TypedTaskHandler( + definition: this, + entrypoint: entrypoint, + argsDecoder: argsDecoder, + isolateEntrypoint: isolateEntrypoint, + executionMode: executionMode, + ); + } + /// Builds metadata for the given arguments. Map encodeMeta(TArgs args) { final metaBuilder = _encodeMeta; @@ -3258,6 +3840,7 @@ class NoArgsTaskDefinition { TaskDefinition<(), TResult> get asDefinition => TaskDefinition<(), TResult>( name: name, encodeArgs: (_) => const {}, + decodeArgs: (_) => const (), defaultOptions: defaultOptions, metadata: metadata, decodeResult: decodeResult, @@ -3265,6 +3848,23 @@ class NoArgsTaskDefinition { /// Decodes a persisted payload into a typed result. TResult? decode(Object? payload) => asDefinition.decode(payload); + + /// Creates a typed handler for a task that takes no input arguments. + /// + /// When [executionMode] is omitted, the mode is inferred from + /// [isolateEntrypoint]. + TypedTaskHandler<(), TResult> handler({ + required Future Function(TaskExecutionContext context, ()) + entrypoint, + TaskEntrypoint? isolateEntrypoint, + TaskExecutionMode? executionMode, + }) { + return asDefinition.handler( + entrypoint: entrypoint, + isolateEntrypoint: isolateEntrypoint, + executionMode: executionMode, + ); + } } /// Represents a pending enqueue operation built from a [TaskDefinition]. @@ -3380,6 +3980,77 @@ abstract class RetryStrategy { Duration nextDelay(int attempt, Object error, StackTrace stackTrace); } +/// Typed rate-limit configuration shared across workers. +/// Since: 0.3.0 +@immutable +class RateLimit { + /// Creates a rate limit allowing [tokens] acquisitions per [interval]. + const RateLimit({required this.tokens, required this.interval}) + : assert(tokens > 0, 'tokens must be positive'); + + /// Creates a per-second rate limit. + const RateLimit.perSecond(int tokens) + : this(tokens: tokens, interval: const Duration(seconds: 1)); + + /// Creates a per-minute rate limit. + const RateLimit.perMinute(int tokens) + : this(tokens: tokens, interval: const Duration(minutes: 1)); + + /// Creates a per-hour rate limit. + const RateLimit.perHour(int tokens) + : this(tokens: tokens, interval: const Duration(hours: 1)); + + /// Parses the legacy `10/s`, `10/m` or `10/h` representation. + /// + /// This is intentionally retained for JSON, YAML and environment + /// configuration. Dart code should prefer the typed constructors. + static RateLimit? parse(Object? value) { + if (value == null) return null; + if (value is RateLimit) return value; + final raw = value.toString().trim().toLowerCase(); + final parts = raw.split('/'); + if (parts.length != 2) return null; + final tokens = int.tryParse(parts[0]); + if (tokens == null || tokens <= 0) return null; + final interval = switch (parts[1]) { + 's' => const Duration(seconds: 1), + 'm' => const Duration(minutes: 1), + 'h' => const Duration(hours: 1), + _ => null, + }; + return interval == null + ? null + : RateLimit(tokens: tokens, interval: interval); + } + + /// Maximum number of tokens granted in [interval]. + final int tokens; + + /// Window over which [tokens] are granted. + final Duration interval; + + @override + String toString() { + final suffix = interval == const Duration(seconds: 1) + ? 's' + : interval == const Duration(minutes: 1) + ? 'm' + : interval == const Duration(hours: 1) + ? 'h' + : interval.toString(); + return '$tokens/$suffix'; + } + + @override + bool operator ==(Object other) => + other is RateLimit && + other.tokens == tokens && + other.interval == interval; + + @override + int get hashCode => Object.hash(tokens, interval); +} + /// Optional rate limiter interface shared across workers. /// Since: 0.1.0 // Intentionally an interface for DI and test doubles. @@ -3465,6 +4136,25 @@ abstract class Lock { Future release(); } +/// A lock handle that carries a monotonically increasing fencing token. +/// +/// The token identifies the acquisition, not merely the owner process. A +/// downstream storage system can reject writes carrying an older token after +/// a lease has expired and been acquired by another process. Implementations +/// of [LockStore] that cannot provide a durable token may continue returning +/// the base [Lock] contract. +abstract interface class FencedLock implements Lock { + /// Token for this specific lock acquisition. + int get fencingToken; +} + +/// Provides the fencing token when a lock implementation supports fencing. +extension LockFencingTokenX on Lock { + /// The acquisition token, or `null` for legacy lock implementations. + int? get fencingToken => + this is FencedLock ? (this as FencedLock).fencingToken : null; +} + /// Middleware hook invoked for lifecycle events around enqueue/consume/execute. /// Since: 0.1.0 abstract class Middleware { diff --git a/packages/stem/lib/src/core/function_task_handler.dart b/packages/stem/lib/src/core/function_task_handler.dart index d47c90d0..d565e5b0 100644 --- a/packages/stem/lib/src/core/function_task_handler.dart +++ b/packages/stem/lib/src/core/function_task_handler.dart @@ -6,7 +6,8 @@ import 'package:stem/src/core/task_invocation.dart'; /// Convenience task handler that delegates execution to a top-level function /// suitable for isolate execution. Set [runInIsolate] to `false` or use /// [FunctionTaskHandler.inline] to keep execution in the worker isolate. -class FunctionTaskHandler implements TaskHandler { +class FunctionTaskHandler + implements TaskHandler, TaskExecutionModeProvider { /// Creates a task handler that delegates to a top-level [entrypoint]. FunctionTaskHandler({ required this.name, @@ -49,6 +50,11 @@ class FunctionTaskHandler implements TaskHandler { @override TaskEntrypoint? get isolateEntrypoint => runInIsolate ? _entrypoint : null; + /// Execution mode selected by [runInIsolate]. + @override + TaskExecutionMode get executionMode => + runInIsolate ? TaskExecutionMode.isolate : TaskExecutionMode.inline; + @override /// Invokes the entrypoint with a normalized invocation context. Future call(TaskContext context, Map args) async { @@ -61,6 +67,7 @@ class FunctionTaskHandler implements TaskHandler { heartbeat: context.heartbeat, extendLease: context.extendLease, progress: (percent, {data}) => context.progress(percent, data: data), + cancellation: context.cancellation, enqueuer: context.enqueuer, workflows: context.workflows, workflowEvents: context.workflowEvents, diff --git a/packages/stem/lib/src/core/queue_events.dart b/packages/stem/lib/src/core/queue_events.dart index 1da56dd7..e6f7f7c4 100644 --- a/packages/stem/lib/src/core/queue_events.dart +++ b/packages/stem/lib/src/core/queue_events.dart @@ -159,7 +159,7 @@ class QueueEventsProducer { const QueueEventsProducer({required this.broker}); /// Broker used for event delivery. - final Broker broker; + final QueueBroker broker; /// Emits [eventName] on [queue] and returns the event id. Future emit( @@ -319,7 +319,7 @@ class QueueEvents { 'stem-queue-events-${generateEnvelopeId().replaceAll('-', '')}'; /// Broker used for event consumption. - final Broker broker; + final QueueBroker broker; /// Queue scope for this listener. final String queue; diff --git a/packages/stem/lib/src/core/stem.dart b/packages/stem/lib/src/core/stem.dart index 370da441..ed2e9681 100644 --- a/packages/stem/lib/src/core/stem.dart +++ b/packages/stem/lib/src/core/stem.dart @@ -75,8 +75,8 @@ import 'package:stem/src/routing/routing_registry.dart'; import 'package:stem/src/security/signing.dart'; import 'package:stem/src/signals/emitter.dart'; -/// Shared typed task-dispatch surface used by producers, apps, and contexts. -abstract interface class TaskResultCaller implements TaskEnqueuer { +/// Observation surface for task and group lifecycle state. +abstract interface class TaskObserver { /// Reads the latest task status by task id. Future getTaskStatus(String taskId); @@ -94,6 +94,27 @@ abstract interface class TaskResultCaller implements TaskEnqueuer { }); } +/// Producer-only task enqueue surface. +/// +/// Depend on this contract in HTTP handlers, schedulers, and application +/// services that publish work. It intentionally exposes no worker lifecycle +/// or task-consumption controls. +abstract interface class StemProducer implements TaskEnqueuer {} + +/// Observer-only task result surface. +/// +/// Depend on this contract in dashboards, health endpoints, and API processes +/// that inspect work without being able to start a worker through the type +/// system. +abstract interface class StemObserver implements TaskObserver {} + +/// Shared typed task-dispatch surface used by producers, apps, and contexts. +/// +/// This compatibility contract combines [StemProducer] and [StemObserver]. +/// New application boundaries should prefer one of those narrower roles. +abstract interface class TaskResultCaller + implements StemProducer, StemObserver {} + /// Facade used by producer applications to enqueue tasks. class Stem implements TaskResultCaller { /// Creates a Stem producer facade with the provided dependencies. @@ -132,7 +153,7 @@ class Stem implements TaskResultCaller { } /// Broker used to publish task envelopes. - final Broker broker; + final QueueBroker broker; /// Task registry used to resolve handlers and metadata. final TaskRegistry registry; diff --git a/packages/stem/lib/src/core/task_invocation.dart b/packages/stem/lib/src/core/task_invocation.dart index 834eb4e0..b80cfae6 100644 --- a/packages/stem/lib/src/core/task_invocation.dart +++ b/packages/stem/lib/src/core/task_invocation.dart @@ -562,6 +562,7 @@ class TaskInvocationContext implements TaskExecutionContext { Map? data, }) progress, + TaskCancellationToken cancellation = const TaskCancellationToken.none(), Map args = const {}, TaskEnqueuer? enqueuer, WorkflowCaller? workflows, @@ -575,6 +576,7 @@ class TaskInvocationContext implements TaskExecutionContext { heartbeat: heartbeat, extendLease: extendLease, progress: progress, + cancellation: cancellation, enqueuer: enqueuer, workflows: workflows, workflowEvents: workflowEvents, @@ -587,6 +589,7 @@ class TaskInvocationContext implements TaskExecutionContext { required Map headers, required Map meta, required int attempt, + TaskCancellationToken cancellation = const TaskCancellationToken.none(), Map args = const {}, }) => TaskInvocationContext._( id: id, @@ -598,6 +601,7 @@ class TaskInvocationContext implements TaskExecutionContext { extendLease: (by) async => controlPort.send(ExtendLeaseSignal(by)), progress: (percent, {data}) async => controlPort.send(ProgressSignal(percent, data: data)), + cancellation: cancellation, enqueuer: _RemoteTaskEnqueuer(controlPort), workflows: _RemoteWorkflowCaller(controlPort), workflowEvents: _RemoteWorkflowEventEmitter(controlPort), @@ -617,6 +621,7 @@ class TaskInvocationContext implements TaskExecutionContext { Map? data, }) progress, + required this.cancellation, TaskEnqueuer? enqueuer, WorkflowCaller? workflows, WorkflowEventEmitter? workflowEvents, @@ -646,6 +651,9 @@ class TaskInvocationContext implements TaskExecutionContext { @override final int attempt; + @override + final TaskCancellationToken cancellation; + final void Function() _heartbeat; final Future Function(Duration) _extendLease; final Future Function( @@ -700,7 +708,7 @@ class TaskInvocationContext implements TaskExecutionContext { ..addAll(headers); final scopeMeta = TaskEnqueueScope.currentMeta(); final mergedMeta = { - if (scopeMeta != null) ...scopeMeta, + ...?scopeMeta, ...this.meta, ...meta, }; @@ -762,7 +770,7 @@ class TaskInvocationContext implements TaskExecutionContext { ..addAll(call.headers); final scopeMeta = TaskEnqueueScope.currentMeta(); final mergedMeta = { - if (scopeMeta != null) ...scopeMeta, + ...?scopeMeta, ...meta, ...call.meta, }; diff --git a/packages/stem/lib/src/memory.dart b/packages/stem/lib/src/memory.dart new file mode 100644 index 00000000..a5de3f13 --- /dev/null +++ b/packages/stem/lib/src/memory.dart @@ -0,0 +1,22 @@ +/// In-memory adapters used for local development and tests. +library; + +export 'memory/backend/in_memory_backend.dart' show InMemoryResultBackend; +export 'memory/brokers/in_memory_broker.dart' show InMemoryBroker; +export 'memory/control/in_memory_revoke_store.dart' show InMemoryRevokeStore; +export 'memory/memory_factories.dart' + show + memoryBrokerFactory, + memoryEventBusFactory, + memoryLockStoreFactory, + memoryResultBackendFactory, + memoryRevokeStoreFactory, + memoryScheduleStoreFactory, + memoryWorkflowStoreFactory; +export 'memory/scheduler/in_memory_lock_store.dart' show InMemoryLockStore; +export 'memory/scheduler/in_memory_schedule_store.dart' + show InMemoryScheduleStore; +export 'memory/workflow/event_bus/in_memory_event_bus.dart' + show InMemoryEventBus; +export 'memory/workflow/store/in_memory_workflow_store.dart' + show InMemoryWorkflowStore; diff --git a/packages/stem_memory/lib/src/backend/in_memory_backend.dart b/packages/stem/lib/src/memory/backend/in_memory_backend.dart similarity index 89% rename from packages/stem_memory/lib/src/backend/in_memory_backend.dart rename to packages/stem/lib/src/memory/backend/in_memory_backend.dart index 95cdff19..473c4c8f 100644 --- a/packages/stem_memory/lib/src/backend/in_memory_backend.dart +++ b/packages/stem/lib/src/memory/backend/in_memory_backend.dart @@ -1,6 +1,5 @@ // This package depends on Stem's core internals while avoiding `stem.dart` // import cycles created by the compatibility re-exports. -// ignore_for_file: implementation_imports import 'dart:async'; import 'package:stem/src/core/chord_metadata.dart'; @@ -9,7 +8,8 @@ import 'package:stem/src/core/contracts.dart'; import 'package:stem/src/observability/heartbeat.dart'; /// Simple in-memory result backend used for tests and local development. -class InMemoryResultBackend implements ResultBackend { +class InMemoryResultBackend + implements ResultBackend, AtomicTerminalResultBackend { /// Creates an in-memory backend with configurable TTLs. InMemoryResultBackend({ this.defaultTtl = const Duration(days: 1), @@ -50,6 +50,9 @@ class InMemoryResultBackend implements ResultBackend { /// Per-worker heartbeat expiry timers. final Map _heartbeatExpiry = {}; + @override + bool get supportsAtomicTerminalWrites => true; + @override Future set( String taskId, @@ -60,27 +63,48 @@ class InMemoryResultBackend implements ResultBackend { Map meta = const {}, Duration? ttl, }) async { - final now = stemNow(); - final existing = _entries[taskId]; - final createdAt = existing?.createdAt ?? now; - final status = TaskStatus( - id: taskId, - state: state, - payload: payload, - error: error, - attempt: attempt, - meta: meta, + _write( + TaskStatus( + id: taskId, + state: state, + payload: payload, + error: error, + attempt: attempt, + meta: meta, + ), + ttl: ttl, ); + } + + @override + Future setTerminalIfAbsent( + TaskStatus status, { + Duration? ttl, + }) async { + final existing = _entries[status.id]; + if (existing != null && existing.expiresAt.isBefore(stemNow())) { + _remove(status.id); + } else if (existing?.status.state.isTerminal ?? false) { + return false; + } + + // This method intentionally performs the check and write without an + // asynchronous gap. It is atomic for this single-isolate backend. + _write(status, ttl: ttl); + return true; + } - _entries[taskId] = _Entry( + void _write(TaskStatus status, {Duration? ttl}) { + final now = stemNow(); + final existing = _entries[status.id]; + _entries[status.id] = _Entry( status: status, - expiresAt: stemNow().add(ttl ?? defaultTtl), - createdAt: createdAt, + expiresAt: now.add(ttl ?? defaultTtl), + createdAt: existing?.createdAt ?? now, updatedAt: now, ); - - _scheduleExpiry(taskId, ttl ?? defaultTtl); - _watchers[taskId]?.add(status); + _scheduleExpiry(status.id, ttl ?? defaultTtl); + _watchers[status.id]?.add(status); } @override @@ -310,10 +334,11 @@ class InMemoryResultBackend implements ResultBackend { timer.cancel(); } _expiryTimers.clear(); - for (final controller in _watchers.values) { + final controllers = _watchers.values.toList(growable: false); + _watchers.clear(); + for (final controller in controllers) { await controller.close(); } - _watchers.clear(); for (final timer in _groupExpiry.values) { timer.cancel(); diff --git a/packages/stem_memory/lib/src/brokers/in_memory_broker.dart b/packages/stem/lib/src/memory/brokers/in_memory_broker.dart similarity index 97% rename from packages/stem_memory/lib/src/brokers/in_memory_broker.dart rename to packages/stem/lib/src/memory/brokers/in_memory_broker.dart index a336ff9a..2ff3694f 100644 --- a/packages/stem_memory/lib/src/brokers/in_memory_broker.dart +++ b/packages/stem/lib/src/memory/brokers/in_memory_broker.dart @@ -1,6 +1,5 @@ // This package depends on Stem's core internals while avoiding `stem.dart` // import cycles created by the compatibility re-exports. -// ignore_for_file: implementation_imports import 'dart:async'; import 'dart:collection'; import 'dart:convert'; @@ -12,7 +11,13 @@ import 'package:stem/src/core/envelope.dart'; import 'package:uuid/uuid.dart'; /// In-memory broker for testing and local development. -class InMemoryBroker implements Broker { +class InMemoryBroker + implements + Broker, + LeaseBroker, + InspectableBroker, + DeadLetterBroker, + BrokerCapabilitiesProvider { /// Creates an in-memory broker with configurable timing defaults. InMemoryBroker({ this.namespace = 'stem', @@ -70,6 +75,18 @@ class InMemoryBroker implements Broker { @override bool get supportsPriority => false; + @override + BrokerCapabilities get capabilities => const BrokerCapabilities( + supportsDelayedDelivery: true, + supportsPriorityOrdering: false, + deliveryGuarantee: BrokerDeliveryGuarantee.atLeastOnce, + supportsBroadcastFanout: true, + supportsQueueInspection: true, + supportsLeaseExtension: true, + supportsDeadLettering: true, + supportsDeadLetterReplay: true, + ); + /// Releases timers and in-memory queue state. void dispose() { if (_disposed) return; diff --git a/packages/stem_memory/lib/src/control/in_memory_revoke_store.dart b/packages/stem/lib/src/memory/control/in_memory_revoke_store.dart similarity index 97% rename from packages/stem_memory/lib/src/control/in_memory_revoke_store.dart rename to packages/stem/lib/src/memory/control/in_memory_revoke_store.dart index 02e71988..e64ea281 100644 --- a/packages/stem_memory/lib/src/control/in_memory_revoke_store.dart +++ b/packages/stem/lib/src/memory/control/in_memory_revoke_store.dart @@ -1,6 +1,5 @@ // This package depends on Stem's core internals while avoiding `stem.dart` // import cycles created by the compatibility re-exports. -// ignore_for_file: implementation_imports import 'dart:async'; import 'package:collection/collection.dart'; diff --git a/packages/stem_memory/lib/src/memory_factories.dart b/packages/stem/lib/src/memory/memory_factories.dart similarity index 84% rename from packages/stem_memory/lib/src/memory_factories.dart rename to packages/stem/lib/src/memory/memory_factories.dart index 49d08c32..3d36dd15 100644 --- a/packages/stem_memory/lib/src/memory_factories.dart +++ b/packages/stem/lib/src/memory/memory_factories.dart @@ -1,18 +1,16 @@ // This package depends on Stem's core internals while avoiding `stem.dart` // import cycles created by the compatibility re-exports. -// ignore_for_file: implementation_imports import 'package:stem/src/bootstrap/factories.dart'; +import 'package:stem/src/memory/backend/in_memory_backend.dart'; +import 'package:stem/src/memory/brokers/in_memory_broker.dart'; +import 'package:stem/src/memory/control/in_memory_revoke_store.dart'; +import 'package:stem/src/memory/scheduler/in_memory_lock_store.dart'; +import 'package:stem/src/memory/scheduler/in_memory_schedule_store.dart'; +import 'package:stem/src/memory/workflow/event_bus/in_memory_event_bus.dart'; +import 'package:stem/src/memory/workflow/store/in_memory_workflow_store.dart'; import 'package:stem/src/scheduler/schedule_calculator.dart'; import 'package:stem/src/workflow/core/workflow_clock.dart'; -import 'package:stem_memory/src/backend/in_memory_backend.dart'; -import 'package:stem_memory/src/brokers/in_memory_broker.dart'; -import 'package:stem_memory/src/control/in_memory_revoke_store.dart'; -import 'package:stem_memory/src/scheduler/in_memory_lock_store.dart'; -import 'package:stem_memory/src/scheduler/in_memory_schedule_store.dart'; -import 'package:stem_memory/src/workflow/event_bus/in_memory_event_bus.dart'; -import 'package:stem_memory/src/workflow/store/in_memory_workflow_store.dart'; - /// Creates a [StemBrokerFactory] backed by [InMemoryBroker]. StemBrokerFactory memoryBrokerFactory({ String namespace = 'stem', diff --git a/packages/stem_memory/lib/src/scheduler/in_memory_lock_store.dart b/packages/stem/lib/src/memory/scheduler/in_memory_lock_store.dart similarity index 88% rename from packages/stem_memory/lib/src/scheduler/in_memory_lock_store.dart rename to packages/stem/lib/src/memory/scheduler/in_memory_lock_store.dart index d59ceaaa..d131da35 100644 --- a/packages/stem_memory/lib/src/scheduler/in_memory_lock_store.dart +++ b/packages/stem/lib/src/memory/scheduler/in_memory_lock_store.dart @@ -1,6 +1,5 @@ // This package depends on Stem's core internals while avoiding `stem.dart` // import cycles created by the compatibility re-exports. -// ignore_for_file: implementation_imports /// Volatile, in-memory implementation of a lock store. /// /// This library provides [InMemoryLockStore], which is primarily intended for @@ -20,6 +19,7 @@ import 'package:uuid/uuid.dart'; /// an owner and an expiration timestamp. class InMemoryLockStore implements LockStore { final Map _locks = {}; + final Map _fencingTokens = {}; @override Future acquire( @@ -36,7 +36,15 @@ class InMemoryLockStore implements LockStore { } final resolvedOwner = owner ?? _InMemoryLock.generateOwner(); - final lock = _InMemoryLock(key, resolvedOwner, now.add(ttl), this); + final fencingToken = (_fencingTokens[key] ?? 0) + 1; + _fencingTokens[key] = fencingToken; + final lock = _InMemoryLock( + key, + resolvedOwner, + fencingToken, + now.add(ttl), + this, + ); _locks[key] = lock; return lock; } @@ -95,8 +103,14 @@ class InMemoryLockStore implements LockStore { } /// Internal representation of an active or expired memory lock. -class _InMemoryLock implements Lock { - _InMemoryLock(this.key, this.owner, this.expiresAt, this.store); +class _InMemoryLock implements FencedLock { + _InMemoryLock( + this.key, + this.owner, + this.fencingToken, + this.expiresAt, + this.store, + ); /// Generates a unique owner identifier using UUID v7. static String generateOwner() => const Uuid().v7(); @@ -108,6 +122,9 @@ class _InMemoryLock implements Lock { @override final String owner; + @override + final int fencingToken; + /// The absolute point in time when this lock will automatically expire. DateTime expiresAt; diff --git a/packages/stem_memory/lib/src/scheduler/in_memory_schedule_store.dart b/packages/stem/lib/src/memory/scheduler/in_memory_schedule_store.dart similarity index 100% rename from packages/stem_memory/lib/src/scheduler/in_memory_schedule_store.dart rename to packages/stem/lib/src/memory/scheduler/in_memory_schedule_store.dart diff --git a/packages/stem_memory/lib/src/workflow/event_bus/in_memory_event_bus.dart b/packages/stem/lib/src/memory/workflow/event_bus/in_memory_event_bus.dart similarity index 95% rename from packages/stem_memory/lib/src/workflow/event_bus/in_memory_event_bus.dart rename to packages/stem/lib/src/memory/workflow/event_bus/in_memory_event_bus.dart index 46b0a09e..9bff0278 100644 --- a/packages/stem_memory/lib/src/workflow/event_bus/in_memory_event_bus.dart +++ b/packages/stem/lib/src/memory/workflow/event_bus/in_memory_event_bus.dart @@ -1,6 +1,5 @@ // This package depends on Stem's core internals while avoiding `stem.dart` // import cycles created by the compatibility re-exports. -// ignore_for_file: implementation_imports import 'package:stem/src/workflow/core/event_bus.dart'; import 'package:stem/src/workflow/core/workflow_store.dart'; diff --git a/packages/stem_memory/lib/src/workflow/store/in_memory_workflow_store.dart b/packages/stem/lib/src/memory/workflow/store/in_memory_workflow_store.dart similarity index 99% rename from packages/stem_memory/lib/src/workflow/store/in_memory_workflow_store.dart rename to packages/stem/lib/src/memory/workflow/store/in_memory_workflow_store.dart index f6ff4509..f089bd78 100644 --- a/packages/stem_memory/lib/src/workflow/store/in_memory_workflow_store.dart +++ b/packages/stem/lib/src/memory/workflow/store/in_memory_workflow_store.dart @@ -1,6 +1,5 @@ // This package depends on Stem's core internals while avoiding `stem.dart` // import cycles created by the compatibility re-exports. -// ignore_for_file: implementation_imports import 'dart:collection'; import 'package:stem/src/workflow/core/run_state.dart'; @@ -87,9 +86,9 @@ class InMemoryWorkflowStore implements WorkflowStore { @override /// Creates a new workflow run and returns its generated id. Future createRun({ - String? runId, required String workflow, required Map params, + String? runId, String? parentRunId, Duration? ttl, WorkflowCancellationPolicy? cancellationPolicy, diff --git a/packages/stem/lib/src/observability/logging.dart b/packages/stem/lib/src/observability/logging.dart index c732c292..5dc22307 100644 --- a/packages/stem/lib/src/observability/logging.dart +++ b/packages/stem/lib/src/observability/logging.dart @@ -3,14 +3,7 @@ import 'dart:convert'; import 'package:ansicolor/ansicolor.dart'; import 'package:contextual/contextual.dart'; -/// Available output formats for the shared Stem logger. -enum StemLogFormat { - /// Plain logfmt-style output without ANSI color codes. - plain, - - /// Colored terminal output intended for interactive local development. - pretty, -} +import 'package:stem/src/observability/logging_types.dart'; /// Creates a formatter matching the shared Stem logging presets. LogMessageFormatter createStemLogFormatter(StemLogFormat format) { diff --git a/packages/stem/lib/src/observability/logging_api.dart b/packages/stem/lib/src/observability/logging_api.dart new file mode 100644 index 00000000..9d2e2c49 --- /dev/null +++ b/packages/stem/lib/src/observability/logging_api.dart @@ -0,0 +1,114 @@ +/// Stable, dependency-neutral logging configuration for Stem. +library; + +import 'package:contextual/contextual.dart' as contextual; + +import 'package:stem/src/observability/logging.dart' as implementation; +import 'package:stem/src/observability/logging_types.dart'; + +/// The shared Stem logger facade. +final StemLogger stemLogger = _StemLoggerFacade(); + +/// Configures Stem's built-in logger without exposing its logging dependency. +void configureStemLogging({ + StemLogLevel level = StemLogLevel.info, + StemLogFormat format = StemLogFormat.pretty, + bool enableConsole = true, +}) { + implementation.configureStemLogging( + level: _toContextualLevel(level), + format: format, + enableConsole: enableConsole, + ); +} + +contextual.Level _toContextualLevel(StemLogLevel level) { + return switch (level) { + StemLogLevel.debug => contextual.Level.debug, + StemLogLevel.info => contextual.Level.info, + StemLogLevel.notice => contextual.Level.notice, + StemLogLevel.warning => contextual.Level.warning, + StemLogLevel.error => contextual.Level.error, + StemLogLevel.critical => contextual.Level.critical, + StemLogLevel.alert => contextual.Level.alert, + StemLogLevel.emergency => contextual.Level.emergency, + }; +} + +class _StemLoggerFacade implements StemLogger { + contextual.Context? _context(Map? fields) { + return fields == null ? null : contextual.Context(fields); + } + + @override + void debug( + Object message, { + Map? fields, + StackTrace? stackTrace, + }) { + implementation.stemLogger.debug(message, _context(fields), stackTrace); + } + + @override + void info( + Object message, { + Map? fields, + StackTrace? stackTrace, + }) { + implementation.stemLogger.info(message, _context(fields), stackTrace); + } + + @override + void notice( + Object message, { + Map? fields, + StackTrace? stackTrace, + }) { + implementation.stemLogger.notice(message, _context(fields), stackTrace); + } + + @override + void warning( + Object message, { + Map? fields, + StackTrace? stackTrace, + }) { + implementation.stemLogger.warning(message, _context(fields), stackTrace); + } + + @override + void error( + Object message, { + Map? fields, + StackTrace? stackTrace, + }) { + implementation.stemLogger.error(message, _context(fields), stackTrace); + } + + @override + void critical( + Object message, { + Map? fields, + StackTrace? stackTrace, + }) { + implementation.stemLogger.critical(message, _context(fields), stackTrace); + } + + @override + void alert( + Object message, { + Map? fields, + StackTrace? stackTrace, + }) { + implementation.stemLogger.alert(message, _context(fields), stackTrace); + } + + @override + void emergency( + Object message, { + Map? fields, + StackTrace? stackTrace, + }) { + implementation.stemLogger.emergency(message, _context(fields), stackTrace); + } +} diff --git a/packages/stem/lib/src/observability/logging_types.dart b/packages/stem/lib/src/observability/logging_types.dart new file mode 100644 index 00000000..a76df875 --- /dev/null +++ b/packages/stem/lib/src/observability/logging_types.dart @@ -0,0 +1,103 @@ +/// Public logging configuration types owned by Stem. +library; + +/// Log severity understood by Stem's logging facade. +enum StemLogLevel { + /// Diagnostic messages useful during local debugging. + debug, + + /// Normal operational messages. + info, + + /// Notice-level operational messages. + notice, + + /// Recoverable or potentially actionable problems. + warning, + + /// Errors that prevented an operation from completing normally. + error, + + /// Serious errors requiring attention. + critical, + + /// Severe failures requiring immediate attention. + alert, + + /// System-level failures requiring immediate attention. + emergency, +} + +/// Output format used by Stem's built-in logging configuration. +enum StemLogFormat { + /// Plain logfmt-style output without ANSI color codes. + plain, + + /// Colored terminal output intended for interactive local development. + pretty, +} + +/// Dependency-neutral facade for emitting Stem log messages. +/// +/// The concrete logger and its formatter types are intentionally kept out of +/// Stem's public API. Structured fields should use the `fields` parameter +/// rather than a +/// logging-library context object; pass structured values through the +/// `fields` parameter. +abstract interface class StemLogger { + /// Emits a debug message. + void debug( + Object message, { + Map? fields, + StackTrace? stackTrace, + }); + + /// Emits an informational message. + void info( + Object message, { + Map? fields, + StackTrace? stackTrace, + }); + + /// Emits a notice message. + void notice( + Object message, { + Map? fields, + StackTrace? stackTrace, + }); + + /// Emits a warning message. + void warning( + Object message, { + Map? fields, + StackTrace? stackTrace, + }); + + /// Emits an error message. + void error( + Object message, { + Map? fields, + StackTrace? stackTrace, + }); + + /// Emits a critical message. + void critical( + Object message, { + Map? fields, + StackTrace? stackTrace, + }); + + /// Emits an alert message. + void alert( + Object message, { + Map? fields, + StackTrace? stackTrace, + }); + + /// Emits an emergency message. + void emergency( + Object message, { + Map? fields, + StackTrace? stackTrace, + }); +} diff --git a/packages/stem/lib/src/observability/metrics.dart b/packages/stem/lib/src/observability/metrics.dart index 7f5035a4..57c9363d 100644 --- a/packages/stem/lib/src/observability/metrics.dart +++ b/packages/stem/lib/src/observability/metrics.dart @@ -628,32 +628,35 @@ class _DartasticMetricsRuntime { : Map.from(event.tags); switch (event.type) { case MetricType.counter: - final counter = _counters.putIfAbsent(instrumentName, () { - return _meter!.createCounter( - name: instrumentName, - unit: event.unit == 'count' ? null : event.unit, - ); - }); - counter.addWithMap(event.value, attributes); + _counters + .putIfAbsent(instrumentName, () { + return _meter!.createCounter( + name: instrumentName, + unit: event.unit == 'count' ? null : event.unit, + ); + }) + .addWithMap(event.value, attributes); case MetricType.histogram: - final histogram = _histograms.putIfAbsent(instrumentName, () { - return _meter!.createHistogram( - name: instrumentName, - unit: _normalizedHistogramUnit(event.unit), - ); - }); - histogram.recordWithMap( - _normalizedHistogramValue(event.value, event.unit), - attributes, - ); + _histograms + .putIfAbsent(instrumentName, () { + return _meter!.createHistogram( + name: instrumentName, + unit: _normalizedHistogramUnit(event.unit), + ); + }) + .recordWithMap( + _normalizedHistogramValue(event.value, event.unit), + attributes, + ); case MetricType.gauge: - final gauge = _gauges.putIfAbsent(instrumentName, () { - return _meter!.createGauge( - name: instrumentName, - unit: event.unit, - ); - }); - gauge.recordWithMap(event.value, attributes); + _gauges + .putIfAbsent(instrumentName, () { + return _meter!.createGauge( + name: instrumentName, + unit: event.unit, + ); + }) + .recordWithMap(event.value, attributes); } } diff --git a/packages/stem/lib/src/observability/tracing.dart b/packages/stem/lib/src/observability/tracing.dart index 9ee0cd85..091680ad 100644 --- a/packages/stem/lib/src/observability/tracing.dart +++ b/packages/stem/lib/src/observability/tracing.dart @@ -6,6 +6,9 @@ import 'package:dartastic_opentelemetry_api/dartastic_opentelemetry_api.dart' class StemTracer { StemTracer._(); + static const _traceLinkHeader = 'stem.trace.link'; + static const _traceLinkStateHeader = 'stem.trace.link.state'; + /// Singleton instance used across the runtime. static final StemTracer instance = StemTracer._(); @@ -35,6 +38,7 @@ class StemTracer { Future Function() fn, { dotel.Context? context, Map attributes = const {}, + List? links, dotel.SpanKind spanKind = dotel.SpanKind.internal, }) async { if (!_isTelemetryReady) { @@ -50,6 +54,7 @@ class StemTracer { context: baseContext, kind: spanKind, attributes: attributeSet, + links: links, ); try { return await tracer.withSpanAsync(span, fn); @@ -64,6 +69,7 @@ class StemTracer { T Function() fn, { dotel.Context? context, Map attributes = const {}, + List? links, dotel.SpanKind spanKind = dotel.SpanKind.internal, }) { if (!_isTelemetryReady) { @@ -79,6 +85,7 @@ class StemTracer { context: baseContext, kind: spanKind, attributes: attributeSet, + links: links, ); try { return tracer.withSpan(span, fn); @@ -109,6 +116,42 @@ class StemTracer { } } + /// Adds the current span as a causal link for a later fan-out operation. + /// + /// This is separate from `traceparent`: a Canvas composition span remains + /// the useful causal anchor even when a consumer also has a normal parent + /// span. The header uses W3C traceparent formatting for interoperability. + void injectTraceLink( + Map headers, { + dotel.Context? context, + }) { + if (!_isTelemetryReady) return; + final spanContext = _spanContextFrom(context ?? dotel.Context.current); + if (spanContext == null) return; + final traceParent = _formatTraceparent(spanContext); + if (traceParent == null) return; + headers[_traceLinkHeader] = traceParent; + final traceState = spanContext.traceState; + if (traceState != null && traceState.entries.isNotEmpty) { + headers[_traceLinkStateHeader] = traceState.toString(); + } else { + headers.remove(_traceLinkStateHeader); + } + } + + /// Extracts the optional Canvas fan-out link from task headers. + List extractTraceLinks(Map headers) { + if (!_isTelemetryReady) return const []; + final traceParent = headers[_traceLinkHeader]; + if (traceParent == null) return const []; + final traceHeaders = {'traceparent': traceParent}; + final traceState = headers[_traceLinkStateHeader]; + if (traceState != null) traceHeaders['tracestate'] = traceState; + final spanContext = _parseTraceContext(traceHeaders); + if (spanContext == null) return const []; + return [dotel.OTel.spanLink(spanContext)]; + } + /// Extracts a trace context from [headers]. dotel.Context extractTraceContext( Map headers, { diff --git a/packages/stem/lib/src/scheduler/beat.dart b/packages/stem/lib/src/scheduler/beat.dart index f3a3a230..2cef36c3 100644 --- a/packages/stem/lib/src/scheduler/beat.dart +++ b/packages/stem/lib/src/scheduler/beat.dart @@ -53,7 +53,7 @@ class Beat { final ScheduleStore store; /// Broker used to publish scheduled tasks. - final Broker broker; + final QueueBroker broker; /// Optional lock store for distributed scheduling. final LockStore? lockStore; @@ -177,6 +177,7 @@ class Beat { Lock? lock; Timer? renewalTimer; Duration? jitterDelay; + var leaseLost = false; if (lockStore != null) { lock = await lockStore!.acquire( 'stem:schedule:${entry.id}', @@ -200,6 +201,7 @@ class Beat { renewalTimer = Timer.periodic(Duration(milliseconds: renewMs), (_) async { final renewed = await lock!.renew(lockTtl); if (!renewed) { + leaseLost = true; StemMetrics.instance.increment( 'stem.scheduler.lock.renew_failed', tags: {'schedule': entry.id}, @@ -225,11 +227,40 @@ class Beat { await Future.delayed(jitterDelay); } + // Revalidate ownership immediately before publication. The periodic + // renewal timer protects long jitter/dispatch windows, but a scheduler + // must not publish after it has already lost the distributed lock. + if (lock != null) { + final alreadyLost = leaseLost; + if (alreadyLost || !await lock.renew(lockTtl)) { + leaseLost = true; + if (!alreadyLost) { + StemMetrics.instance.increment( + 'stem.scheduler.lock.renew_failed', + tags: {'schedule': entry.id}, + ); + } + throw StateError( + 'Scheduler lock was lost before publishing schedule ' + '"${entry.id}".', + ); + } + } + + final headers = { + 'scheduled-from': 'beat', + 'schedule-id': entry.id, + }; + final fencingToken = lock?.fencingToken; + if (fencingToken != null) { + headers['stem-lock-fencing-token'] = fencingToken.toString(); + } + var envelope = Envelope( name: entry.taskName, args: entry.args, queue: entry.queue, - headers: {'scheduled-from': 'beat', 'schedule-id': entry.id}, + headers: headers, meta: entry.kwargs.isEmpty ? entry.meta : {...entry.meta, 'kwargs': entry.kwargs}, diff --git a/packages/stem/lib/src/worker/README.md b/packages/stem/lib/src/worker/README.md index 4b96939a..121a89e7 100644 --- a/packages/stem/lib/src/worker/README.md +++ b/packages/stem/lib/src/worker/README.md @@ -61,16 +61,29 @@ The **Worker Package** is the task execution runtime of the Stem task queue fram └─────────────────────────────────────────────────────────────────┘ ``` +The worker is an orchestration boundary rather than the owner of every +subsystem. `TaskHandler.executionMode` is the source of truth for inline +versus isolate execution. `WorkerExecutionSupervisor` owns that dispatch, +hard-timeout handling, and isolate-pool lifecycle; `WorkerLeaseCoordinator`, +`WorkerAcknowledgementCoordinator`, and `WorkerDeliveryTracker` own their +corresponding stateful mechanics. Delivery state, retries, result persistence, +and control-plane policy remain in `Worker`. + --- ## File Structure | File | Lines | Description | |------|-------|-------------| -| `worker.dart` | 3374 | Main worker runtime - task consumption, lifecycle, observability | +| `worker.dart` | — | Delivery loop, state transitions, retries, control plane, and lifecycle orchestration | +| `worker_execution_supervisor.dart` | — | Inline/isolate execution, hard limits, and isolate-pool lifecycle | +| `worker_consumer_loop.dart` | — | Broker subscription ownership, replacement, and stream error boundaries | | `isolate_pool.dart` | 573 | Isolate pool management for concurrent task execution | | `isolate_messages.dart` | 177 | Message types for isolate communication | | `worker_config.dart` | 136 | Configuration classes for autoscaling and lifecycle | +| `worker_acknowledgement.dart` | — | Terminal-state acknowledgement recovery | +| `worker_lease_coordinator.dart` | — | Lease renewal timers and failure containment | +| `worker_delivery_tracker.dart` | — | Active delivery and in-flight queue tracking | --- @@ -86,7 +99,7 @@ The main runtime that consumes tasks from a broker and executes handlers. | Parameter | Type | Default | Description | |-----------|------|---------|-------------| -| `broker` | `Broker` | **required** | Message broker for publishing/consuming | +| `broker` | `QueueBroker` | **required** | Queue transport for publishing/consuming | | `registry` | `TaskRegistry` | **required** | Registry containing task handlers | | `backend` | `ResultBackend` | **required** | Backend for persisting task state | | `enqueuer` | `Stem?` | `null` | Stem instance for spawning child tasks | @@ -118,7 +131,7 @@ The main runtime that consumes tasks from a broker and executes handlers. | Field | Type | Description | |-------|------|-------------| -| `broker` | `Broker` | Broker for consuming/acknowledging deliveries | +| `broker` | `QueueBroker` | Broker for consuming/acknowledging deliveries | | `registry` | `TaskRegistry` | Task handler registry | | `backend` | `ResultBackend` | Result persistence backend | | `rateLimiter` | `RateLimiter?` | Optional rate limiter | @@ -161,13 +174,13 @@ The main runtime that consumes tasks from a broker and executes handlers. | `_runConsumeMiddleware(Delivery)` | Executes consume middleware before task handling | | `_notifyErrorMiddleware(TaskContext, Object, StackTrace)` | Notifies middleware about task processing errors | | `_invokeWithMiddleware(TaskContext, Future Function())` | Invokes handler through middleware chain | -| `_executeWithHardLimit(TaskHandler, TaskContext, Envelope, Map)` | Runs handler with optional hard time limit | +| `_executeWithHardLimit(TaskHandler, TaskContext, Envelope, Map)` | Delegates execution and hard limits to `WorkerExecutionSupervisor` | | `_scheduleSoftLimit(Envelope, TaskOptions)` | Schedules soft timeout warning | -| `_startHeartbeat(String)` | Starts heartbeat timer for a task | -| `_scheduleLeaseRenewal(Delivery)` | Schedules visibility/lease renewal | -| `_restartLeaseTimer(Delivery, Duration)` | Restarts lease timer after renewal | -| `_startLeaseTimer(Delivery, Duration)` | Starts lease renewal timer | -| `_cancelLeaseTimer(String)` | Cancels lease renewal tracking | +| `_startHeartbeat(Delivery)` | Starts a heartbeat timer for one delivery | +| `_scheduleLeaseRenewal(Delivery)` | Delegates visibility/lease renewal to the lease coordinator | +| `_restartLeaseTimer(Delivery, Duration)` | Restarts lease renewal through the coordinator | +| `_cancelLeaseTimer(Delivery)` | Cancels lease renewal after terminal handling | +| `WorkerLeaseCoordinator` | Owns renewal timers and contains broker renewal failures | | `_noteLeaseRenewal(Delivery)` | Records lease renewal timestamp | | `_releaseUniqueLock(Envelope)` | Releases unique task locks | | `_maybeDispatchChord(GroupStatus)` | Dispatches chord callbacks when group completes | @@ -180,17 +193,17 @@ The main runtime that consumes tasks from a broker and executes handlers. | `_handleFailure(...)` | Handles task failure with retries and status updates | | `_handleRetryRequest(...)` | Handles explicit retry requests from handlers | | `_rateLimitKey(TaskOptions, Envelope)` | Builds rate-limit key | -| `_parseRate(String)` | Parses rate limit string (e.g., "10/m") | +| `RateLimit` | Typed rate limit value; legacy strings are parsed at configuration boundaries | | `_sendHeartbeat(String)` | Emits heartbeat update for running task | | `_trackDelivery(Delivery)` | Tracks in-flight delivery | -| `_releaseDelivery(Envelope)` | Removes delivery from tracking | +| `_releaseDelivery(Delivery)` | Removes the exact broker delivery from tracking | | `_recordInflightGauge()` | Records in-flight task count metric | | `_recordConcurrencyGauge()` | Records active concurrency metric | | `_recordQueueDepth()` | Updates queue depth metrics | | `_collectQueueDepths()` | Collects pending counts per queue | | `_cancelAllSubscriptions()` | Cancels broker subscriptions | | `_cancelTimers()` | Cancels active runtime timers | -| `_disposePool()` | Disposes isolate pool | +| `_disposePool()` | Disposes the execution supervisor and isolate pool | | `_forceStopActiveTasks()` | Forcibly stops tasks, requeues deliveries | | `_requestTerminationForActiveTasks({String reason})` | Requests cooperative termination | | `_awaitDrainWithTimeout(Duration?)` | Waits for tasks to drain with timeout | @@ -200,7 +213,7 @@ The main runtime that consumes tasks from a broker and executes handlers. | `_startWorkerHeartbeatLoop()` | Starts periodic worker heartbeat | | `_startAutoscaler()` | Starts autoscaling evaluation timer | | `_evaluateAutoscale()` | Evaluates autoscale rules | -| `_updateConcurrency(int, {String reason, int backlog, int inflight})` | Updates isolate pool size | +| `_updateConcurrency(int, {String reason, int backlog, int inflight})` | Updates worker concurrency and delegates pool resizing | | `_cooldownElapsed(DateTime?, Duration, DateTime)` | Checks autoscale cooldown | | `_handleIsolateRecycle(IsolateRecycleEvent)` | Handles isolate pool recycling | | `_installSignalHandlers()` | Installs process signal handlers | diff --git a/packages/stem/lib/src/worker/worker.dart b/packages/stem/lib/src/worker/worker.dart index 5354e64c..7cd7295b 100644 --- a/packages/stem/lib/src/worker/worker.dart +++ b/packages/stem/lib/src/worker/worker.dart @@ -23,6 +23,13 @@ /// │ │ │ /// │ ▼ │ /// │ ┌──────────────────────────────────────────────────┐ │ +/// │ │ WorkerExecutionSupervisor │ │ +/// │ │ • Inline/isolate mode • Hard limits │ │ +/// │ │ • Pool lifecycle • Recycling │ │ +/// │ └──────────────────────┬───────────────────────────┘ │ +/// │ │ │ +/// │ ▼ │ +/// │ ┌──────────────────────────────────────────────────┐ │ /// │ │ TaskIsolatePool │ │ /// │ │ ┌────────┐ ┌────────┐ ┌────────┐ ┌────────┐ │ │ /// │ │ │Isolate1│ │Isolate2│ │Isolate3│ │IsolateN│ │ │ @@ -102,6 +109,7 @@ import 'package:dartastic_opentelemetry/dartastic_opentelemetry.dart' as dotel; import 'package:stem/src/control/control_messages.dart'; import 'package:stem/src/control/revoke_store.dart'; import 'package:stem/src/core/chord_metadata.dart'; +import 'package:stem/src/core/chord_policy.dart'; import 'package:stem/src/core/clock.dart'; import 'package:stem/src/core/contracts.dart'; import 'package:stem/src/core/encoder_keys.dart'; @@ -124,7 +132,12 @@ import 'package:stem/src/security/signing.dart'; import 'package:stem/src/signals/emitter.dart'; import 'package:stem/src/signals/payloads.dart'; import 'package:stem/src/worker/isolate_pool.dart'; +import 'package:stem/src/worker/worker_acknowledgement.dart'; import 'package:stem/src/worker/worker_config.dart'; +import 'package:stem/src/worker/worker_consumer_loop.dart'; +import 'package:stem/src/worker/worker_delivery_tracker.dart'; +import 'package:stem/src/worker/worker_execution_supervisor.dart'; +import 'package:stem/src/worker/worker_lease_coordinator.dart'; import 'package:stem/src/workflow/core/workflow_cancellation_policy.dart'; import 'package:stem/src/workflow/core/workflow_event_ref.dart'; import 'package:stem/src/workflow/core/workflow_ref.dart'; @@ -308,7 +321,7 @@ class Worker { /// - [argsEncoder]: Default encoder for task arguments. /// - [additionalEncoders]: Additional payload encoders to register. Worker({ - required Broker broker, + required QueueBroker broker, required ResultBackend backend, Iterable> tasks = const [], TaskRegistry? registry, @@ -488,6 +501,37 @@ class Worker { subscriptionQueues = List.unmodifiable(normalizedQueues); subscriptionBroadcasts = List.unmodifiable(normalizedBroadcasts); _signals = StemSignalEmitter(defaultSender: _workerIdentifier); + _consumer = WorkerConsumerLoop(broker: broker); + _execution = WorkerExecutionSupervisor( + concurrency: _currentConcurrency, + lifecycle: lifecycleConfig, + onRecycle: _handleIsolateRecycle, + onSpawned: (isolateId) { + unawaited( + _signals.workerChildLifecycle( + _workerInfoSnapshot, + isolateId, + initializing: true, + ), + ); + }, + onDisposed: (isolateId) { + unawaited( + _signals.workerChildLifecycle( + _workerInfoSnapshot, + isolateId, + initializing: false, + ), + ); + }, + ); + _acknowledgements = WorkerAcknowledgementCoordinator(broker); + _leases = WorkerLeaseCoordinator( + broker: broker, + onLeaseUpdated: _noteLeaseRenewal, + onLeaseRenewed: _recordLeaseRenewal, + onRenewalFailure: _handleLeaseRenewalFailure, + ); } static TaskRegistry _resolveTaskRegistry( @@ -500,7 +544,19 @@ class Worker { } /// Broker used to consume and acknowledge deliveries. - final Broker broker; + final QueueBroker broker; + + /// Coordinates recoverable terminal acknowledgements. + late final WorkerAcknowledgementCoordinator _acknowledgements; + + /// Coordinates automatic lease renewal for active deliveries. + late final WorkerLeaseCoordinator _leases; + + /// Supervises inline and isolate-backed task execution. + late final WorkerExecutionSupervisor _execution; + + /// Owns broker subscription replacement and cancellation. + late final WorkerConsumerLoop _consumer; /// Task registry containing handlers and metadata. final TaskRegistry registry; @@ -595,12 +651,8 @@ class Worker { String get primaryQueue => _effectiveQueues.isNotEmpty ? _effectiveQueues.first : queue; - final Map _leaseTimers = {}; - final Map _heartbeatTimers = {}; - final Map> _subscriptions = {}; + final Map _heartbeatTimers = {}; final StreamController _events = StreamController.broadcast(); - TaskIsolatePool? _isolatePool; - Future? _poolFuture; late final int _maxConcurrency; late int _currentConcurrency; Timer? _autoscaleTimer; @@ -616,10 +668,11 @@ class Worker { StreamSubscription? _sigquitSub; bool _running = false; - final Map _activeDeliveries = {}; - final Map _inflightPerQueue = {}; - final Set _queueSubscriptionNames = {}; - int _inflight = 0; + final WorkerDeliveryTracker _deliveryTracker = WorkerDeliveryTracker(); + Map get _activeDeliveries => + _deliveryTracker.active; + Map get _inflightPerQueue => _deliveryTracker.inflightPerQueue; + int get _inflight => _deliveryTracker.inflight; Timer? _workerHeartbeatTimer; DateTime? _lastLeaseRenewal; int? _lastQueueDepth; @@ -637,6 +690,17 @@ class Worker { /// Includes events like task start, completion, failure, and heartbeats. Stream get events => _events.stream; + /// Emits an event while the worker event stream is still open. + /// + /// Broker callbacks, timers, and isolate completions can race with + /// shutdown. Those callbacks must not turn a successful shutdown into an + /// asynchronous `Bad state: Cannot add new events` error. + void _emitEvent(WorkerEvent event) { + if (!_events.isClosed) { + _events.add(event); + } + } + /// Current active concurrency for isolate-backed tasks. int get activeConcurrency => _currentConcurrency; @@ -731,8 +795,7 @@ class Worker { _revocations.clear(); _queuePauses.clear(); _latestRevocationVersion = 0; - _inflightPerQueue.clear(); - _inflight = 0; + _deliveryTracker.clear(); _idleSince = null; _recordConcurrencyGauge(); @@ -751,6 +814,7 @@ class Worker { final envelope = delivery.envelope; final tracer = StemTracer.instance; final parentContext = tracer.extractTraceContext(envelope.headers); + final traceLinks = tracer.extractTraceLinks(envelope.headers); final baseSpanAttributes = _deliverySpanAttributes(envelope); final consumeSpanAttributes = { ...baseSpanAttributes, @@ -764,36 +828,102 @@ class Worker { await tracer.trace( 'stem.consume', () async { - final handler = registry.resolve(envelope.name); - if (handler == null) { - await broker.deadLetter(delivery, reason: 'unregistered-task'); - await _releaseUniqueLock(envelope); - return; - } + // Start lease protection as soon as the delivery enters the worker. + // Consume middleware, signature verification, status lookups, and + // rate-limit calls all happen before handler execution and can be + // slower than a short broker visibility timeout. + _scheduleLeaseRenewal(delivery); + var deliveryTracked = false; + try { + final handler = registry.resolve(envelope.name); + if (handler == null) { + await _deadLetterOrDiscard(delivery, reason: 'unregistered-task'); + await _releaseUniqueLock(envelope); + return; + } - final argsEncoder = _resolveArgsEncoder(handler); - final resultEncoder = _resolveResultEncoder(handler); + final argsEncoder = _resolveArgsEncoder(handler); + final resultEncoder = _resolveResultEncoder(handler); - await _runConsumeMiddleware(delivery); + await _runConsumeMiddleware(delivery); - final groupId = envelope.headers['stem-group-id']; + final groupId = envelope.headers['stem-group-id']; - if (_isTaskRevoked(envelope.id)) { - await _handleRevokedDelivery( - delivery, - envelope, - resultEncoder, - groupId: groupId, - ); - await _releaseUniqueLock(envelope); - return; - } + if (_isTaskRevoked(envelope.id)) { + await _handleRevokedDelivery( + delivery, + envelope, + resultEncoder, + groupId: groupId, + ); + await _releaseUniqueLock(envelope); + return; + } - if (signer != null) { + if (signer != null) { + try { + await signer!.verify(envelope); + } on SignatureVerificationException catch (error, stack) { + await _handleSignatureFailure( + delivery, + envelope, + resultEncoder, + error, + stack, + groupId, + ); + await _releaseUniqueLock(envelope); + return; + } + } + + if (_isExpired(envelope)) { + await _handleExpiredDelivery(delivery, envelope, resultEncoder); + await _releaseUniqueLock(envelope); + return; + } + + final priorStatus = await backend.get(envelope.id); + if (priorStatus?.state.isTerminal ?? false) { + // An acknowledgement can be lost after the result is durable. Do + // not execute a redelivered terminal task a second time. + await _acknowledgements.tryAcknowledge( + delivery, + envelope: envelope, + phase: 'terminal recovery', + ); + await _releaseUniqueLock(envelope); + return; + } + + if (_deliveryTracker.containsEnvelopeId(envelope.id)) { + // A short lease or broker retry can deliver the same envelope to + // this worker while its first delivery is still in progress. The + // active delivery owns execution; acknowledging this duplicate + // avoids concurrent side effects without affecting recovery after + // a process-wide worker failure. + StemMetrics.instance.increment( + 'stem.tasks.duplicate_suppressed', + tags: {'task': envelope.name, 'queue': envelope.queue}, + ); + await _acknowledgements.tryAcknowledge( + delivery, + envelope: envelope, + phase: 'active duplicate recovery', + ); + return; + } + + if (_isQueuePaused(envelope.queue)) { + await _handlePausedQueueDelivery(delivery, envelope, resultEncoder); + return; + } + + Map decodedArgs; try { - await signer!.verify(envelope); - } on SignatureVerificationException catch (error, stack) { - await _handleSignatureFailure( + decodedArgs = _decodeArgs(envelope, argsEncoder); + } on Object catch (error, stack) { + await _handlePayloadDecodeFailure( delivery, envelope, resultEncoder, @@ -804,39 +934,103 @@ class Worker { await _releaseUniqueLock(envelope); return; } - } - - if (_isExpired(envelope)) { - await _handleExpiredDelivery(delivery, envelope, resultEncoder); - await _releaseUniqueLock(envelope); - return; - } - - if (_isQueuePaused(envelope.queue)) { - await _handlePausedQueueDelivery(delivery, envelope, resultEncoder); - return; - } - final decodedArgs = _decodeArgs(envelope, argsEncoder); + final groupRateSpec = handler.options.groupRateLimit; + if (rateLimiter != null && groupRateSpec != null) { + final groupKey = _groupRateLimitKey(handler.options, envelope); + try { + final decision = await rateLimiter!.acquire( + 'group:$groupKey', + tokens: groupRateSpec.tokens, + interval: groupRateSpec.interval, + meta: {'task': envelope.name, 'queue': envelope.queue}, + ); + if (!decision.allowed) { + final backoff = + decision.retryAfter ?? + retryStrategy.nextDelay( + envelope.attempt, + StateError('group-rate-limit'), + StackTrace.current, + ); + await _retryDelivery( + delivery, + envelope, + resultEncoder, + backoff: backoff, + extra: { + 'rateLimited': true, + 'groupRateLimited': true, + 'groupKey': groupKey, + 'retryAfterMs': backoff.inMilliseconds, + }, + ); + return; + } + } on Object catch (error, stack) { + final mode = handler.options.groupRateLimiterFailureMode; + if (mode == RateLimiterFailureMode.failOpen) { + StemMetrics.instance.increment( + 'stem.rate_limiter.degraded', + tags: { + 'task': envelope.name, + 'queue': envelope.queue, + 'scope': 'group', + 'mode': mode.name, + }, + ); + stemLogger.warning( + 'Group rate limiter unavailable; continuing in ' + 'fail-open mode', + Context( + _logContext({ + 'task': envelope.name, + 'id': envelope.id, + 'queue': envelope.queue, + 'group': groupKey, + 'error': error.toString(), + 'stack': stack.toString(), + }), + ), + ); + } else { + final backoff = retryStrategy.nextDelay( + envelope.attempt, + error, + stack, + ); + await _retryDelivery( + delivery, + envelope, + resultEncoder, + backoff: backoff, + extra: { + 'groupRateLimited': true, + 'groupKey': groupKey, + 'rateLimiterUnavailable': true, + 'failureMode': mode.name, + 'error': error.toString(), + }, + ); + return; + } + } + } - final groupRateSpec = handler.options.groupRateLimit != null - ? _parseRate(handler.options.groupRateLimit!) - : null; - if (rateLimiter != null && groupRateSpec != null) { - final groupKey = _groupRateLimitKey(handler.options, envelope); - try { + final rateSpec = handler.options.rateLimit; + if (rateLimiter != null && rateSpec != null) { final decision = await rateLimiter!.acquire( - 'group:$groupKey', - tokens: groupRateSpec.tokens, - interval: groupRateSpec.period, - meta: {'task': envelope.name, 'queue': envelope.queue}, + _rateLimitKey(handler.options, envelope), + tokens: rateSpec.tokens, + interval: rateSpec.interval, + meta: {'task': envelope.name}, ); if (!decision.allowed) { final backoff = decision.retryAfter ?? retryStrategy.nextDelay( envelope.attempt, - StateError('group-rate-limit'), + StateError('rate-limit'), StackTrace.current, ); await _retryDelivery( @@ -846,304 +1040,279 @@ class Worker { backoff: backoff, extra: { 'rateLimited': true, - 'groupRateLimited': true, - 'groupKey': groupKey, 'retryAfterMs': backoff.inMilliseconds, }, ); return; } - } on Object catch (error, stack) { - final mode = handler.options.groupRateLimiterFailureMode; - if (mode == RateLimiterFailureMode.failOpen) { - StemMetrics.instance.increment( - 'stem.rate_limiter.degraded', - tags: { - 'task': envelope.name, - 'queue': envelope.queue, - 'scope': 'group', - 'mode': mode.name, - }, - ); - stemLogger.warning( - 'Group rate limiter unavailable; continuing in fail-open mode', - Context( - _logContext({ - 'task': envelope.name, - 'id': envelope.id, - 'queue': envelope.queue, - 'group': groupKey, - 'error': error.toString(), - 'stack': stack.toString(), - }), - ), - ); - } else { - final backoff = retryStrategy.nextDelay( - envelope.attempt, - error, - stack, - ); - await _retryDelivery( - delivery, - envelope, - resultEncoder, - backoff: backoff, - extra: { - 'groupRateLimited': true, - 'groupKey': groupKey, - 'rateLimiterUnavailable': true, - 'failureMode': mode.name, - 'error': error.toString(), - }, - ); - return; - } - } - } - - final rateSpec = handler.options.rateLimit != null - ? _parseRate(handler.options.rateLimit!) - : null; - if (rateLimiter != null && rateSpec != null) { - final decision = await rateLimiter!.acquire( - _rateLimitKey(handler.options, envelope), - tokens: rateSpec.tokens, - interval: rateSpec.period, - meta: {'task': envelope.name}, - ); - if (!decision.allowed) { - final backoff = - decision.retryAfter ?? - retryStrategy.nextDelay( - envelope.attempt, - StateError('rate-limit'), - StackTrace.current, - ); - await _retryDelivery( - delivery, - envelope, - resultEncoder, - backoff: backoff, - extra: { - 'rateLimited': true, - 'retryAfterMs': backoff.inMilliseconds, - }, - ); - return; } - } - - _trackDelivery(delivery); - - await _signals.taskReceived(envelope, _workerInfoSnapshot); - - stemLogger.debug( - 'Task {task} started', - Context(_deliveryLogContext(envelope)), - ); - StemMetrics.instance.increment( - 'stem.tasks.started', - tags: {'task': envelope.name, 'queue': envelope.queue}, - ); - _startedCount += 1; - - String? startedAtIso; - final startedAt = stemNow().toUtc(); - final runningMeta = _statusMeta( - envelope, - resultEncoder, - extra: { - 'queue': envelope.queue, - 'worker': consumerName, - 'startedAt': (startedAtIso = startedAt.toIso8601String()), - }, - ); - await backend.set( - envelope.id, - TaskState.running, - attempt: envelope.attempt, - meta: runningMeta, - ); - - void checkTermination() => _enforceTerminationIfRequested(envelope.id); - - final context = TaskContext( - id: envelope.id, - args: envelope.args, - attempt: envelope.attempt, - headers: envelope.headers, - meta: envelope.meta, - heartbeat: () { - checkTermination(); - _sendHeartbeat(envelope.id); - }, - extendLease: (duration) async { - checkTermination(); - await broker.extendLease(delivery, duration); - _recordLeaseRenewal(delivery); - _restartLeaseTimer(delivery, duration); - _noteLeaseRenewal(delivery); - }, - progress: (progress, {data}) async { - checkTermination(); - _reportProgress(envelope, progress, data: data); - }, - enqueuer: _enqueuer, - workflows: workflows, - workflowEvents: workflowEvents, - ); - await _signals.taskPrerun(envelope, _workerInfoSnapshot, context); + _trackDelivery(delivery); + deliveryTracked = true; - Timer? heartbeatTimer; - Timer? softTimer; - _scheduleLeaseRenewal(delivery); - - dynamic result; - var completionState = TaskState.running; + await _signals.taskReceived(envelope, _workerInfoSnapshot); - try { - checkTermination(); - heartbeatTimer = _startHeartbeat(envelope.id); - softTimer = _scheduleSoftLimit(envelope, handler.options); - - result = await tracer.trace( - 'stem.execute.${envelope.name}', - () => _invokeWithMiddleware( - context, - () => _executeWithHardLimit( - handler, - context, - envelope, - decodedArgs, - ), - ), - attributes: executeSpanAttributes, + stemLogger.debug( + 'Task {task} started', + Context(_deliveryLogContext(envelope)), ); + StemMetrics.instance.increment( + 'stem.tasks.started', + tags: {'task': envelope.name, 'queue': envelope.queue}, + ); + _startedCount += 1; - _cancelLeaseTimer(delivery.receipt); - _heartbeatTimers.remove(envelope.id)?.cancel(); - - final ignoreResult = _shouldIgnoreResult(envelope); - final persistedResult = ignoreResult ? null : result; - final successMeta = _statusMeta( + String? startedAtIso; + final startedAt = stemNow().toUtc(); + final runningMeta = _statusMeta( envelope, resultEncoder, extra: { 'queue': envelope.queue, 'worker': consumerName, - 'completedAt': stemNow().toIso8601String(), - 'startedAt': startedAtIso, + 'startedAt': (startedAtIso = startedAt.toIso8601String()), }, ); - final successStatus = TaskStatus( - id: envelope.id, - state: TaskState.succeeded, - payload: persistedResult, - attempt: envelope.attempt, - meta: successMeta, - ); - await broker.ack(delivery); await backend.set( envelope.id, - TaskState.succeeded, - payload: persistedResult, + TaskState.running, attempt: envelope.attempt, - meta: successMeta, - ); - GroupStatus? groupStatus; - if (groupId != null) { - groupStatus = await backend.addGroupResult(groupId, successStatus); - } - StemMetrics.instance.increment( - 'stem.tasks.succeeded', - tags: {'task': envelope.name, 'queue': envelope.queue}, - ); - _completedCount += 1; - stemLogger.debug( - 'Task {task} succeeded', - Context(_deliveryLogContext(envelope)), + meta: runningMeta, ); - _events.add( - WorkerEvent(type: WorkerEventType.completed, envelope: envelope), - ); - await _signals.taskSucceeded( - envelope, - _workerInfoSnapshot, - result: result, + + final cancellation = TaskCancellationToken( + isCancelled: () => _revocationFor(envelope.id)?.terminate ?? false, ); - if (groupStatus != null) { - await _maybeDispatchChord(groupStatus); + + void checkTermination() { + try { + _enforceTerminationIfRequested(envelope.id); + } on TaskRevokedException { + cancellation.cancel(); + rethrow; + } } - completionState = TaskState.succeeded; - } on TaskRevokedException catch (_) { - _cancelLeaseTimer(delivery.receipt); - _heartbeatTimers.remove(envelope.id)?.cancel(); - await _handleRevokedDelivery( - delivery, - envelope, - resultEncoder, - groupId: groupId, - ); - completionState = TaskState.cancelled; - } on TaskRetryRequest catch (request) { - _cancelLeaseTimer(delivery.receipt); - _heartbeatTimers.remove(envelope.id)?.cancel(); - completionState = await _handleRetryRequest( - handler, - delivery, - envelope, - resultEncoder, - request, - groupId, - ); - } on Object catch (error, stack) { - await _notifyErrorMiddleware(context, error, stack); - _cancelLeaseTimer(delivery.receipt); - _heartbeatTimers.remove(envelope.id)?.cancel(); - completionState = await _handleFailure( - handler, - delivery, - envelope, - resultEncoder, - error, - stack, - groupId, - startedAtIso, + + final context = TaskContext( + id: envelope.id, + args: envelope.args, + attempt: envelope.attempt, + headers: envelope.headers, + meta: envelope.meta, + heartbeat: () { + checkTermination(); + _sendHeartbeat(envelope.id); + }, + extendLease: (duration) async { + checkTermination(); + await broker.extendLease(delivery, duration); + _recordLeaseRenewal(delivery); + _restartLeaseTimer(delivery, duration); + _noteLeaseRenewal(delivery); + }, + progress: (progress, {data}) async { + checkTermination(); + _reportProgress(envelope, progress, data: data); + }, + cancellation: cancellation, + enqueuer: _enqueuer, + workflows: workflows, + workflowEvents: workflowEvents, ); - } finally { - if (completionState == TaskState.succeeded) { - await _dispatchLinkedTasks(envelope, onSuccess: true); - } else if (completionState == TaskState.failed) { - await _dispatchLinkedTasks(envelope, onSuccess: false); - } - heartbeatTimer?.cancel(); - softTimer?.cancel(); - final completed = _releaseDelivery(envelope); - if (completed != null) { - final duration = stemNow().toUtc().difference( - completed.startedAt, + + await _signals.taskPrerun(envelope, _workerInfoSnapshot, context); + + Timer? heartbeatTimer; + Timer? softTimer; + + dynamic result; + var completionState = TaskState.running; + var terminalWriteOwned = false; + + try { + checkTermination(); + heartbeatTimer = _startHeartbeat(delivery); + softTimer = _scheduleSoftLimit(envelope, handler.options); + + result = await tracer.trace( + 'stem.execute.${envelope.name}', + () => _invokeWithMiddleware( + context, + () => _executeWithHardLimit( + handler, + context, + envelope, + decodedArgs, + ), + ), + attributes: executeSpanAttributes, ); - StemMetrics.instance.recordDuration( - 'stem.task.duration', - duration, - tags: {'task': envelope.name, 'queue': envelope.queue}, + + _heartbeatTimers.remove(delivery)?.cancel(); + + final ignoreResult = _shouldIgnoreResult(envelope); + final persistedResult = ignoreResult ? null : result; + final successMeta = _statusMeta( + envelope, + resultEncoder, + extra: { + 'queue': envelope.queue, + 'worker': consumerName, + 'completedAt': stemNow().toIso8601String(), + 'startedAt': startedAtIso, + }, ); + final successStatus = TaskStatus( + id: envelope.id, + state: TaskState.succeeded, + payload: persistedResult, + attempt: envelope.attempt, + meta: successMeta, + ); + final terminal = await _writeTerminalStatus(successStatus); + completionState = terminal.status.state; + terminalWriteOwned = terminal.applied; + GroupStatus? groupStatus; + if (terminal.applied) { + if (groupId != null) { + groupStatus = await backend.addGroupResult( + groupId, + successStatus, + ); + } + StemMetrics.instance.increment( + 'stem.tasks.succeeded', + tags: {'task': envelope.name, 'queue': envelope.queue}, + ); + _completedCount += 1; + stemLogger.debug( + 'Task {task} succeeded', + Context(_deliveryLogContext(envelope)), + ); + _emitEvent( + WorkerEvent( + type: WorkerEventType.completed, + envelope: envelope, + ), + ); + await _signals.taskSucceeded( + envelope, + _workerInfoSnapshot, + result: result, + ); + if (groupStatus != null) { + await _maybeDispatchChord(groupStatus); + } + } + // Persisting success before acknowledgement makes an uncertain + // acknowledgement safe to recover as an at-least-once duplicate. + // A later delivery can observe the terminal result and avoid + // executing the handler again. + await _acknowledgements.tryAcknowledge( + delivery, + envelope: envelope, + phase: 'terminal success', + ); + } on TaskCancellationException catch (_) { + _heartbeatTimers.remove(delivery)?.cancel(); + final completion = await _handleRevokedDelivery( + delivery, + envelope, + resultEncoder, + groupId: groupId, + ); + completionState = completion.state; + terminalWriteOwned = completion.terminalWriteOwned; + } on TaskRevokedException catch (_) { + _heartbeatTimers.remove(delivery)?.cancel(); + final completion = await _handleRevokedDelivery( + delivery, + envelope, + resultEncoder, + groupId: groupId, + ); + completionState = completion.state; + terminalWriteOwned = completion.terminalWriteOwned; + } on TaskRetryRequest catch (request) { + _heartbeatTimers.remove(delivery)?.cancel(); + final completion = await _handleRetryRequest( + handler, + delivery, + envelope, + resultEncoder, + request, + groupId, + ); + completionState = completion.state; + terminalWriteOwned = completion.terminalWriteOwned; + } on Object catch (error, stack) { + await _notifyErrorMiddleware(context, error, stack); + _heartbeatTimers.remove(delivery)?.cancel(); + final completion = await _handleFailure( + handler, + delivery, + envelope, + resultEncoder, + error, + stack, + groupId, + startedAtIso, + ); + completionState = completion.state; + terminalWriteOwned = completion.terminalWriteOwned; + } finally { + if (terminalWriteOwned && completionState == TaskState.succeeded) { + await _dispatchLinkedTasks(envelope, onSuccess: true); + } else if (terminalWriteOwned && + completionState == TaskState.failed) { + await _dispatchLinkedTasks(envelope, onSuccess: false); + } + heartbeatTimer?.cancel(); + softTimer?.cancel(); + final completed = _releaseDelivery(delivery); + if (completed != null) { + final duration = stemNow().toUtc().difference( + completed.startedAt, + ); + StemMetrics.instance.recordDuration( + 'stem.task.duration', + duration, + tags: {'task': envelope.name, 'queue': envelope.queue}, + ); + } + await _signals.taskPostrun( + envelope, + _workerInfoSnapshot, + context, + result: result, + state: completionState, + ); + if (terminalWriteOwned && _isTerminalState(completionState)) { + await _releaseUniqueLock(envelope); + } + // Keep the broker lease alive through terminal result persistence, + // group/chord bookkeeping, retry or dead-letter publication, linked + // task dispatch, acknowledgement, postrun hooks, and unique-lock + // release. Those operations can outlive the handler itself and + // must not create a duplicate delivery by allowing the lease to + // expire before the lifecycle is complete. + _cancelLeaseTimer(delivery); + if (deliveryTracked) { + // The normal lifecycle finally releases this entry. Keep the + // outer guard for failures in prerun/status setup before that + // inner finally is entered. + _releaseDelivery(delivery); + } } - await _signals.taskPostrun( - envelope, - _workerInfoSnapshot, - context, - result: result, - state: completionState, - ); - if (_isTerminalState(completionState)) { - await _releaseUniqueLock(envelope); - } + } finally { + // Early exits before handler execution use the same lease cleanup as + // normal terminal handling. The inner lifecycle finally also + // cancels this timer after acknowledgement and task postrun hooks. + _cancelLeaseTimer(delivery); } }, context: parentContext, + links: traceLinks, spanKind: dotel.SpanKind.consumer, attributes: consumeSpanAttributes, ); @@ -1274,19 +1443,13 @@ class Worker { Map args, ) { final hard = _resolveHardTimeLimit(envelope, handler.options); - if (_shouldUseIsolate(handler)) { - return _runInIsolate(handler, context, envelope, args, hardTimeout: hard); - } - - final future = handler.call(context, args); - if (hard == null) { - return future; - } - return future.timeout( - hard, - onTimeout: () => throw TimeoutException( - 'hard time limit exceeded for ${handler.name}', - ), + return _execution.execute( + handler: handler, + context: context, + envelope: envelope, + args: args, + controlHandler: _controlHandler(context), + hardTimeout: hard, ); } @@ -1306,7 +1469,7 @@ class Worker { final soft = _resolveSoftTimeLimit(envelope, options); if (soft == null) return null; return Timer(soft, () { - _events.add( + _emitEvent( WorkerEvent( type: WorkerEventType.timeout, envelope: envelope, @@ -1321,24 +1484,29 @@ class Worker { /// Heartbeats are emitted at [heartbeatInterval] intervals and serve as: /// - Keep-alive signals to monitoring systems /// - Progress indicators for long-running tasks - /// - Lease renewal triggers for broker visibility /// - /// The timer is stored in [_heartbeatTimers] and cancelled when the task - /// completes or is revoked. + /// Broker lease renewal is handled by [WorkerLeaseCoordinator]. A heartbeat + /// is an observability signal; task code should call `extendLease` when it + /// needs an explicit lease extension. + /// + /// The timer is stored in [_heartbeatTimers] by delivery identity and + /// cancelled when that delivery completes or is revoked. /// /// Returns `null` if heartbeat interval is zero or negative. - Timer? _startHeartbeat(String envelopeId) { + Timer? _startHeartbeat(Delivery delivery) { if (heartbeatInterval <= Duration.zero) return null; final timer = Timer.periodic( heartbeatInterval, - (_) => _sendHeartbeat(envelopeId), + (_) => _sendHeartbeat(delivery.envelope.id), ); - _heartbeatTimers[envelopeId]?.cancel(); - _heartbeatTimers[envelopeId] = timer; + _heartbeatTimers[delivery]?.cancel(); + _heartbeatTimers[delivery] = timer; return timer; } - /// Schedules automatic lease/visibility renewal while a task is running. + /// Schedules automatic lease/visibility renewal as soon as a delivery is + /// accepted by the worker, including consume middleware and pre-execution + /// validation. /// /// ## Implementation Details /// @@ -1351,7 +1519,7 @@ class Worker { /// /// 1. Get the lease expiration time from `delivery.leaseExpiresAt` /// 2. Calculate remaining time: `expiresAt - now` - /// 3. Set renewal interval to half the remaining time, clamped to 1-30s + /// 3. Set renewal interval to half the remaining time, capped at 30s /// 4. Start a periodic timer that calls `broker.extendLease` /// /// The "half remaining" strategy ensures we renew well before expiration @@ -1361,23 +1529,16 @@ class Worker { /// /// - If `leaseExpiresAt` is null: broker doesn't support leases, skip /// - If remaining time <= 0: lease already expired, skip - /// - Minimum interval: 1 second (prevents tight loops) + /// - Minimum interval: 1 second for ordinary leases; short leases use the + /// shorter deadline-safe interval instead of being delayed past expiry /// - Maximum interval: 30 seconds (ensures timely renewal) /// /// ## State Changes /// - /// - Adds timer to `_leaseTimers` keyed by receipt + /// - Adds a renewal timer keyed by the delivery identity /// - Updates `_lastLeaseRenewal` via `_noteLeaseRenewal` void _scheduleLeaseRenewal(Delivery delivery) { - final expiresAt = delivery.leaseExpiresAt; - if (expiresAt == null) return; - final remainingMs = expiresAt.difference(stemNow()).inMilliseconds; - if (remainingMs <= 0) return; - final interval = Duration( - milliseconds: (remainingMs ~/ 2).clamp(1000, 30000), - ); - _startLeaseTimer(delivery, interval); - _noteLeaseRenewal(delivery); + _leases.schedule(delivery); } /// Restarts the lease timer after a successful manual renewal. @@ -1386,7 +1547,9 @@ class Worker { /// /// Called when task code explicitly extends its lease via /// [TaskContext.extendLease]. The new timer interval is calculated as - /// half the granted [duration], clamped to 1-30 seconds. + /// half the granted [duration], capped at 30 seconds. Short explicit + /// extensions use the shorter interval so the configured minimum cannot + /// delay renewal past the new deadline. /// /// This differs from [_scheduleLeaseRenewal] in that: /// - It uses a provided duration rather than calculating from expiry time @@ -1398,51 +1561,17 @@ class Worker { /// - Creates new timer with recalculated interval /// - Updates [_lastLeaseRenewal] timestamp void _restartLeaseTimer(Delivery delivery, Duration duration) { - final intervalMs = (duration.inMilliseconds ~/ 2).clamp(1000, 30000); - _startLeaseTimer(delivery, Duration(milliseconds: intervalMs)); - _noteLeaseRenewal(delivery); - } - - /// Starts or replaces the lease renewal timer for a delivery. - /// - /// ## Implementation Details - /// - /// Creates a [Timer.periodic] that fires every [interval] to extend - /// the broker lease. The timer callback: - /// - /// 1. Calls `broker.extendLease` to renew visibility - /// 2. Increments the 'stem.lease.renewed' metric via `_recordLeaseRenewal` - /// 3. Updates internal tracking via `_noteLeaseRenewal` - /// - /// ## Timer Management - /// - /// - Keyed by `delivery.receipt` (unique broker message ID) - /// - Any existing timer for the same receipt is cancelled first - /// - Timer stored in `_leaseTimers` map for later cancellation - /// - /// ## Thread Safety - /// - /// The timer callback is async but runs in the main isolate's event loop. - /// If task completes before callback fires, the timer is cancelled via - /// `_cancelLeaseTimer` and the callback becomes a no-op. - void _startLeaseTimer(Delivery delivery, Duration interval) { - _leaseTimers[delivery.receipt]?.cancel(); - final timer = Timer.periodic(interval, (_) async { - await broker.extendLease(delivery, interval); - _recordLeaseRenewal(delivery); - _noteLeaseRenewal(delivery); - }); - _leaseTimers[delivery.receipt] = timer; + _leases.restart(delivery, duration); } - /// Cancels and removes the lease renewal timer for a broker receipt. + /// Cancels and removes the lease renewal timer for a delivery. /// /// ## Implementation Details /// - /// Called when: - /// - Task completes successfully or fails - /// - Task is revoked - /// - Worker is shutting down + /// Called after the delivery's terminal handling is complete, including + /// result persistence, group/chord bookkeeping, retry or dead-letter + /// publication, linked-task dispatch, and acknowledgement. It is also + /// called when a delivery is revoked or the worker is shutting down. /// /// Uses [Map.remove] with null-safe chaining to atomically remove /// and cancel in one operation. If no timer exists for the receipt, @@ -1450,10 +1579,34 @@ class Worker { /// /// ## Parameters /// - /// - [receipt]: The broker-assigned unique message identifier - /// (e.g., SQS ReceiptHandle, RabbitMQ delivery tag) - void _cancelLeaseTimer(String receipt) { - _leaseTimers.remove(receipt)?.cancel(); + /// - [delivery]: The broker delivery whose timer should be cancelled. + void _cancelLeaseTimer(Delivery delivery) { + _leases.cancel(delivery); + } + + /// Records a failed lease renewal without escaping the timer callback. + void _handleLeaseRenewalFailure( + Delivery delivery, + Object error, + StackTrace stackTrace, + ) { + StemMetrics.instance.increment( + 'stem.lease.renewal_failed', + tags: {'task': delivery.envelope.name, 'queue': delivery.envelope.queue}, + ); + stemLogger.warning( + 'Task lease renewal failed', + Context( + _logContext({ + 'task': delivery.envelope.name, + 'id': delivery.envelope.id, + 'queue': delivery.envelope.queue, + 'receipt': delivery.receipt, + 'error': error.toString(), + 'stack': stackTrace.toString(), + }), + ), + ); } /// Records the timestamp of the most recent lease renewal. @@ -1465,11 +1618,11 @@ class Worker { /// 1. **Worker-level**: [_lastLeaseRenewal] - used in worker heartbeats /// to show when any task last renewed its lease /// - /// 2. **Per-delivery**: [_ActiveDelivery.lastLeaseRenewal] - used for + /// 2. **Per-delivery**: [WorkerActiveDelivery.lastLeaseRenewal] - used for /// per-task observability and debugging stalled tasks /// /// This is called after every successful lease extension, both automatic - /// (from [_startLeaseTimer]) and manual (from [TaskContext.extendLease]). + /// (from the lease coordinator) and manual (from [TaskContext.extendLease]). /// /// ## Difference from [_recordLeaseRenewal] /// @@ -1478,7 +1631,7 @@ class Worker { void _noteLeaseRenewal(Delivery delivery) { final now = stemNow().toUtc(); _lastLeaseRenewal = now; - final active = _activeDeliveries[delivery.envelope.id]; + final active = _deliveryTracker.forDelivery(delivery); if (active != null) { active.lastLeaseRenewal = now; } @@ -1596,18 +1749,34 @@ class Worker { /// - `stem.chords.dispatched`: Successful callback enqueue /// - `stem.chords.dispatch_failed`: Failed callback enqueue Future _maybeDispatchChord(GroupStatus status) async { - if (!status.isComplete) return; - final allSucceeded = status.results.values.every( - (s) => s.state == TaskState.succeeded, - ); - if (!allSucceeded) return; + final policy = ChordPolicy.fromJson(status.meta[ChordMetadata.policy]); + if (!policy.shouldDispatch(status)) return; final callbackData = status.meta[ChordMetadata.callbackEnvelope]; if (callbackData is! Map) { return; } - final resultsPayload = status.results.values.map((s) => s.payload).toList(); + final resultsPayload = status.results.values + .map( + (result) => + result.state == TaskState.succeeded ? result.payload : null, + ) + .toList(); + final failurePayload = status.results.values + .where( + (result) => + result.state == TaskState.failed || + result.state == TaskState.cancelled, + ) + .map( + (result) => { + 'taskId': result.id, + 'state': result.state.name, + if (result.error != null) 'error': result.error!.toJson(), + }, + ) + .toList(growable: false); final dispatchedAt = stemNow().toUtc(); final callbackTaskId = (callbackData['id'] as String?) ?? generateEnvelopeId(); @@ -1632,6 +1801,8 @@ class Worker { ...callbackEnvelope.meta, 'chordId': status.id, 'chordResults': resultsPayload, + ChordMetadata.policy: policy.toJson(), + if (failurePayload.isNotEmpty) ChordMetadata.failures: failurePayload, }, ); @@ -1891,7 +2062,7 @@ class Worker { /// /// This method doesn't throw - all errors are handled internally /// to ensure the task is properly failed and cleaned up. - Future _handleSignatureFailure( + Future<_TaskCompletion> _handleSignatureFailure( Delivery delivery, Envelope envelope, TaskPayloadEncoder resultEncoder, @@ -1899,15 +2070,6 @@ class Worker { StackTrace stack, String? groupId, ) async { - await broker.deadLetter( - delivery, - reason: 'signature-invalid', - meta: { - 'error': error.message, - if (error.keyId != null) 'keyId': error.keyId, - }, - ); - final failureMeta = _statusMeta( envelope, resultEncoder, @@ -1935,12 +2097,25 @@ class Worker { meta: failureMeta, ); - await backend.set( - envelope.id, - TaskState.failed, - attempt: envelope.attempt, - error: failureStatus.error, - meta: failureMeta, + final terminal = await _writeTerminalStatus(failureStatus); + if (!terminal.applied) { + await _acknowledgements.tryAcknowledge( + delivery, + envelope: envelope, + phase: 'terminal signature duplicate', + ); + return _TaskCompletion( + state: terminal.status.state, + terminalWriteOwned: false, + ); + } + await _deadLetterOrDiscard( + delivery, + reason: 'signature-invalid', + meta: { + 'error': error.message, + if (error.keyId != null) 'keyId': error.keyId, + }, ); if (groupId != null) { await backend.addGroupResult(groupId, failureStatus); @@ -1968,7 +2143,7 @@ class Worker { ), ); - _events.add( + _emitEvent( WorkerEvent( type: WorkerEventType.failed, envelope: envelope, @@ -1983,6 +2158,10 @@ class Worker { error: error, stackTrace: stack, ); + return const _TaskCompletion( + state: TaskState.failed, + terminalWriteOwned: true, + ); } /// Handles task failure, determining retry behavior and updating state. @@ -2028,7 +2207,7 @@ class Worker { /// ## Returns /// /// The resulting [TaskState]: either `retried` or `failed`. - Future _handleFailure( + Future<_TaskCompletion> _handleFailure( TaskHandler handler, Delivery delivery, Envelope envelope, @@ -2078,7 +2257,7 @@ class Worker { 'stem.tasks.retried', tags: {'task': envelope.name, 'queue': envelope.queue}, ); - _events.add( + _emitEvent( WorkerEvent( type: WorkerEventType.retried, envelope: envelope, @@ -2107,7 +2286,10 @@ class Worker { ), ), ); - return TaskState.retried; + return const _TaskCompletion( + state: TaskState.retried, + terminalWriteOwned: false, + ); } else { final failureMeta = _statusMeta( envelope, @@ -2130,20 +2312,29 @@ class Worker { attempt: envelope.attempt, meta: failureMeta, ); - await broker.deadLetter( + final terminal = await _writeTerminalStatus(failureStatus); + if (!terminal.applied) { + await _acknowledgements.tryAcknowledge( + delivery, + envelope: envelope, + phase: 'terminal failure duplicate', + ); + return _TaskCompletion( + state: terminal.status.state, + terminalWriteOwned: false, + ); + } + await _deadLetterOrDiscard( delivery, reason: 'max-retries-exhausted', meta: {'error': error.toString()}, ); - await backend.set( - envelope.id, - TaskState.failed, - attempt: envelope.attempt, - error: failureStatus.error, - meta: failureMeta, - ); + GroupStatus? groupStatus; if (groupId != null) { - await backend.addGroupResult(groupId, failureStatus); + groupStatus = await backend.addGroupResult(groupId, failureStatus); + } + if (groupStatus != null) { + await _maybeDispatchChord(groupStatus); } StemMetrics.instance.increment( 'stem.tasks.failed', @@ -2162,7 +2353,7 @@ class Worker { ), ), ); - _events.add( + _emitEvent( WorkerEvent( type: WorkerEventType.failed, envelope: envelope, @@ -2176,12 +2367,15 @@ class Worker { error: error, stackTrace: stack, ); - return TaskState.failed; + return const _TaskCompletion( + state: TaskState.failed, + terminalWriteOwned: true, + ); } } /// Handles explicit retry requests surfaced from task handlers. - Future _handleRetryRequest( + Future<_TaskCompletion> _handleRetryRequest( TaskHandler handler, Delivery delivery, Envelope envelope, @@ -2205,18 +2399,38 @@ class Worker { 'retryExhausted': true, }, ); - await broker.nack(delivery, requeue: false); - await backend.set( - envelope.id, - TaskState.failed, + const failureError = TaskError( + type: 'RetryExhausted', + message: 'retry requested but max retries exceeded', + ); + final failureStatus = TaskStatus( + id: envelope.id, + state: TaskState.failed, + error: failureError, attempt: envelope.attempt, - error: const TaskError( - type: 'RetryExhausted', - message: 'retry requested but max retries exceeded', - ), meta: failureMeta, ); - _events.add( + final terminal = await _writeTerminalStatus(failureStatus); + if (!terminal.applied) { + await _acknowledgements.tryAcknowledge( + delivery, + envelope: envelope, + phase: 'terminal retry-exhausted duplicate', + ); + return _TaskCompletion( + state: terminal.status.state, + terminalWriteOwned: false, + ); + } + await broker.nack(delivery, requeue: false); + GroupStatus? groupStatus; + if (groupId != null) { + groupStatus = await backend.addGroupResult(groupId, failureStatus); + } + if (groupStatus != null) { + await _maybeDispatchChord(groupStatus); + } + _emitEvent( WorkerEvent( type: WorkerEventType.failed, envelope: envelope, @@ -2232,7 +2446,10 @@ class Worker { if (_isTerminalState(TaskState.failed)) { await _releaseUniqueLock(envelope); } - return TaskState.failed; + return const _TaskCompletion( + state: TaskState.failed, + terminalWriteOwned: true, + ); } final scheduledAt = @@ -2285,7 +2502,7 @@ class Worker { ), meta: retriedMeta, ); - _events.add( + _emitEvent( WorkerEvent( type: WorkerEventType.retried, envelope: envelope, @@ -2311,7 +2528,10 @@ class Worker { ), ), ); - return TaskState.retried; + return const _TaskCompletion( + state: TaskState.retried, + terminalWriteOwned: false, + ); } /// Builds a rate-limit key based on task options and the envelope. @@ -2356,7 +2576,7 @@ class Worker { attempt: envelope.attempt, meta: _statusMeta(envelope, resultEncoder, extra: data), ); - _events.add( + _emitEvent( WorkerEvent( type: WorkerEventType.retried, envelope: envelope, @@ -2375,6 +2595,24 @@ class Worker { await broker.publish(signed); } + /// Retains a failed delivery when the broker supports dead letters, or + /// discards it safely when the transport only implements [QueueBroker]. + /// + /// A queue-only adapter must not fail task processing merely because it has + /// no dead-letter store. The terminal task status remains persisted in the + /// result backend; the broker fallback removes the delivery without + /// requeueing it. + Future _deadLetterOrDiscard( + Delivery delivery, { + String? reason, + Map? meta, + }) { + if (broker.capabilities.supportsDeadLettering) { + return broker.deadLetter(delivery, reason: reason, meta: meta); + } + return broker.nack(delivery, requeue: false); + } + /// Requeues deliveries from paused queues without executing handlers. Future _handlePausedQueueDelivery( Delivery delivery, @@ -2391,57 +2629,24 @@ class Worker { ); } - /// Parses a rate limit string such as "10/m" into a spec. - _RateSpec? _parseRate(String rate) { - final parts = rate.split('/'); - if (parts.length != 2) return null; - final tokens = int.tryParse(parts[0]); - if (tokens == null || tokens <= 0) return null; - switch (parts[1]) { - case 's': - return _RateSpec(tokens: tokens, period: const Duration(seconds: 1)); - case 'm': - return _RateSpec(tokens: tokens, period: const Duration(minutes: 1)); - case 'h': - return _RateSpec(tokens: tokens, period: const Duration(hours: 1)); - default: - return null; - } - } - /// Emits a heartbeat update for a running task. void _sendHeartbeat(String id) { - _events.add(WorkerEvent(type: WorkerEventType.heartbeat, envelopeId: id)); + _emitEvent(WorkerEvent(type: WorkerEventType.heartbeat, envelopeId: id)); } /// Tracks an in-flight delivery for lease renewal and metrics. void _trackDelivery(Delivery delivery) { - final envelope = delivery.envelope; - final id = envelope.id; - final queueName = envelope.queue; - final startedAt = stemNow().toUtc(); - _activeDeliveries[id] = _ActiveDelivery( - queue: queueName, - startedAt: startedAt, - envelope: envelope, - delivery: delivery, + _deliveryTracker.track( + delivery, + startedAt: stemNow().toUtc(), ); - _inflight += 1; - _inflightPerQueue[queueName] = (_inflightPerQueue[queueName] ?? 0) + 1; _recordInflightGauge(); } /// Removes an in-flight delivery from tracking. - _ActiveDelivery? _releaseDelivery(Envelope envelope) { - final entry = _activeDeliveries.remove(envelope.id); + WorkerActiveDelivery? _releaseDelivery(Delivery delivery) { + final entry = _deliveryTracker.release(delivery); if (entry != null) { - _inflight = math.max(0, _inflight - 1); - final queueCount = (_inflightPerQueue[entry.queue] ?? 0) - 1; - if (queueCount <= 0) { - _inflightPerQueue.remove(entry.queue); - } else { - _inflightPerQueue[entry.queue] = queueCount; - } if (_activeDeliveries.isEmpty) { _lastLeaseRenewal = null; final drain = _drainCompleter; @@ -2513,26 +2718,6 @@ class Worker { /// Refreshes queue subscriptions after pause/resume changes. Future _refreshQueueSubscriptions() async { - final existing = List.from(_queueSubscriptionNames); - for (final queueName in existing) { - final subscription = _subscriptions.remove(queueName); - _queueSubscriptionNames.remove(queueName); - if (subscription == null) continue; - try { - await subscription.cancel(); - } on Object catch (error, stack) { - stemLogger.warning( - 'Failed to cancel queue subscription: $error', - Context( - _logContext({ - 'queue': queueName, - 'stack': stack.toString(), - }), - ), - ); - } - } - final activeQueues = _effectiveQueues .where((queueName) => !_isQueuePaused(queueName)) .toList(growable: false); @@ -2546,78 +2731,42 @@ class Worker { }), ), ); - return; } - for (var index = 0; index < activeQueues.length; index += 1) { - final queueName = activeQueues[index]; - final stream = broker.consume( - RoutingSubscription( - queues: [queueName], - broadcastChannels: index == 0 - ? _broadcastSubscriptions - : const [], - ), - prefetch: prefetch, - consumerName: consumerName, - ); - // Subscriptions are tracked and cancelled in _cancelAllSubscriptions(). - // ignore: cancel_subscriptions - final subscription = stream.listen( - (delivery) { - final task = _handle(delivery); - unawaited( - task.catchError((Object error, StackTrace stack) { - _events.add( - WorkerEvent( - type: WorkerEventType.error, - envelope: delivery.envelope, - error: error, - stackTrace: stack, - ), - ); - }), - ); - }, - onError: (Object error, StackTrace stack) { - _events.add( - WorkerEvent( - type: WorkerEventType.error, - error: error, - stackTrace: stack, - ), - ); - }, - ); - _subscriptions[queueName] = subscription; - _queueSubscriptionNames.add(queueName); - } + await _consumer.replaceQueueSubscriptions( + queues: activeQueues, + broadcastChannels: _broadcastSubscriptions, + prefetch: prefetch, + consumerName: consumerName, + onDelivery: _handle, + onDeliveryError: (delivery, error, stack) { + _emitEvent( + WorkerEvent( + type: WorkerEventType.error, + envelope: delivery.envelope, + error: error, + stackTrace: stack, + ), + ); + }, + onStreamError: (error, stack) { + _emitEvent( + WorkerEvent( + type: WorkerEventType.error, + error: error, + stackTrace: stack, + ), + ); + }, + ); } /// Cancels broker subscriptions and clears tracking state. - Future _cancelAllSubscriptions() async { - if (_subscriptions.isEmpty) return; - final subs = List>.from(_subscriptions.values); - _subscriptions.clear(); - _queueSubscriptionNames.clear(); - for (final sub in subs) { - try { - await sub.cancel(); - } on Object catch (error, stack) { - stemLogger.warning( - 'Failed to cancel subscription: $error', - Context(_logContext({'stack': stack.toString()})), - ); - } - } - } + Future _cancelAllSubscriptions() => _consumer.close(); /// Cancels active timers used by the worker runtime. void _cancelTimers() { - for (final timer in _leaseTimers.values) { - timer.cancel(); - } - _leaseTimers.clear(); + _leases.cancelAll(); for (final timer in _heartbeatTimers.values) { timer.cancel(); } @@ -2626,24 +2775,21 @@ class Worker { _workerHeartbeatTimer = null; } - /// Disposes the isolate pool if one is active. + /// Disposes the execution supervisor if an isolate pool is active. Future _disposePool() async { - final pool = _isolatePool; - _isolatePool = null; - _poolFuture = null; - if (pool != null) { - await pool.dispose(); - } + await _execution.dispose(); } /// Forcibly stops active tasks, requeuing deliveries when possible. Future _forceStopActiveTasks() async { - final deliveries = List<_ActiveDelivery>.from(_activeDeliveries.values); + final deliveries = List.from( + _activeDeliveries.values, + ); if (deliveries.isEmpty) return; for (final active in deliveries) { - _cancelLeaseTimer(active.delivery.receipt); - _heartbeatTimers.remove(active.envelope.id)?.cancel(); - _releaseDelivery(active.envelope); + _cancelLeaseTimer(active.delivery); + _heartbeatTimers.remove(active.delivery)?.cancel(); + _releaseDelivery(active.delivery); try { await broker.nack(active.delivery); } on Object catch (error, stack) { @@ -3160,10 +3306,7 @@ class Worker { _idleSince = null; } - final pool = _isolatePool; - if (pool != null) { - await pool.resize(_currentConcurrency); - } + await _execution.resize(_currentConcurrency); _recordConcurrencyGauge(); @@ -3310,9 +3453,9 @@ class Worker { /// Builds a worker heartbeat payload from current runtime state. WorkerHeartbeat _buildHeartbeat() { final now = stemNow().toUtc(); - final isolatePool = _isolatePool; - final activeIsolates = - isolatePool?.activeCount ?? math.min(_inflight, _currentConcurrency); + final activeIsolates = _execution.activeIsolates == 0 + ? math.min(_inflight, _currentConcurrency) + : _execution.activeIsolates; final queues = _inflightPerQueue.entries .where((entry) => entry.value > 0) @@ -3453,7 +3596,7 @@ class Worker { double progress, { Map? data, }) { - _events.add( + _emitEvent( WorkerEvent( type: WorkerEventType.progress, envelope: envelope, @@ -3806,6 +3949,43 @@ class Worker { return Duration(milliseconds: jittered); } + /// Persists a terminal status with backend arbitration when available. + /// + /// Terminal writes use the optional backend arbitration capability so a + /// late duplicate delivery cannot replace a terminal result persisted by a + /// different worker. Non-capable custom backends retain the historical + /// unconditional write behavior. + Future<_TerminalWriteResult> _writeTerminalStatus(TaskStatus status) async { + final candidate = backend; + if (candidate is AtomicTerminalResultBackend && + (candidate as AtomicTerminalResultBackend) + .supportsAtomicTerminalWrites) { + final atomic = candidate as AtomicTerminalResultBackend; + final applied = await atomic.setTerminalIfAbsent(status); + if (!applied) { + final existing = await backend.get(status.id); + if (existing != null && existing.state.isTerminal) { + StemMetrics.instance.increment( + 'stem.tasks.terminal_write_lost', + tags: {'state': status.state.name}, + ); + return _TerminalWriteResult(applied: false, status: existing); + } + } + return _TerminalWriteResult(applied: applied, status: status); + } + + await backend.set( + status.id, + status.state, + payload: status.payload, + error: status.error, + attempt: status.attempt, + meta: status.meta, + ); + return _TerminalWriteResult(applied: true, status: status); + } + /// Builds status metadata for result backend writes. Map _statusMeta( Envelope envelope, @@ -3832,14 +4012,13 @@ class Worker { } /// Marks revoked deliveries and ensures the broker is acknowledged. - Future _handleRevokedDelivery( + Future<_TaskCompletion> _handleRevokedDelivery( Delivery delivery, Envelope envelope, TaskPayloadEncoder resultEncoder, { String? groupId, }) async { final revokeEntry = _revocationFor(envelope.id); - await broker.ack(delivery); final meta = _statusMeta( envelope, resultEncoder, @@ -3860,12 +4039,19 @@ class Worker { attempt: envelope.attempt, meta: meta, ); - await backend.set( - envelope.id, - TaskState.cancelled, - attempt: envelope.attempt, - meta: meta, - ); + final terminal = await _writeTerminalStatus(status); + if (!terminal.applied) { + await _acknowledgements.tryAcknowledge( + delivery, + envelope: envelope, + phase: 'terminal revocation duplicate', + ); + return _TaskCompletion( + state: terminal.status.state, + terminalWriteOwned: false, + ); + } + await broker.ack(delivery); if (groupId != null) { await backend.addGroupResult(groupId, status); } @@ -3884,7 +4070,7 @@ class Worker { }), ), ); - _events.add( + _emitEvent( WorkerEvent( type: WorkerEventType.revoked, envelope: envelope, @@ -3900,6 +4086,10 @@ class Worker { reason: revokeEntry?.reason ?? 'revoked', ); _revocations.remove(envelope.id); + return const _TaskCompletion( + state: TaskState.cancelled, + terminalWriteOwned: true, + ); } /// Returns true when the delivery is past its expiration time. @@ -3914,12 +4104,11 @@ class Worker { } /// Marks expired deliveries and acknowledges them. - Future _handleExpiredDelivery( + Future<_TaskCompletion> _handleExpiredDelivery( Delivery delivery, Envelope envelope, TaskPayloadEncoder resultEncoder, ) async { - await broker.ack(delivery); final meta = _statusMeta( envelope, resultEncoder, @@ -3930,9 +4119,9 @@ class Worker { 'stem.expired': true, }, ); - await backend.set( - envelope.id, - TaskState.cancelled, + final status = TaskStatus( + id: envelope.id, + state: TaskState.cancelled, attempt: envelope.attempt, error: const TaskError( type: 'TaskExpired', @@ -3940,6 +4129,121 @@ class Worker { ), meta: meta, ); + final terminal = await _writeTerminalStatus(status); + await _acknowledgements.tryAcknowledge( + delivery, + envelope: envelope, + phase: terminal.applied + ? 'expired task' + : 'terminal expiration duplicate', + ); + return _TaskCompletion( + state: terminal.status.state, + terminalWriteOwned: terminal.applied, + ); + } + + /// Permanently rejects a delivery whose arguments cannot be decoded. + /// + /// A payload-decoding error is a poison message, not a handler failure. It + /// cannot be repaired by retrying the same bytes, so the delivery is + /// terminally failed and dead-lettered even when the task's normal retry + /// policy allows handler retries. + Future<_TaskCompletion> _handlePayloadDecodeFailure( + Delivery delivery, + Envelope envelope, + TaskPayloadEncoder resultEncoder, + Object error, + StackTrace stack, + String? groupId, + ) async { + final meta = _statusMeta( + envelope, + resultEncoder, + extra: { + 'queue': envelope.queue, + 'worker': consumerName, + 'failedAt': stemNow().toIso8601String(), + 'invalidPayload': true, + }, + ); + final status = TaskStatus( + id: envelope.id, + state: TaskState.failed, + attempt: envelope.attempt, + error: TaskError( + type: error.runtimeType.toString(), + message: error.toString(), + stack: stack.toString(), + meta: const {'reason': 'invalid-payload'}, + ), + meta: meta, + ); + final terminal = await _writeTerminalStatus(status); + if (!terminal.applied) { + await _acknowledgements.tryAcknowledge( + delivery, + envelope: envelope, + phase: 'terminal invalid-payload duplicate', + ); + return _TaskCompletion( + state: terminal.status.state, + terminalWriteOwned: false, + ); + } + + await _deadLetterOrDiscard( + delivery, + reason: 'invalid-payload', + meta: { + 'error': error.toString(), + 'errorType': error.runtimeType.toString(), + }, + ); + if (groupId != null) { + await backend.addGroupResult(groupId, status); + } + StemMetrics.instance.increment( + 'stem.tasks.invalid_payload', + tags: {'task': envelope.name, 'queue': envelope.queue}, + ); + StemMetrics.instance.increment( + 'stem.tasks.failed', + tags: {'task': envelope.name, 'queue': envelope.queue}, + ); + _failedCount += 1; + stemLogger.error( + 'Task {task} payload decoding failed', + Context( + _deliveryLogContext( + envelope, + extra: { + 'error': error.toString(), + 'stack': stack.toString(), + 'reason': 'invalid-payload', + }, + ), + ), + ); + _emitEvent( + WorkerEvent( + type: WorkerEventType.failed, + envelope: envelope, + error: error, + stackTrace: stack, + data: const {'reason': 'invalid-payload'}, + ), + ); + await _signals.taskFailed( + envelope, + _workerInfoSnapshot, + error: error, + stackTrace: stack, + ); + return const _TaskCompletion( + state: TaskState.failed, + terminalWriteOwned: true, + ); } /// Processes control-plane revoke commands and updates the cache. @@ -4060,7 +4364,7 @@ class Worker { _latestRevocationVersion = entry.version; } applied.add(entry.taskId); - if (_activeDeliveries.containsKey(entry.taskId)) { + if (_deliveryTracker.containsEnvelopeId(entry.taskId)) { inflight.add(entry.taskId); } } @@ -4173,18 +4477,27 @@ class Worker { ControlQueueNames.broadcast(namespace), }; for (final queueName in controlQueues) { - if (_subscriptions.containsKey(queueName)) { + if (_consumer.contains(queueName)) { continue; } - final stream = broker.consume( - RoutingSubscription.singleQueue(queueName), + _consumer.subscribe( + name: queueName, + routing: RoutingSubscription.singleQueue(queueName), consumerName: '$_workerIdentifier-control', - ); - // Subscriptions are tracked and cancelled in _cancelAllSubscriptions(). - // ignore: cancel_subscriptions - final subscription = stream.listen( - (delivery) => unawaited(_processControlCommandDelivery(delivery)), - onError: (Object error, StackTrace stack) { + onDelivery: _processControlCommandDelivery, + onDeliveryError: (delivery, error, stack) { + stemLogger.warning( + 'Control command error: $error', + Context( + _logContext({ + 'queue': queueName, + 'task': delivery.envelope.id, + 'stack': stack.toString(), + }), + ), + ); + }, + onStreamError: (error, stack) { stemLogger.warning( 'Control channel error: $error', Context( @@ -4193,7 +4506,6 @@ class Worker { ); }, ); - _subscriptions[queueName] = subscription; } } @@ -4456,51 +4768,6 @@ class Worker { }; } - bool _shouldUseIsolate(TaskHandler handler) => - handler.isolateEntrypoint != null; - - Future _runInIsolate( - TaskHandler handler, - TaskContext context, - Envelope envelope, - Map args, { - Duration? hardTimeout, - }) async { - final entrypoint = handler.isolateEntrypoint; - if (entrypoint == null) { - return handler.call(context, args); - } - - final pool = await _ensureIsolatePool(); - - final outcome = await pool.execute( - entrypoint, - args, - envelope.headers, - envelope.meta, - envelope.attempt, - _controlHandler(context), - taskName: handler.name, - taskId: envelope.id, - hardTimeout: hardTimeout, - ); - - if (outcome is TaskExecutionSuccess) { - return outcome.value; - } else if (outcome is TaskExecutionRetry) { - throw outcome.request; - } else if (outcome is TaskExecutionFailure) { - Error.throwWithStackTrace(outcome.error, outcome.stackTrace); - } else if (outcome is TaskExecutionTimeout) { - throw TimeoutException( - 'hard time limit exceeded for ${outcome.taskName}', - outcome.limit, - ); - } - - throw StateError('Unexpected execution outcome: $outcome'); - } - /// Returns a [TaskControlHandler] that manages signals from isolate tasks. /// /// The handler processes [HeartbeatSignal]s, [ExtendLeaseSignal]s, @@ -4636,53 +4903,6 @@ class Worker { ); } - /// Lazily creates or returns the worker isolate pool. - Future _ensureIsolatePool() { - final existing = _isolatePool; - if (existing != null) return Future.value(existing); - final future = _poolFuture; - if (future != null) return future; - final creation = _createPool(); - _poolFuture = creation; - return creation; - } - - /// Creates and configures the [TaskIsolatePool] for this worker. - /// - /// Connects the pool's lifecycle events to the worker's internal state - /// for metrics tracking and logging. - Future _createPool() async { - final pool = - TaskIsolatePool( - size: _currentConcurrency, - onRecycle: _handleIsolateRecycle, - onSpawned: (isolateId) { - unawaited( - _signals.workerChildLifecycle( - _workerInfoSnapshot, - isolateId, - initializing: true, - ), - ); - }, - onDisposed: (isolateId) { - unawaited( - _signals.workerChildLifecycle( - _workerInfoSnapshot, - isolateId, - initializing: false, - ), - ); - }, - )..updateRecyclePolicy( - maxTasksPerIsolate: lifecycleConfig.maxTasksPerIsolate, - maxMemoryBytes: lifecycleConfig.maxMemoryPerIsolateBytes, - ); - await pool.start(); - _isolatePool = pool; - return pool; - } - static int _normalizeConcurrency(int? value) => math.max(1, value ?? Platform.numberOfProcessors); @@ -4886,6 +5106,26 @@ enum WorkerEventType { error, } +class _TaskCompletion { + const _TaskCompletion({ + required this.state, + required this.terminalWriteOwned, + }); + + final TaskState state; + final bool terminalWriteOwned; +} + +class _TerminalWriteResult { + const _TerminalWriteResult({ + required this.applied, + required this.status, + }); + + final bool applied; + final TaskStatus status; +} + /// Exception thrown when a task is revoked during execution. class TaskRevokedException implements Exception { /// Creates a revoked-task exception. @@ -4903,52 +5143,3 @@ class TaskRevokedException implements Exception { @override String toString() => 'Task $taskId revoked'; } - -/// Parsed rate limit specification for a specific task or queue. -/// -/// Encapsulates the token-bucket or sliding-window parameters used to -/// enforce throughput constraints. -class _RateSpec { - /// Creates a rate spec. - /// - /// [tokens] is the number allowed per [period]. - const _RateSpec({required this.tokens, required this.period}); - - /// The number of tokens allowed. - final int tokens; - - /// The period over which tokens apply. - final Duration period; -} - -/// Tracks an active task delivery during its lifecycle in the worker. -/// -/// This internal structure links a [Delivery] with its execution [Future], -/// start time, and any active lease timers. It is used to monitor in-flight -/// work and handle graceful shutdown. -class _ActiveDelivery { - /// Creates an active delivery record. - /// - /// [queue] is the queue name. [startedAt] is the start time. - _ActiveDelivery({ - required this.queue, - required this.startedAt, - required this.envelope, - required this.delivery, - }); - - /// The queue this delivery belongs to. - final String queue; - - /// When the task started. - final DateTime startedAt; - - /// The original envelope for this task. - final Envelope envelope; - - /// The underlying delivery from the broker. - final Delivery delivery; - - /// The last lease renewal time. - DateTime? lastLeaseRenewal; -} diff --git a/packages/stem/lib/src/worker/worker_acknowledgement.dart b/packages/stem/lib/src/worker/worker_acknowledgement.dart new file mode 100644 index 00000000..ee3c4d42 --- /dev/null +++ b/packages/stem/lib/src/worker/worker_acknowledgement.dart @@ -0,0 +1,50 @@ +import 'package:contextual/contextual.dart'; +import 'package:stem/src/core/contracts.dart'; +import 'package:stem/src/core/envelope.dart'; +import 'package:stem/src/observability/logging.dart'; +import 'package:stem/src/observability/metrics.dart'; + +/// Coordinates acknowledgements whose failure is recoverable after terminal +/// task state has been persisted. +/// +/// A broker ACK can be lost after a result backend write succeeds. In that +/// case the delivery may be redelivered, but the worker must not execute the +/// handler again. Keeping this path in one component makes that guarantee +/// explicit and gives all terminal ACK failures consistent telemetry. +class WorkerAcknowledgementCoordinator { + /// Creates an acknowledgement coordinator for [broker]. + const WorkerAcknowledgementCoordinator(this.broker); + + /// Broker used to acknowledge deliveries. + final QueueBroker broker; + + /// Attempts to acknowledge a delivery without masking already durable task + /// state when the broker is unavailable. + Future tryAcknowledge( + Delivery delivery, { + required Envelope envelope, + required String phase, + }) async { + try { + await broker.ack(delivery); + return true; + } on Object catch (error, stack) { + StemMetrics.instance.increment( + 'stem.acks.failed', + tags: {'task': envelope.name, 'queue': envelope.queue}, + ); + stemLogger.warning( + 'Task acknowledgement failed during {phase}', + Context({ + 'phase': phase, + 'task': envelope.name, + 'id': envelope.id, + 'queue': envelope.queue, + 'error': error.toString(), + 'stack': stack.toString(), + }), + ); + return false; + } + } +} diff --git a/packages/stem/lib/src/worker/worker_consumer_loop.dart b/packages/stem/lib/src/worker/worker_consumer_loop.dart new file mode 100644 index 00000000..a554173c --- /dev/null +++ b/packages/stem/lib/src/worker/worker_consumer_loop.dart @@ -0,0 +1,119 @@ +import 'dart:async'; + +import 'package:stem/src/core/contracts.dart'; +import 'package:stem/src/core/envelope.dart'; + +/// Handles broker subscription ownership for a worker. +/// +/// Task execution, control-command processing, and event reporting remain +/// owned by the worker. This class only manages subscription replacement, +/// cancellation, and the asynchronous error boundary around stream data +/// callbacks. +class WorkerConsumerLoop { + /// Creates a consumer loop for [broker]. + WorkerConsumerLoop({required QueueBroker broker}) : _broker = broker; + + final QueueBroker _broker; + final Map> _subscriptions = {}; + final Set _queueSubscriptionNames = {}; + + /// Names of task-queue subscriptions currently owned by this loop. + Iterable get queueSubscriptionNames => + List.unmodifiable(_queueSubscriptionNames); + + /// Whether a subscription with [name] is already registered. + bool contains(String name) => _subscriptions.containsKey(name); + + /// Replaces all task-queue subscriptions while leaving control subscriptions + /// intact. + Future replaceQueueSubscriptions({ + required Iterable queues, + required Iterable broadcastChannels, + required int prefetch, + required Future Function(Delivery delivery) onDelivery, + required void Function(Delivery delivery, Object error, StackTrace stack) + onDeliveryError, + required void Function(Object error, StackTrace stack) onStreamError, + String? consumerName, + }) async { + await cancel(_queueSubscriptionNames); + + final resolvedQueues = queues.toList(growable: false); + final resolvedBroadcasts = broadcastChannels.toList(growable: false); + for (var index = 0; index < resolvedQueues.length; index += 1) { + final queueName = resolvedQueues[index]; + subscribe( + name: queueName, + routing: RoutingSubscription( + queues: [queueName], + broadcastChannels: index == 0 ? resolvedBroadcasts : const [], + ), + prefetch: prefetch, + consumerName: consumerName, + queueSubscription: true, + onDelivery: onDelivery, + onDeliveryError: onDeliveryError, + onStreamError: onStreamError, + ); + } + } + + /// Adds one subscription, typically for a worker control queue. + void subscribe({ + required String name, + required RoutingSubscription routing, + required Future Function(Delivery delivery) onDelivery, + required void Function(Delivery delivery, Object error, StackTrace stack) + onDeliveryError, + required void Function(Object error, StackTrace stack) onStreamError, + int prefetch = 1, + String? consumerName, + bool queueSubscription = false, + }) { + if (_subscriptions.containsKey(name)) return; + + final stream = _broker.consume( + routing, + prefetch: prefetch, + consumerName: consumerName, + ); + // The loop owns all subscriptions and cancels them from [cancel] or + // [close], so this is intentionally not left to a caller. + // ignore: cancel_subscriptions + final subscription = stream.listen( + (delivery) { + final task = onDelivery(delivery); + unawaited( + task.catchError((Object error, StackTrace stack) { + onDeliveryError(delivery, error, stack); + }), + ); + }, + onError: onStreamError, + ); + _subscriptions[name] = subscription; + if (queueSubscription) { + _queueSubscriptionNames.add(name); + } + } + + /// Cancels subscriptions named in [names]. + Future cancel(Iterable names) async { + for (final name in List.from(names)) { + final subscription = _subscriptions.remove(name); + _queueSubscriptionNames.remove(name); + if (subscription == null) continue; + try { + await subscription.cancel(); + } on Object { + // A broker may already have closed its stream during shutdown. The + // worker owns the lifecycle, so cancellation remains best effort. + } + } + } + + /// Cancels every subscription owned by this loop. + Future close() async { + await cancel(_subscriptions.keys); + } +} diff --git a/packages/stem/lib/src/worker/worker_delivery_tracker.dart b/packages/stem/lib/src/worker/worker_delivery_tracker.dart new file mode 100644 index 00000000..ce307c6a --- /dev/null +++ b/packages/stem/lib/src/worker/worker_delivery_tracker.dart @@ -0,0 +1,120 @@ +import 'package:stem/src/core/envelope.dart'; + +/// Tracks deliveries currently being executed by a worker. +/// +/// Keeping active-delivery accounting outside the worker makes shutdown, +/// heartbeat, and observability code depend on one lifecycle component rather +/// than mutating several loosely-coupled maps. Worker-local handles are used +/// because some durable brokers reuse a row ID as the receipt after lease +/// expiry. +class WorkerDeliveryTracker { + /// Active deliveries keyed by a worker-local tracking handle. + final Map _active = {}; + int _nextTrackingHandle = 0; + final Map _inflightPerQueue = {}; + int _inflight = 0; + + /// Active deliveries keyed by a worker-local tracking handle. + Map get active => _active; + + /// In-flight counts grouped by queue. + Map get inflightPerQueue => _inflightPerQueue; + + /// Total number of active deliveries. + int get inflight => _inflight; + + /// Adds a delivery to the active set. + WorkerActiveDelivery track( + Delivery delivery, { + required DateTime startedAt, + }) { + final envelope = delivery.envelope; + final active = WorkerActiveDelivery( + queue: envelope.queue, + startedAt: startedAt, + envelope: envelope, + delivery: delivery, + ); + _active[++_nextTrackingHandle] = active; + _inflight += 1; + _inflightPerQueue[envelope.queue] = + (_inflightPerQueue[envelope.queue] ?? 0) + 1; + return active; + } + + /// Removes a delivery from the active set, if it is still tracked. + WorkerActiveDelivery? release(Delivery delivery) { + int? handle; + for (final entry in _active.entries) { + if (identical(entry.value.delivery, delivery)) { + handle = entry.key; + break; + } + } + if (handle == null) return null; + final active = _active.remove(handle); + if (active == null) return null; + + _inflight = _inflight > 0 ? _inflight - 1 : 0; + final queueCount = (_inflightPerQueue[active.queue] ?? 0) - 1; + if (queueCount <= 0) { + _inflightPerQueue.remove(active.queue); + } else { + _inflightPerQueue[active.queue] = queueCount; + } + return active; + } + + /// Returns the active delivery for [envelopeId], if any. + WorkerActiveDelivery? forEnvelopeId(String envelopeId) { + for (final delivery in _active.values) { + if (delivery.envelope.id == envelopeId) return delivery; + } + return null; + } + + /// Returns the active record for the exact broker [delivery]. + WorkerActiveDelivery? forDelivery(Delivery delivery) { + for (final active in _active.values) { + if (identical(active.delivery, delivery)) return active; + } + return null; + } + + /// Whether at least one delivery for [envelopeId] is active. + bool containsEnvelopeId(String envelopeId) => + _active.values.any((delivery) => delivery.envelope.id == envelopeId); + + /// Clears all active-delivery accounting during worker shutdown. + void clear() { + _active.clear(); + _inflightPerQueue.clear(); + _inflight = 0; + } +} + +/// A delivery and its worker-local execution timestamps. +class WorkerActiveDelivery { + /// Creates an active delivery record. + WorkerActiveDelivery({ + required this.queue, + required this.startedAt, + required this.envelope, + required this.delivery, + }); + + /// Queue containing the delivery. + final String queue; + + /// Time at which execution began. + final DateTime startedAt; + + /// Original task envelope. + final Envelope envelope; + + /// Broker delivery wrapper. + final Delivery delivery; + + /// Most recent successful lease renewal. + DateTime? lastLeaseRenewal; +} diff --git a/packages/stem/lib/src/worker/worker_execution_supervisor.dart b/packages/stem/lib/src/worker/worker_execution_supervisor.dart new file mode 100644 index 00000000..14e34adb --- /dev/null +++ b/packages/stem/lib/src/worker/worker_execution_supervisor.dart @@ -0,0 +1,155 @@ +import 'dart:async'; + +import 'package:stem/src/core/contracts.dart'; +import 'package:stem/src/core/envelope.dart'; +import 'package:stem/src/worker/isolate_pool.dart'; +import 'package:stem/src/worker/worker_config.dart'; + +/// Owns isolate-backed task execution and its pool lifecycle for a worker. +/// +/// Delivery acknowledgement, task state persistence, retries, and revocation +/// remain the responsibility of the owning worker. Keeping this boundary +/// focused makes +/// the execution guarantee explicit: isolate handlers can be terminated by +/// disposing their isolate, while inline handlers can only stop being awaited +/// after a timeout. +class WorkerExecutionSupervisor { + /// Creates an execution supervisor with the worker's isolate settings. + WorkerExecutionSupervisor({ + required int concurrency, + required WorkerLifecycleConfig lifecycle, + required void Function(IsolateRecycleEvent event) onRecycle, + required FutureOr Function(int isolateId) onSpawned, + required FutureOr Function(int isolateId) onDisposed, + }) : _concurrency = concurrency, + _lifecycle = lifecycle, + _onRecycle = onRecycle, + _onSpawned = onSpawned, + _onDisposed = onDisposed; + + int _concurrency; + final WorkerLifecycleConfig _lifecycle; + final void Function(IsolateRecycleEvent event) _onRecycle; + final FutureOr Function(int isolateId) _onSpawned; + final FutureOr Function(int isolateId) _onDisposed; + + TaskIsolatePool? _pool; + Future? _poolFuture; + bool _disposed = false; + + /// Number of active isolates, or zero before the pool is created. + int get activeIsolates => _pool?.activeCount ?? 0; + + /// Executes [handler] according to its declared [TaskExecutionMode]. + Future execute({ + required TaskHandler handler, + required TaskContext context, + required Envelope envelope, + required Map args, + required TaskControlHandler controlHandler, + Duration? hardTimeout, + }) async { + if (handler.executionMode == TaskExecutionMode.inline) { + final future = handler.call(context, args); + if (hardTimeout == null) return future; + return future.timeout( + hardTimeout, + onTimeout: () => throw TimeoutException( + 'hard time limit exceeded for ${handler.name}', + ), + ); + } + + final entrypoint = handler.isolateEntrypoint; + if (entrypoint == null) { + throw StateError( + 'Task "${handler.name}" declares isolate execution but does not ' + 'provide an isolateEntrypoint.', + ); + } + + final outcome = await (await _ensurePool()).execute( + entrypoint, + args, + envelope.headers, + envelope.meta, + envelope.attempt, + controlHandler, + taskName: handler.name, + taskId: envelope.id, + hardTimeout: hardTimeout, + ); + + if (outcome is TaskExecutionSuccess) return outcome.value; + if (outcome is TaskExecutionRetry) throw outcome.request; + if (outcome is TaskExecutionFailure) { + Error.throwWithStackTrace(outcome.error, outcome.stackTrace); + } + if (outcome is TaskExecutionTimeout) { + throw TimeoutException( + 'hard time limit exceeded for ${outcome.taskName}', + outcome.limit, + ); + } + + throw StateError('Unexpected execution outcome: $outcome'); + } + + /// Resizes an already-created pool, if one exists. + Future resize(int concurrency) async { + _concurrency = concurrency; + await _pool?.resize(concurrency); + } + + /// Disposes the isolate pool and clears its lazy creation state. + Future dispose() async { + _disposed = true; + final pool = _pool; + final pending = _poolFuture; + _pool = null; + _poolFuture = null; + var resolvedPool = pool; + if (resolvedPool == null && pending != null) { + try { + resolvedPool = await pending; + } on Object { + // Pool creation failures are already reported to the execution + // future; shutdown should still complete and release its state. + } + } + if (resolvedPool != null) await resolvedPool.dispose(); + } + + Future _ensurePool() { + if (_disposed) { + throw StateError('Task execution supervisor has been disposed.'); + } + final existing = _pool; + if (existing != null) return Future.value(existing); + final pending = _poolFuture; + if (pending != null) return pending; + final creation = _createPool(); + _poolFuture = creation; + return creation; + } + + Future _createPool() async { + final pool = + TaskIsolatePool( + size: _concurrency, + onRecycle: _onRecycle, + onSpawned: _onSpawned, + onDisposed: _onDisposed, + )..updateRecyclePolicy( + maxTasksPerIsolate: _lifecycle.maxTasksPerIsolate, + maxMemoryBytes: _lifecycle.maxMemoryPerIsolateBytes, + ); + await pool.start(); + if (_disposed) { + await pool.dispose(); + throw StateError('Task execution supervisor was disposed during start.'); + } + _pool = pool; + return pool; + } +} diff --git a/packages/stem/lib/src/worker/worker_lease_coordinator.dart b/packages/stem/lib/src/worker/worker_lease_coordinator.dart new file mode 100644 index 00000000..f226b56c --- /dev/null +++ b/packages/stem/lib/src/worker/worker_lease_coordinator.dart @@ -0,0 +1,181 @@ +import 'dart:async'; + +import 'package:stem/src/core/contracts.dart'; +import 'package:stem/src/core/envelope.dart'; + +/// Coordinates visibility/lease renewal for active worker deliveries. +/// +/// Lease renewal is deliberately isolated from task execution. A failed +/// renewal is reported to the worker and retried on a shorter interval rather +/// than waiting until the original lease deadline. The broker remains the +/// source of truth: if a lease is ultimately lost, normal at-least-once +/// delivery and terminal result deduplication handle the resulting redelivery. +class WorkerLeaseCoordinator { + /// Creates a lease coordinator for [broker]. + WorkerLeaseCoordinator({ + required QueueBroker broker, + required void Function(Delivery delivery) onLeaseUpdated, + required void Function(Delivery delivery) onLeaseRenewed, + required void Function( + Delivery delivery, + Object error, + StackTrace stackTrace, + ) + onRenewalFailure, + DateTime Function()? now, + this.minimumInterval = const Duration(seconds: 1), + this.maximumInterval = const Duration(seconds: 30), + }) : _broker = broker, + _onLeaseUpdated = onLeaseUpdated, + _onLeaseRenewed = onLeaseRenewed, + _onRenewalFailure = onRenewalFailure, + _now = now ?? DateTime.now { + if (minimumInterval <= Duration.zero) { + throw ArgumentError.value( + minimumInterval, + 'minimumInterval', + 'must be positive', + ); + } + if (maximumInterval < minimumInterval) { + throw ArgumentError.value( + maximumInterval, + 'maximumInterval', + 'must be at least minimumInterval', + ); + } + } + + final QueueBroker _broker; + final void Function(Delivery delivery) _onLeaseUpdated; + final void Function(Delivery delivery) _onLeaseRenewed; + final void Function( + Delivery delivery, + Object error, + StackTrace stackTrace, + ) + _onRenewalFailure; + final DateTime Function() _now; + + /// Lower bound for renewal intervals when the lease permits it. + /// + /// A lease deadline takes precedence: short leases must be renewed before + /// expiry even when that requires using an interval below this value. + final Duration minimumInterval; + + /// Upper bound for renewal intervals. + final Duration maximumInterval; + + final Map _timers = {}; + final Set _renewalsInFlight = {}; + final Map _generations = {}; + int _generationSeed = 0; + + /// Schedules renewal based on the delivery's current lease expiry. + void schedule(Delivery delivery) { + final expiresAt = delivery.leaseExpiresAt; + if (expiresAt == null) return; + final remaining = expiresAt.difference(_now()).inMilliseconds; + if (remaining <= 0) return; + + final halfRemaining = Duration(milliseconds: remaining ~/ 2); + final interval = _clampInterval(halfRemaining); + final leaseDuration = _leaseDuration( + delivery, + Duration(milliseconds: remaining), + ); + _start(delivery, interval, leaseDuration); + _onLeaseUpdated(delivery); + } + + /// Restarts renewal after an explicit lease extension. + void restart(Delivery delivery, Duration leaseDuration) { + final halfDuration = Duration( + milliseconds: leaseDuration.inMilliseconds ~/ 2, + ); + final interval = _clampInterval(halfDuration); + _start(delivery, interval, leaseDuration); + _onLeaseUpdated(delivery); + } + + /// Cancels renewal for one delivery. + void cancel(Delivery delivery) { + _timers.remove(delivery)?.cancel(); + _generations.remove(delivery); + } + + /// Cancels all renewal timers. + void cancelAll() { + for (final timer in _timers.values) { + timer.cancel(); + } + _timers.clear(); + _generations.clear(); + } + + Duration _clampInterval(Duration value) { + // The value is derived from half of the remaining lease, so preserving it + // is safer than applying the normal minimum to short-lived leases. + if (value <= Duration.zero) return minimumInterval; + if (value > maximumInterval) return maximumInterval; + return value; + } + + Duration _leaseDuration(Delivery delivery, Duration remaining) { + final configured = delivery.envelope.visibilityTimeout; + if (configured != null && configured > Duration.zero) { + return configured; + } + return remaining; + } + + void _start( + Delivery delivery, + Duration interval, + Duration leaseDuration, + ) { + cancel(delivery); + final generation = ++_generationSeed; + _generations[delivery] = generation; + final timer = Timer.periodic(interval, (_) { + if (!_renewalsInFlight.add(delivery)) return; + unawaited(_renew(delivery, interval, leaseDuration, generation)); + }); + _timers[delivery] = timer; + } + + Future _renew( + Delivery delivery, + Duration interval, + Duration leaseDuration, + int generation, + ) async { + try { + await _broker.extendLease(delivery, leaseDuration); + if (!_isCurrent(delivery, generation)) return; + _onLeaseUpdated(delivery); + _onLeaseRenewed(delivery); + } on Object catch (error, stackTrace) { + if (!_isCurrent(delivery, generation)) return; + _onRenewalFailure(delivery, error, stackTrace); + _start(delivery, _retryInterval(interval), leaseDuration); + } finally { + _renewalsInFlight.remove(delivery); + } + } + + bool _isCurrent(Delivery delivery, int generation) => + _generations[delivery] == generation; + + Duration _retryInterval(Duration interval) { + final milliseconds = interval.inMilliseconds; + if (milliseconds <= 1) return const Duration(milliseconds: 1); + final retryMilliseconds = milliseconds ~/ 2; + final minimumMilliseconds = minimumInterval.inMilliseconds; + return Duration( + milliseconds: retryMilliseconds < minimumMilliseconds + ? retryMilliseconds + : minimumMilliseconds, + ); + } +} diff --git a/packages/stem/lib/src/workflow/runtime/workflow_runtime.dart b/packages/stem/lib/src/workflow/runtime/workflow_runtime.dart index 4e5015ed..a6e790cd 100644 --- a/packages/stem/lib/src/workflow/runtime/workflow_runtime.dart +++ b/packages/stem/lib/src/workflow/runtime/workflow_runtime.dart @@ -2400,7 +2400,7 @@ Map _coerceEventPayload(String topic, Object? payload) { } if (payload is Map) { final encoded = {}; - for (final MapEntry(key: key, value: value) in payload.entries) { + for (final MapEntry(:key, :value) in payload.entries) { if (key is! String) { throw ArgumentError.value( payload, diff --git a/packages/stem/lib/src/workflow/workflow.dart b/packages/stem/lib/src/workflow/workflow.dart index b1d1931a..55f10c22 100644 --- a/packages/stem/lib/src/workflow/workflow.dart +++ b/packages/stem/lib/src/workflow/workflow.dart @@ -1,5 +1,3 @@ -export 'package:stem_memory/stem_memory.dart' - show InMemoryEventBus, InMemoryWorkflowStore; export 'annotations.dart'; export 'core/event_bus.dart'; export 'core/flow.dart'; diff --git a/packages/stem/lib/stable.dart b/packages/stem/lib/stable.dart new file mode 100644 index 00000000..26d1e459 --- /dev/null +++ b/packages/stem/lib/stable.dart @@ -0,0 +1,38 @@ +/// Stable, high-level Stem API. +/// +/// New applications should prefer this library. The historical +/// `package:stem/stem.dart` barrel remains available for compatibility, while +/// low-level transport, signal, and instrumentation APIs are collected under +/// `package:stem/advanced.dart`. +library; + +export 'src/bootstrap/factories.dart'; +export 'src/bootstrap/stem_app.dart'; +export 'src/bootstrap/stem_client.dart'; +export 'src/bootstrap/stem_module.dart'; +export 'src/bootstrap/stem_stack.dart'; +export 'src/bootstrap/workflow_app.dart'; +export 'src/canvas/canvas.dart'; +export 'src/core/contracts.dart' + hide + InMemoryTaskRegistry, + TaskArgsEncoder, + TaskEnqueuer, + TaskEnqueuerBuilderExtension, + TaskHandler, + TaskInputContext, + TaskInputContextArgs, + TaskRegistrationEvent, + TaskRegistry; +export 'src/core/payload_codec.dart'; +export 'src/core/payload_map.dart'; +export 'src/core/retry.dart'; +export 'src/core/stem.dart'; +export 'src/core/task_invocation.dart'; +export 'src/core/task_result.dart'; +export 'src/routing/routing_config.dart'; +export 'src/scheduler/schedule_spec.dart'; +export 'src/security/signing.dart'; +export 'src/worker/worker.dart'; +export 'src/worker/worker_config.dart'; +export 'src/workflow/workflow.dart'; diff --git a/packages/stem/lib/stem.dart b/packages/stem/lib/stem.dart index b0cd219c..9188b62c 100644 --- a/packages/stem/lib/stem.dart +++ b/packages/stem/lib/stem.dart @@ -1,8 +1,7 @@ /// Distributed task queue and worker framework for Dart. /// -/// Stem is a robust, production-ready background job system designed to -/// orchestrate complex asynchronous workflows across multiple processes -/// and machines. +/// Stem is an experimental Dart-native background job system designed to +/// orchestrate asynchronous workflows across multiple processes and machines. /// /// ## Key Concepts /// @@ -29,25 +28,34 @@ /// * **Coordination**: High-level workflows like Chords, Groups, and Chains, /// plus unique task enforcement. /// +/// Stem-owned structured logging is available from the optional +/// `package:stem/observability.dart` entrypoint; its logging dependency stays +/// an implementation detail. +/// +/// New application code may use `package:stem/stable.dart` for the narrower +/// high-level surface. This historical barrel remains source-compatible, and +/// `package:stem/advanced.dart` collects the low-level integration APIs. +/// /// ## Quick Start /// /// ```dart -/// import 'package:stem/stem.dart'; +/// import 'package:stem/stable.dart'; +/// import 'package:stem/memory.dart'; /// /// void main() async { /// // 1. Define a typed task /// final addDefinition = TaskDefinition, int>( /// name: 'add_task', /// encodeArgs: (args) => args, +/// decodeArgs: (args) => args.cast(), /// ); /// /// // 2. Define the handler -/// final addHandler = FunctionTaskHandler( -/// name: 'add_task', -/// entrypoint: (context, args) async { -/// return (args['a'] as int) + (args['b'] as int); -/// }, -/// ); +/// final addHandler = addDefinition.handler( +/// entrypoint: (context, args) async { +/// return args['a']! + args['b']!; +/// }, +/// ); /// /// // 3. Initialize Stem with a broker (e.g., In-Memory for testing) /// final stem = Stem( @@ -72,14 +80,6 @@ import 'package:stem/src/core/stem.dart'; import 'package:stem/src/scheduler/beat.dart'; import 'package:stem/src/worker/worker.dart'; -export 'package:contextual/contextual.dart' show Context, Level, Logger; -export 'package:stem_memory/stem_memory.dart' - show - InMemoryBroker, - InMemoryLockStore, - InMemoryResultBackend, - InMemoryRevokeStore, - InMemoryScheduleStore; export 'src/backend/encoding_result_backend.dart'; export 'src/bootstrap/factories.dart'; export 'src/bootstrap/stem_app.dart'; @@ -92,6 +92,7 @@ export 'src/control/control_messages.dart'; export 'src/control/file_revoke_store.dart'; export 'src/control/revoke_store.dart'; export 'src/core/chord_metadata.dart'; +export 'src/core/chord_policy.dart'; export 'src/core/clock.dart' hide FakeStemClock; export 'src/core/config.dart'; export 'src/core/contracts.dart'; @@ -111,7 +112,6 @@ export 'src/core/unique_task_coordinator.dart'; export 'src/observability/config.dart'; export 'src/observability/heartbeat.dart'; export 'src/observability/heartbeat_transport.dart'; -export 'src/observability/logging.dart'; export 'src/observability/metrics.dart'; export 'src/observability/snapshots.dart'; export 'src/observability/tracing.dart'; @@ -127,7 +127,6 @@ export 'src/signals/middleware.dart'; export 'src/signals/payloads.dart'; export 'src/signals/signal.dart'; export 'src/signals/stem_signals.dart'; -export 'src/testing/fake_stem.dart'; export 'src/worker/worker.dart'; export 'src/worker/worker_config.dart'; export 'src/workflow/workflow.dart'; diff --git a/packages/stem/pubspec.yaml b/packages/stem/pubspec.yaml index b526d43e..6ebf233e 100644 --- a/packages/stem/pubspec.yaml +++ b/packages/stem/pubspec.yaml @@ -1,6 +1,6 @@ name: stem -description: "Stem is a Dart-native background job platform with Redis Streams, retries, scheduling, observability, and security tooling." -version: 0.2.1 +description: "Experimental Dart-native background job platform with Redis Streams, retries, scheduling, observability, and security tooling." +version: 0.3.0 repository: https://github.com/kingwill101/stem resolution: workspace environment: @@ -17,7 +17,6 @@ dependencies: dartastic_opentelemetry_api: ^0.8.8 glob: ^2.1.3 meta: ^1.18.0 - stem_memory: ^0.1.0 timezone: ^0.11.0 uuid: ^4.5.2 yaml: ^3.1.3 @@ -30,11 +29,11 @@ dev_dependencies: lints: ^6.0.0 path: ^1.9.1 property_testing: ^0.3.2 - stem_adapter_tests: ^0.1.0 - # stem_cli: ^0.1.0 - # stem_postgres: ^0.1.0 - # stem_redis: ^0.1.0 - # stem_sqlite: ^0.1.0 + stem_adapter_tests: ^0.2.0 + # stem_cli: ^0.2.0 + # stem_postgres: ^0.2.0 + # stem_redis: ^0.2.0 + # stem_sqlite: ^0.2.0 test: ^1.29.0 very_good_analysis: ^10.0.0 @@ -49,10 +48,3 @@ topics: - background-processing - message-queue - observability - - - -false_secrets: - - /example/microservice/certs/* - - /example/postgres_tls/certs/* - - /docker/testing/postgres/certs/* diff --git a/packages/stem/spec.md b/packages/stem/spec.md deleted file mode 100644 index 4b0f5627..00000000 --- a/packages/stem/spec.md +++ /dev/null @@ -1,1700 +0,0 @@ -Skip to content -You said: -What it takes to build a cerlery and beat like package in dart. - -Would possibly be a dameon with something like the bus package etc - -We'd have stores similar to laravel where a store implements a interface for a data store etc. - -We'd need to define all the interfaces needed for it to be very practical and useful alternative. - -Find all the good parts in celery and other similar solutions which we'll build out from scratch in dart. - -But of course we'd need a very good plan. - - -Also a detailed plan about what it would mean for it to be production ready. - - - -ChatGPT said: -> **Documentation:** The living developer, operations, and scaling guides now live in the Docusaurus site under `.site/`. Run `cd .site && npm start` to browse them locally. -Request for GPT-5 Pro -Reasoned for 5m 1s - -Below is a concrete, end‑to‑end plan to build a Celery + Beat–style background job system in Dart—including the core architecture, key interfaces, feature set (inspired by Celery, Sidekiq, Dramatiq, BullMQ, RQ), and a production‑readiness checklist. I’ll call the project “Stem” for the sake of naming in examples. -0) What we’re aiming for (scope & semantics) - -Goals - - A language‑native Dart library + daemon(s) for async task execution and periodic scheduling (Celery + Celery Beat analogue). - - At‑least‑once delivery semantics by default, with clear, ergonomic tools for idempotency, retries, visibility/lease renewal, dead‑lettering, rate limiting, and backpressure. (This mirrors best practice in Celery/Sidekiq ecosystems.) - Celery Documentation+1 - - Pluggable brokers (Redis Streams, RabbitMQ, SQS first), result backends (Redis, Postgres), and schedule stores (Redis ZSET, Postgres). - Celery Documentation - - A small “canvas” layer for chains, groups, chords (Celery’s canvas), implemented via the result backend and a chord‑aggregator. - Celery Documentation - - A separate Beat‑like scheduler process with configurable stores (file/Redis/SQL), similar to Celery Beat. - Celery Documentation - -Non‑goals (v1.0) - - Exactly‑once semantics (document limitations + patterns to simulate). - Stack Overflow - - Heavy distributed workflow/DAG engine. Keep the canvas light. - -1) What to borrow from Celery & friends (the “good parts”) - - Task API: function‑first registration, namespacing, per‑task options: retries, time limits, rate limits, queue routing. - Celery Documentation+2Celery Documentation+2 - - Beat (periodic scheduling): cron/interval entries, pluggable stores. - Celery Documentation - - Canvas: chains (A→B→C), groups (A,B,C in parallel), chords (group + callback when all done). - Celery Documentation - - Retries with backoff & DLQ: automatic retries + exponential backoff; move permanently failing jobs to “dead” set/queue with UI to replay (pattern seen in Sidekiq/BullMQ). - GitHub+1 - - Prefetch limits & worker pools: control prefetch to avoid hoarding, combine with concurrency for fairness. - Celery Documentation - - Soft/hard time limits: gracefully interrupt vs. hard kill. - Celery Documentation+1 - - Brokers/backends: Redis/RabbitMQ/SQS for broker; Redis/DB for results (document tradeoffs). - Celery Documentation - - Observability: task lifecycle events, metrics, tracing, admin UI (Celery has Flower; Sidekiq has a web UI). - GitHub - -Also, things to improve vs. Celery: - - Strong defaults for visibility/lease renewal, reject_on_worker_lost, and acks_late style behavior to reduce lost tasks. - Celery Documentation+1 - - Straightforward global rate limiting (document how we approximate across workers with Redis tokens). (Celery’s global rate limiting is a common pain point.) - Hatchet Documentation - - First‑class, well‑documented idempotency helpers (dedupe keys, outbox pattern). - GitGuardian Blog - -2) High‑level architecture (Dart) - - +------------------+ +--------------------+ - Enqueuers --> | Client Library | --XADD--> | Broker Adapter | (Redis Streams / RabbitMQ / SQS) - +------------------+ +--------------------+ - | | - v v - +--------------+ +------------------+ - | Result API |<---store----- | Result Backend | (Redis / Postgres) - +--------------+ +------------------+ - ^ - | group/chord updates - | -+------------------+ | +-------------------------+ -| Beat scheduler |---- | ---> | Chord Aggregator | -| (cron/interval) | | | (wait-all and call) | -+------------------+ | +-------------------------+ - emits task to broker when ready - -+------------------+ consume & execute +-------------------+ -| Worker Daemon | <-------------------------- | Broker Adapter | -| (Isolates pool) | --ack/nack/extend lease--> | (subsystem) | -+------------------+ +-------------------+ - - Workers use Dart isolates for true parallelism on a single host. - Dart - - A lightweight event bus can be used internally for in‑process signals (e.g., progress) but is not a distributed transport. (Event bus packages are in‑memory publish/subscribe.) - Dart packages - -3) Core interfaces (Dart) - - These are the “stores” / “drivers” you mentioned—each backend implements a common interface (Laravel‑style). - -3.1 Message / envelope - -// Unique task identity uses UUID/ULID. -class Envelope { - final String id; // task id - final String name; // task name - final Map args; // positional/keyword collapsed - final Map headers; // trace-id, tenant-id, etc. - final DateTime enqueuedAt; - final DateTime? notBefore; // ETA/delay - final int priority; // 0..9 (adapter may map) - final int attempt; // current attempt # - final int maxRetries; - final Duration? visibilityTimeout; // extend/renew lease semantics - final String queue; - - Envelope({ - required this.id, - required this.name, - required this.args, - required this.headers, - required this.enqueuedAt, - this.notBefore, - this.priority = 0, - this.attempt = 0, - this.maxRetries = 0, - this.visibilityTimeout, - required this.queue, - }); -} - -3.2 Broker (queue transport) - -abstract class Broker { - Future publish(Envelope msg, {String? queue}); - Stream consume(String queue, {int prefetch = 1, String? consumerGroup}); - Future ack(Delivery d); - Future nack(Delivery d, {bool requeue = true}); - Future deadLetter(Delivery d, {String? reason}); - Future purge(String queue); - - bool get supportsDelayed; // ETA/delay without sidecar - bool get supportsPriority; -} - -// Runtime wrapper: message + receipt/handle/metadata. -class Delivery { - final Envelope message; - final String receipt; // ack handle / tag - final DateTime leaseExpireAt; // if applicable - - Delivery(this.message, this.receipt, this.leaseExpireAt); -} - -Initial adapters & building blocks (Dart packages): - - Redis Streams (XADD/XREADGROUP/XACK/XCLAIM): simple, fast, easy to start. (Use a Redis client.) - Dart packages - - RabbitMQ (AMQP 0‑9‑1) (queues, routing keys, per‑queue priority, TTL+DLX for delay): dart_amqp. - Dart packages - - Amazon SQS (visibility timeout, DelaySeconds, DLQ): aws_sqs_api. - Dart packages - -3.3 Result backend - -enum TaskState { queued, running, succeeded, failed, cancelled, retried } - -class TaskStatus { - final String id; - final TaskState state; - final Object? result; - final Object? error; - final int attempt; - final DateTime updatedAt; - - TaskStatus(this.id, this.state, this.result, this.error, this.attempt, this.updatedAt); -} - -abstract class ResultBackend { - Future set(String taskId, TaskState state, { - Object? result, - Object? error, - int attempt = 0, - Duration? ttl, - Map? meta, - }); - - Future get(String taskId); - Stream watch(String taskId); // optional - Future expire(String taskId, Duration ttl); -} - -3.4 Schedule (“Beat”) store - -class ScheduleEntry { - final String id; // human-readable name - final String taskName; - final Map args; - final String spec; // cron string | "every:5m" - final String queue; - final bool enabled; - final DateTime? lastRunAt; - final Duration? jitter; // +/- jitter - - ScheduleEntry({ - required this.id, - required this.taskName, - required this.args, - required this.spec, - required this.queue, - this.enabled = true, - this.lastRunAt, - this.jitter, - }); -} - -abstract class ScheduleStore { - Future> due(DateTime now, {int limit = 100}); - Future upsert(ScheduleEntry entry); - Future remove(String id); -} - -3.5 Worker & registry - -class TaskOptions { - final String queue; - final int maxRetries; - final Duration? softTimeLimit; - final Duration? hardTimeLimit; - final String? rate; // e.g. "5/s", "300/m" - final bool unique; // de-dupe lock per signature - final Duration? uniqueFor; // hold lock for duration - final int priority; - final bool acksLate; // ack after success - - const TaskOptions({ - this.queue = 'default', - this.maxRetries = 0, - this.softTimeLimit, - this.hardTimeLimit, - this.rate, - this.unique = false, - this.uniqueFor, - this.priority = 0, - this.acksLate = true, - }); -} - -abstract class TaskHandler { - String get name; - TaskOptions get options; - TaskMetadata get metadata => const TaskMetadata(); - Future call(TaskContext ctx, Map args); -} - -class TaskMetadata { - final String? description; - final List tags; - final bool idempotent; - final Map attributes; - - const TaskMetadata({ - this.description, - this.tags = const [], - this.idempotent = false, - this.attributes = const {}, - }); -} - -abstract class TaskRegistry { - void register(TaskHandler handler, {bool overrideExisting = false}); - TaskHandler? resolve(String name); - Iterable get handlers; -} - -typedef TaskArgsEncoder = Map Function(TArgs args); - -class TaskDefinition { - const TaskDefinition({ - required this.name, - required TaskArgsEncoder encodeArgs, - this.defaultOptions = const TaskOptions(), - this.metadata = const TaskMetadata(), - }) : _encodeArgs = encodeArgs; - - final String name; - final TaskOptions defaultOptions; - final TaskMetadata metadata; - final TaskArgsEncoder _encodeArgs; - - TaskCall call( - TArgs args, { - Map headers = const {}, - TaskOptions? options, - DateTime? notBefore, - Map meta = const {}, - }) => - TaskCall( - definition: this, - args: args, - headers: headers, - options: options, - notBefore: notBefore, - meta: meta, - ); - - Map encodeArgs(TArgs args) => _encodeArgs(args); -} - -class TaskCall { - const TaskCall({ - required this.definition, - required this.args, - required this.headers, - this.options, - this.notBefore, - this.meta = const {}, - }); - - final TaskDefinition definition; - final TArgs args; - final Map headers; - final TaskOptions? options; - final DateTime? notBefore; - final Map meta; - - String get name => definition.name; - Map encodeArgs() => definition.encodeArgs(args); - TaskOptions resolveOptions() => options ?? definition.defaultOptions; -} - -class TaskContext { - final String id; - final int attempt; - final void Function() heartbeat; - final Future Function(Duration) extendLease; - final Future Function(double pct, {Map? data}) progress; - - TaskContext({ - required this.id, - required this.attempt, - required this.heartbeat, - required this.extendLease, - required this.progress, - }); -} - -3.6 Cross‑cutting “stores” (Laravel‑style drivers) - -abstract class LockStore { - Future acquire(String key, {Duration ttl = const Duration(seconds: 30), String? owner}); -} -abstract class Lock { - String get key; - Future renew(Duration ttl); - Future release(); -} - -abstract class RateLimiter { - /// Return true if acquired 'tokens' for interval window. - Future tryAcquire(String key, {int tokens = 1, required Duration per}); -} - -abstract class RetryStrategy { - Duration nextDelay(int attempt, Object error); -} - -abstract class Middleware { - Future onEnqueue(Envelope m, Future Function() next); - Future onConsume(Delivery d, Future Function() next); - Future onExecute(TaskContext ctx, Future Function() next); - Future onError(TaskContext ctx, Object error, StackTrace st); -} - - These interfaces let you build “drivers” for Redis/RabbitMQ/SQS, Redis/Postgres result backends, Redis/Postgres/FS schedule stores, Redis locks, Redis‑based token bucket rate limiting, etc. (pub.dev has viable starting points for Redis, AMQP, and SQS.) - Dart packages+2Dart packages+2 - -4) Execution model & semantics - - Concurrency: Worker uses an isolate pool (e.g., default = number of CPUs). Each isolate pulls deliveries from the broker stream and executes handlers. - Dart - - Prefetch: Per‑worker prefetch = concurrency × multiplier (configurable) to avoid queue starvation but prevent hoarding; defaults conservative (e.g., multiplier=2). (Inspired by Celery’s prefetch semantics.) - Celery Documentation - - Acks & leases: - - Default acks_late: ack after successful completion. For SQS/Streams, we renew visibility/lease periodically during execution. Provide ctx.extendLease. - Celery Documentation - - On isolate/worker crash (SIGKILL), messages become visible again (SQS/Streams) or we NACK on next heartbeat—thus at‑least‑once. Document idempotency. - Stack Overflow - - Retries & backoff: Configure per task; default exponential backoff with jitter. Dead‑letter after max attempts; expose a replay API and dashboard view (as in Sidekiq’s Dead set idea). - GitHub - - Time limits: softTimeLimit triggers a cooperative interruption (let handler cleanup); hardTimeLimit kills the isolate (and NACK/lease expiry triggers retry). - Celery Documentation+1 - - Rate limits: Token‑bucket per task and global (across workers) via Redis keys; e.g., rate: "5/s" behaves like Celery’s per‑task limit (and we add optional global aggregation). - deepnote.com+1 - - Scheduling (Beat): - - Cron & interval specs; jitter support; persistent last‑run tracking; dynamic reload without restart—like Celery Beat’s custom stores. - Celery Documentation - - For brokers without native delay, use a delayed set (Redis ZSET) mover. - - Canvas: - - chain: enqueue B with link=id(A) on A success; same for C. - - group: enqueue N tasks; collect GroupResult IDs in the backend. - - chord: aggregator watches for all group members done, enqueues callback (Celery’s chord). - Celery Documentation - -5) Initial adapters (“stores”) to implement - - Broker - - Redis Streams (XADD/XREADGROUP/XACK/XCLAIM). - - RabbitMQ via dart_amqp, supporting priority queues and TTL+DLX for delayed jobs. - Dart packages - - SQS using aws_sqs_api, leveraging visibility timeout & delay. - Dart packages - - Result backend: Redis (fast), Postgres (durable). - - Schedule store: Redis ZSET; Postgres table. - - Lock store: Redis SET key val NX PX. - - Rate limiter: Redis token bucket (INCR + TTL window). - - Note: You mentioned a “bus package”. Dart’s event_bus is useful for in‑process events, but you’ll want a true broker for durability/distribution; use event bus only internally (signal progress/telemetry). - Dart packages - -6) Minimal developer experience (DX) - -Task definition - -class SendEmailTask implements TaskHandler { - @override - String get name => 'email.send'; - - @override - TaskOptions get options => const TaskOptions( - queue: 'emails', - maxRetries: 5, - softTimeLimit: Duration(seconds: 20), - hardTimeLimit: Duration(seconds: 30), - rate: '5/s', - acksLate: true, - ); - - @override - Future call(TaskContext ctx, Map args) async { - // idempotency key example: messageId - final msgId = args['messageId'] as String; - // ... send email, heartbeat() periodically, etc. - ctx.heartbeat(); - } -} - -Enqueue - -await stem.enqueue('email.send', {'messageId': '123', 'to': 'a@b.com'}); - -Chains / groups / chords - -final g = group([ - task('resize.image', {'id': 1}), - task('resize.image', {'id': 2}), -]); - -await chord(g, task('notify.user', {'userId': 42})); - -Beat entry - -await scheduleStore.upsert(ScheduleEntry( - id: 'cleanup-invoices', - taskName: 'billing.cleanup', - args: {}, - spec: '0 2 * * *', // daily 02:00 - queue: 'maintenance', - jitter: Duration(minutes: 5), -)); - -7) Reliability details & defaults - - Visibility renewal every min(visibilityTimeout/2, 30s) while running. - - Prefetch default = concurrency * 2; cap result sets to avoid memory bloat (a Celery gotcha). - Celery Documentation - - acksLate=true + “reject on worker lost”–style behavior baked in where the broker supports it (SQS/Streams naturally, AMQP via NACK). - Celery Documentation+1 - - Time limits always recommended; doc soft vs hard with examples (Celery precedent). - Celery Documentation - - DLQ: automatic move after max attempts; CLI/UI to replay (Sidekiq pattern). - GitHub - - Rate limiting: per‑task and optional global via Redis keys (Celery only offers per‑task/worker). - deepnote.com+1 - -8) Phased build plan -Phase A — MVP (single node ok) - - Core types + registry + worker (isolates). - - Redis Streams broker + Redis result backend. - Dart packages - - Basic retries (exp backoff), time limits, prefetch, acks‑late, visibility renewal. - Celery Documentation+1 - - Beat with in‑memory + Redis ZSET store. - Celery Documentation - - CLI: stem worker, stem beat, stem enqueue. - -Phase B — Hardening - - DLQ + replay CLI. (Inspired by Sidekiq Dead set.) - GitHub - - Idempotency helpers: unique jobs (signature lock), outbox pattern docs. - - OpenTelemetry metrics/tracing; structured logs. - - Rate limiter (Redis), Lock store (Redis). - - Canvas: chain, group, chord + chord aggregator. - Celery Documentation - -Phase C — Adapters & Scale - - RabbitMQ adapter with priorities & delayed delivery via TTL+DLX. - Dart packages - - SQS adapter (visibility, delay, DLQ). - Dart packages - - Postgres result backend; Postgres schedule store. - - Dashboard (Flutter web): live queues, workers, retries, DLQ, schedules. - -Phase D — Production polish - - Config management, secrets handling, TLS everywhere. - - Canary tasks, graceful shutdown, autoscaling hints (queue depth, lag). - - Compatibility & migrations; semver; upgrade guide. - - Docs: operations book, SLOs, runbooks. - -9) Production‑readiness checklist (what it means to be ready) - -Correctness & semantics - - Clear delivery semantics documented (at‑least‑once), with idempotency recipes and helpers. - Stack Overflow - - Retries with exponential backoff + jitter; DLQ with replay tools. - GitHub - - Soft/hard time limits enforced across all pools; integration tests proving behavior. - Celery Documentation - - Lease renewal/visibility logic proven under node crashes/pauses (SQS/Streams). - -Performance - - Benchmarks per adapter: enqueue, consume, p95/p99 latencies; throughput vs. concurrency/prefetch (like Celery tuning docs emphasize). - Celery Documentation - - Backpressure strategy documented (prefetch, max in‑flight per worker). - -Reliability - - Chaos tests: kill workers mid‑task; network partitions; Redis/RabbitMQ restarts; ensure no silent losses (Celery’s acks_late pitfalls are well known). - Celery Documentation - - Health checks: liveness/readiness endpoints; worker heartbeat. - -Security - - TLS to brokers; credential rotation; optional payload signing and encryption at rest/in transit. - - Multi‑tenant isolation via namespaces/headers; per‑tenant rate limits. - -Observability - - OpenTelemetry traces: enqueue→consume→execute; metrics: queue depth, in‑flight, success/failure, retry counts, execution time, lease extensions; logs: structured with task id & trace id. - -Operations - - Admin UI: queues, workers, retry/DLQ, schedules, rate limits. - - CLI: drain, move, purge, replay, pause/resume queues. - - Config: env vars + file, precedence rules. - -Docs & support - - Versioned docs; migration notes; troubleshooting (e.g., Redis vs RabbitMQ tradeoffs as documented in Celery ecosystem). - Celery Documentation - -10) Adapter design notes (by broker) - - Redis Streams: - - XADD (enqueue), consumer groups with XREADGROUP (consume), XACK (ack), XCLAIM (reclaim abandoned). Use a scheduled mover (ZSET) for delays/ETA. - - Result backend & locks & rate limiting also Redis; easy development path. - Dart packages - - RabbitMQ: - - AMQP queues, routing keys, prefetch via basic.qos; priority queues; delays via per‑queue/message TTL + DLX. dart_amqp exists. - Dart packages - - SQS: - - Visibility timeout (lease), DelaySeconds, DLQ redrive policies; long polling. aws_sqs_api packages available. - Dart packages - - Celery supports Redis/RabbitMQ/SQS, with tradeoffs (e.g., SQS lacks worker inspection; Redis is fast but memory‑bounded; RabbitMQ strong in routing). We’ll document similar tradeoffs for Dart. - Celery Documentation+2celery-safwan.readthedocs.io+2 - -11) A few pragmatic defaults (learned from Celery production use) - - acksLate=true + worker‑lost handling by default (document why). - Celery Documentation+1 - - Prefetch multiplier small (e.g., 2) to avoid starving other workers or blowing memory. - Celery Documentation - - Time limits opt‑in but strongly recommended; templates in docs. - Celery Documentation - - Max retries finite; final state → DLQ with UI to retry (Sidekiq pattern). - GitHub - - Rate limits expressed as strings ("5/s", "300/m") but enforced globally via Redis token bucket (papering over Celery’s limitations). - Hatchet Documentation - -12) Risks & how we’ll handle them - - Global rate limiting across many workers is approximate with Redis (clock skew, retries)—document error bounds. (Celery lacks first‑class global limits.) - Hatchet Documentation - - Chord fan‑in on very large groups can stress the backend; shard chord state and throttle callback enqueue. - Celery Documentation - - AMQP delayed delivery requires TTL/DLX (or plugins); our Redis/SQS adapters have first‑class delay semantics. - celery-safwan.readthedocs.io - -13) Concrete next steps (you can start here) - - Create repo layout - - packages/core (types, registry, middleware, task DSL) - - packages/broker_redis, broker_amqp, broker_sqs - - packages/backend_redis, backend_postgres - - packages/scheduler (Beat) - - packages/dashboard (Flutter web) - - cli/ (worker/beat/admin commands) - - Implement MVP path - - Redis Streams broker + Redis backend + worker (isolates) + simple beat (Redis ZSET) + retries/backoff + time limits + CLI. - Dart packages - - Ship sample app - - Email task, image processing task, chain/group/chord demo; docker‑compose with Redis. - - Write production docs - - “Choosing a broker”, “Idempotency patterns”, “Deploying in K8s (graceful shutdown & leases)”, “Tuning prefetch & concurrency”. - -Appendix: Why Dart is a good fit here - - Isolates give true parallelism, clean cancellation boundaries (good for soft/hard timeouts & heartbeats). - Dart - - Rich ecosystem clients for Redis, RabbitMQ, and SQS exist today. - Dart packages+2Dart packages+2 - -If you’d like, I can sketch a Redis Streams adapter outline (XADD/XREADGROUP/XACK) and a tiny “Beat” loop with ZSET to make this concrete, or start from the interface stubs above and wire up the first task end‑to‑end. -You said: -Sure go ahead -ChatGPT said: -Request for GPT-5 Pro -Reasoned for 5m 35s - -Awesome — here’s a concrete, minimal skeleton for a Celery+Beat–style system in Dart using Redis Streams for the broker and Redis ZSET for delays & periodic schedules. - - What this gives you now - - Interfaces + working skeleton for: Broker (Redis Streams), Result backend (Redis), Beat (ZSET), Worker loop, Retries with backoff, DLQ, Delayed jobs. - - A tiny “task registry” and an example task. - - Exact Redis commands are shown in comments so you can wire to any Redis client you prefer. - - What you’ll still plug in - - A real Redis client (e.g., dartis, redis). Below I define a RedisCommands interface you implement with your client of choice. - - Proper isolate pooling & hard time limits (notes/TODOs included). - - Security/telemetry (left as clean extension points). - -Repo layout (suggested) - -stem/ - lib/core.dart - lib/redis_broker.dart - lib/redis_backend.dart - lib/beat.dart - lib/worker.dart - lib/tasks.dart - bin/enqueue_example.dart - bin/worker_example.dart - bin/beat_example.dart - -1) Core types (lib/core.dart) - -// lib/core.dart -import 'dart:async'; -import 'dart:convert'; -import 'dart:math'; - -String _randId() { - final r = Random(); - return '${DateTime.now().microsecondsSinceEpoch}-${r.nextInt(1 << 32)}'; -} - -class Envelope { - final String id; - final String name; - final Map args; - final Map headers; - final DateTime enqueuedAt; - final DateTime? notBefore; - final int priority; - final int attempt; - final int maxRetries; - final Duration? visibilityTimeout; // (For SQS-like brokers; Redis Streams uses idle+claim) - final String queue; - - Envelope({ - required this.id, - required this.name, - required this.args, - this.headers = const {}, - DateTime? enqueuedAt, - this.notBefore, - this.priority = 0, - this.attempt = 0, - this.maxRetries = 0, - this.visibilityTimeout, - this.queue = 'default', - }) : enqueuedAt = enqueuedAt ?? DateTime.now(); - - Envelope copyWith({ - String? id, - int? attempt, - DateTime? notBefore, - int? maxRetries, - }) { - return Envelope( - id: id ?? this.id, - name: name, - args: args, - headers: headers, - enqueuedAt: enqueuedAt, - notBefore: notBefore ?? this.notBefore, - priority: priority, - attempt: attempt ?? this.attempt, - maxRetries: maxRetries ?? this.maxRetries, - visibilityTimeout: visibilityTimeout, - queue: queue, - ); - } - - Map toJson() => { - 'id': id, - 'name': name, - 'args': args, - 'headers': headers, - 'enqueuedAt': enqueuedAt.toIso8601String(), - 'notBefore': notBefore?.toIso8601String(), - 'priority': priority, - 'attempt': attempt, - 'maxRetries': maxRetries, - 'visibilityTimeoutMs': visibilityTimeout?.inMilliseconds, - 'queue': queue, - }; - - static Envelope fromJson(Map j) => Envelope( - id: j['id'], - name: j['name'], - args: (j['args'] as Map).cast(), - headers: (j['headers'] as Map?)?.cast() ?? {}, - enqueuedAt: DateTime.parse(j['enqueuedAt']), - notBefore: j['notBefore'] != null ? DateTime.parse(j['notBefore']) : null, - priority: j['priority'] ?? 0, - attempt: j['attempt'] ?? 0, - maxRetries: j['maxRetries'] ?? 0, - visibilityTimeout: (j['visibilityTimeoutMs'] != null) - ? Duration(milliseconds: j['visibilityTimeoutMs']) - : null, - queue: j['queue'] ?? 'default', - ); -} - -class Delivery { - final Envelope message; - final String receipt; // e.g., 'key:group:id' for Redis Streams - final DateTime leaseExpireAt; // brokers with leases; Redis Streams ignores - - Delivery(this.message, this.receipt, this.leaseExpireAt); -} - -abstract class Broker { - Future publish(Envelope msg, {String? queue}); - Stream consume(String queue, {int prefetch = 1, String? consumerGroup, String? consumerName}); - Future ack(Delivery d); - Future nack(Delivery d, {bool requeue = true}); // Redis: ack+re-enqueue - Future deadLetter(Delivery d, {String? reason}); - Future purge(String queue); - - bool get supportsDelayed; - bool get supportsPriority; -} - -enum TaskState { queued, running, succeeded, failed, retried } - -class TaskStatus { - final String id; - final TaskState state; - final Object? result; - final Object? error; - final int attempt; - final DateTime updatedAt; - - TaskStatus(this.id, this.state, this.result, this.error, this.attempt, this.updatedAt); - - Map toJson() => { - 'id': id, - 'state': state.name, - 'result': result, - 'error': error?.toString(), - 'attempt': attempt, - 'updatedAt': updatedAt.toIso8601String(), - }; -} - -abstract class ResultBackend { - Future set(String taskId, TaskState state, { - Object? result, - Object? error, - int attempt = 0, - Duration? ttl, - Map? meta, - }); - - Future get(String taskId); -} - -class TaskOptions { - final String queue; - final int maxRetries; - final Duration? softTimeLimit; - final Duration? hardTimeLimit; - final String? rate; // "5/s", "300/m" (left for later) - final bool unique; - final Duration? uniqueFor; - final int priority; - final bool acksLate; - - const TaskOptions({ - this.queue = 'default', - this.maxRetries = 0, - this.softTimeLimit, - this.hardTimeLimit, - this.rate, - this.unique = false, - this.uniqueFor, - this.priority = 0, - this.acksLate = true, - }); -} - -abstract class TaskHandler { - String get name; - TaskOptions get options; - TaskMetadata get metadata => const TaskMetadata(); - Future call(TaskContext ctx, Map args); -} - -abstract class TaskRegistry { - void register(TaskHandler handler, {bool overrideExisting = false}); - TaskHandler? resolve(String name); - Iterable get handlers; - Stream get onRegister; -} - -class InMemoryTaskRegistry implements TaskRegistry { - final Map _m = {}; - final _onRegister = StreamController.broadcast(); - @override - void register(TaskHandler handler, {bool overrideExisting = false}) { - if (_m.containsKey(handler.name) && !overrideExisting) { - throw ArgumentError('Task handler "${handler.name}" already registered'); - } - _m[handler.name] = handler; - _onRegister.add( - TaskRegistrationEvent( - name: handler.name, - handler: handler, - overridden: overrideExisting, - ), - ); - } - - @override - TaskHandler? resolve(String name) => _m[name]; - - @override - Iterable get handlers => _m.values; - - @override - Stream get onRegister => _onRegister.stream; -} - -class TaskContext { - final String id; - final int attempt; - final void Function() heartbeat; - final Future Function(Duration) extendLease; // no-op for Redis Streams - final Future Function(double pct, {Map? data}) progress; - - TaskContext({ - required this.id, - required this.attempt, - required this.heartbeat, - required this.extendLease, - required this.progress, - }); -} - -abstract class RetryStrategy { - Duration nextDelay(int attempt, Object error); -} - -class ExponentialJitterRetry implements RetryStrategy { - final Duration base; - final Duration max; - final Random _r = Random(); - ExponentialJitterRetry({this.base = const Duration(seconds: 2), this.max = const Duration(minutes: 5)}); - @override - Duration nextDelay(int attempt, Object error) { - final ms = (base.inMilliseconds * pow(2, attempt)).toInt(); - final withCap = min(ms, max.inMilliseconds); - final jitter = _r.nextInt(withCap ~/ 4 + 1); // +/- 25% - return Duration(milliseconds: max(0, withCap - jitter)); - } -} - -class Stem { - final Broker broker; - Stem(this.broker); - - Future enqueue(String name, Map args, { - TaskOptions options = const TaskOptions(), - DateTime? notBefore, - int? maxRetries, - Map headers = const {}, - }) async { - final id = _randId(); - final env = Envelope( - id: id, - name: name, - args: args, - headers: headers, - notBefore: notBefore, - maxRetries: maxRetries ?? options.maxRetries, - priority: options.priority, - queue: options.queue, - ); - await broker.publish(env); - return id; - } -} - -2) Redis Streams broker (lib/redis_broker.dart) - -// lib/redis_broker.dart -import 'dart:async'; -import 'dart:convert'; - -import 'core.dart'; - -/// Implement this using your Redis client (dartis, redis, etc.) -/// Methods here are named closely to Redis commands for clarity. -abstract class RedisCommands { - Future send(List args); - Future multi(List> batched); // optional helper -} - -class RedisBroker implements Broker { - final RedisCommands redis; - final String namespace; - final int blockMs; - final int delayedDrainBatch; - final Duration reclaimAfter; // Idle time before we try to autoclalm (if we add XAUTOCLAIM later) - - RedisBroker( - this.redis, { - this.namespace = 'stem', - this.blockMs = 5000, - this.delayedDrainBatch = 128, - this.reclaimAfter = const Duration(minutes: 5), - }); - - String _kStream(String q) => '$namespace:stream:$q'; - String _kGroup(String q) => '$namespace:group:$q'; - String _kDelayed(String q) => '$namespace:delayed:$q'; - String _kDead(String q) => '$namespace:dead:$q'; - - @override - bool get supportsDelayed => true; - - @override - bool get supportsPriority => false; // can emulate with multiple streams/queues - - Future _ensureGroup(String q) async { - final key = _kStream(q); - final group = _kGroup(q); - // XGROUP CREATE key group $ MKSTREAM - try { - await redis.send(['XGROUP', 'CREATE', key, group, r'$', 'MKSTREAM']); - } catch (e) { - // BUSYGROUP is fine; ignore - } - } - - @override - Future publish(Envelope m, {String? queue}) async { - final q = queue ?? m.queue; - if (m.notBefore != null) { - // Delayed: ZADD delayed: score= member= - final score = m.notBefore!.millisecondsSinceEpoch.toString(); - await redis.send(['ZADD', _kDelayed(q), score, jsonEncode(m.toJson())]); - return; - } - await _ensureGroup(q); - // XADD stream * fields... - final fields = [ - 'id', m.id, - 'name', m.name, - 'args', jsonEncode(m.args), - 'headers', jsonEncode(m.headers), - 'enqueuedAt', m.enqueuedAt.toIso8601String(), - 'attempt', m.attempt.toString(), - 'maxRetries', m.maxRetries.toString(), - 'priority', m.priority.toString(), - 'queue', m.queue, - ]; - await redis.send(['XADD', _kStream(q), '*', ...fields]); - } - - Future _drainDelayed(String q) async { - final now = DateTime.now().millisecondsSinceEpoch.toString(); - // ZRANGEBYSCORE key -inf now LIMIT 0 N - final res = await redis.send(['ZRANGEBYSCORE', _kDelayed(q), '-inf', now, 'LIMIT', '0', '$delayedDrainBatch']); - if (res is List && res.isNotEmpty) { - for (final member in res.cast()) { - final ok = await redis.send(['ZREM', _kDelayed(q), member]); - if (ok == 1 || ok == '1') { - final env = Envelope.fromJson(jsonDecode(member)); - await publish(env.copyWith(notBefore: null), queue: q); - } - } - } - } - - @override - Stream consume(String queue, {int prefetch = 1, String? consumerGroup, String? consumerName}) async* { - final group = consumerGroup ?? _kGroup(queue); - final consumer = consumerName ?? 'c-${DateTime.now().millisecondsSinceEpoch}'; - await _ensureGroup(queue); - - while (true) { - await _drainDelayed(queue); - - // XREADGROUP GROUP COUNT BLOCK STREAMS > - final resp = await redis.send([ - 'XREADGROUP','GROUP',group,consumer, - 'COUNT','$prefetch', - 'BLOCK','$blockMs', - 'STREAMS', _kStream(queue), '>' - ]); - - if (resp == null) continue; - if (resp is! List || resp.isEmpty) continue; - - // resp format: [[streamKey, [[id, [field, value, field, value, ...]], ...]]] - for (final streamArr in resp.cast()) { - final entries = (streamArr[1] as List).cast(); - for (final entry in entries) { - final id = entry[0] as String; - final fieldList = (entry[1] as List).cast(); - final map = {}; - for (var i = 0; i < fieldList.length; i += 2) { - map[fieldList[i]] = fieldList[i + 1]; - } - final env = Envelope( - id: map['id']!, - name: map['name']!, - args: jsonDecode(map['args']!) as Map, - headers: (jsonDecode(map['headers']!) as Map).cast(), - enqueuedAt: DateTime.parse(map['enqueuedAt']!), - notBefore: null, - priority: int.parse(map['priority'] ?? '0'), - attempt: int.parse(map['attempt'] ?? '0'), - maxRetries: int.parse(map['maxRetries'] ?? '0'), - queue: map['queue'] ?? queue, - ); - // For Redis Streams, "lease" is conceptual. We encode receipt as "streamKey|group|id" - yield Delivery(env, '${_kStream(queue)}|$group|$id', DateTime.now()); - } - } - } - } - - @override - Future ack(Delivery d) async { - final parts = d.receipt.split('|'); // [key, group, id] - await redis.send(['XACK', parts[0], parts[1], parts[2]]); - } - - @override - Future nack(Delivery d, {bool requeue = true}) async { - // Redis Streams has no NACK; strategy: ACK the message then re-enqueue if requested. - await ack(d); - if (requeue) { - final env = d.message.copyWith( - attempt: d.message.attempt + 1, - id: d.message.id, // keep id for visibility (or new id if you prefer) - ); - await publish(env); - } - } - - @override - Future deadLetter(Delivery d, {String? reason}) async { - await ack(d); - // LPUSH dead list with a compact JSON - final payload = jsonEncode({ - 'envelope': d.message.toJson(), - 'reason': reason, - 'deadAt': DateTime.now().toIso8601String(), - }); - await redis.send(['LPUSH', _kDead(d.message.queue), payload]); - } - - @override - Future purge(String queue) async { - await redis.send(['DEL', _kStream(queue)]); - await redis.send(['DEL', _kDelayed(queue)]); - } -} - - Command mapping notes - - Enqueue: XADD stem:stream: * id <...> name <...> ... - - Consume: XREADGROUP GROUP stem:group: COUNT BLOCK STREAMS stem:stream: > - - Ack: XACK - - Delayed: ZADD stem:delayed: then a small mover (_drainDelayed) pushes due items to stream. - -3) Redis result backend (lib/redis_backend.dart) - -// lib/redis_backend.dart -import 'dart:convert'; -import 'core.dart'; -import 'redis_broker.dart'; - -class RedisResultBackend implements ResultBackend { - final RedisCommands redis; - final String namespace; - final Duration defaultTtl; - - RedisResultBackend(this.redis, {this.namespace = 'stem', this.defaultTtl = const Duration(days: 1)}); - - String _k(String id) => '$namespace:result:$id'; - - @override - Future set(String taskId, TaskState state, {Object? result, Object? error, int attempt = 0, Duration? ttl, Map? meta}) async { - final t = TaskStatus(taskId, state, result, error, attempt, DateTime.now()); - await redis.send(['SET', _k(taskId), jsonEncode(t.toJson())]); - final exp = ttl ?? defaultTtl; - await redis.send(['PEXPIRE', _k(taskId), '${exp.inMilliseconds}']); - } - - @override - Future get(String taskId) async { - final s = await redis.send(['GET', _k(taskId)]); - if (s == null) return null; - final j = jsonDecode(s as String) as Map; - return TaskStatus( - j['id'], - TaskState.values.firstWhere((e) => e.name == j['state']), - j['result'], - j['error'], - j['attempt'], - DateTime.parse(j['updatedAt']), - ); - } -} - -4) Beat (periodic scheduler) on ZSET (lib/beat.dart) - -// lib/beat.dart -import 'dart:async'; -import 'dart:convert'; -import 'core.dart'; -import 'redis_broker.dart'; - -class ScheduleEntry { - final String id; - final String taskName; - final Map args; - final String spec; // "every:5m" for MVP. (Extend with cron later.) - final String queue; - final bool enabled; - final Duration? jitter; - - ScheduleEntry({ - required this.id, - required this.taskName, - this.args = const {}, - required this.spec, - this.queue = 'default', - this.enabled = true, - this.jitter, - }); - - Map toJson() => { - 'id': id, - 'taskName': taskName, - 'args': args, - 'spec': spec, - 'queue': queue, - 'enabled': enabled, - 'jitterMs': jitter?.inMilliseconds, - }; - - static ScheduleEntry fromJson(Map j) => ScheduleEntry( - id: j['id'], - taskName: j['taskName'], - args: (j['args'] as Map).cast(), - spec: j['spec'], - queue: j['queue'] ?? 'default', - enabled: j['enabled'] ?? true, - jitter: j['jitterMs'] != null ? Duration(milliseconds: j['jitterMs']) : null, - ); -} - -class Beat { - final RedisCommands redis; - final Broker broker; // to publish - final String namespace; - final Duration tick; - Timer? _timer; - - Beat(this.redis, this.broker, {this.namespace = 'stem', this.tick = const Duration(seconds: 1)}); - - String get _kSchedule => '$namespace:schedules'; // ZSET of - String _kEntry(String id) => '$namespace:schedule:$id'; - String _kLock(String id) => '$namespace:beatlock:$id'; - - Future upsert(ScheduleEntry e, {DateTime? nextAt}) async { - final now = DateTime.now(); - final first = nextAt ?? _nextFromSpec(e.spec, now, jitter: e.jitter); - await redis.send(['SET', _kEntry(e.id), jsonEncode(e.toJson())]); - await redis.send(['ZADD', _kSchedule, '${first.millisecondsSinceEpoch}', e.id]); - } - - Future remove(String id) async { - await redis.send(['DEL', _kEntry(id)]); - await redis.send(['ZREM', _kSchedule, id]); - } - - Future run() async { - _timer = Timer.periodic(tick, (_) => _tick()); - } - - Future _tick() async { - final now = DateTime.now().millisecondsSinceEpoch.toString(); - final due = await redis.send(['ZRANGEBYSCORE', _kSchedule, '-inf', now, 'LIMIT', '0', '100']); - if (due is! List || due.isEmpty) return; - - for (final id in due.cast()) { - // Acquire short lock so multiple beats don't double fire: SET key val NX PX 5000 - final lock = await redis.send(['SET', _kLock(id), '1', 'NX', 'PX', '5000']); - if (lock == null) continue; - - final raw = await redis.send(['GET', _kEntry(id)]); - if (raw == null) { - await redis.send(['ZREM', _kSchedule, id]); - continue; - } - final entry = ScheduleEntry.fromJson(jsonDecode(raw as String)); - - // Fire if enabled - if (entry.enabled) { - final env = Envelope( - id: 'sch-$id-${DateTime.now().microsecondsSinceEpoch}', - name: entry.taskName, - args: entry.args, - notBefore: null, - queue: entry.queue, - ); - await broker.publish(env.copyWith(queue: entry.queue)); - } - - // Re-schedule - final next = _nextFromSpec(entry.spec, DateTime.now(), jitter: entry.jitter); - await redis.send(['ZADD', _kSchedule, '${next.millisecondsSinceEpoch}', id]); - await redis.send(['ZREM', _kSchedule, id]); - } - } - - DateTime _nextFromSpec(String spec, DateTime from, {Duration? jitter}) { - // MVP: "every:5m" | "every:30s" | "every:2h" - // TODO: add cron parser later. - Duration d = const Duration(minutes: 1); - if (spec.startsWith('every:')) { - final s = spec.substring(6); - if (s.endsWith('ms')) d = Duration(milliseconds: int.parse(s.replaceAll('ms',''))); - else if (s.endsWith('s')) d = Duration(seconds: int.parse(s.replaceAll('s',''))); - else if (s.endsWith('m')) d = Duration(minutes: int.parse(s.replaceAll('m',''))); - else if (s.endsWith('h')) d = Duration(hours: int.parse(s.replaceAll('h',''))); - else if (s.endsWith('d')) d = Duration(days: int.parse(s.replaceAll('d',''))); - } - var next = from.add(d); - if (jitter != null && jitter > Duration.zero) { - final ms = jitter.inMilliseconds; - final j = (DateTime.now().microsecondsSinceEpoch % (ms + 1)); - next = next.add(Duration(milliseconds: j)); - } - return next; - } -} - -5) Worker (consume, execute, retry, DLQ) (lib/worker.dart) - -// lib/worker.dart -import 'dart:async'; - -import 'core.dart'; -import 'redis_broker.dart'; - -class Worker { - final Broker broker; - final ResultBackend backend; - final TaskRegistry registry; - final String queue; - final String consumerGroup; - final String consumerName; - final int prefetch; - final RetryStrategy retry; - final bool stopOnEmpty; - - Worker({ - required this.broker, - required this.backend, - required this.registry, - this.queue = 'default', - String? consumerGroup, - String? consumerName, - this.prefetch = 10, - RetryStrategy? retry, - this.stopOnEmpty = false, - }) : consumerGroup = consumerGroup ?? 'stem:$queue', - consumerName = consumerName ?? 'w-${DateTime.now().millisecondsSinceEpoch}', - retry = retry ?? ExponentialJitterRetry(); - - Future run() async { - await for (final d in broker.consume(queue, prefetch: prefetch, consumerGroup: consumerGroup, consumerName: consumerName)) { - unawaited(_handle(d)); - } - } - - Future _handle(Delivery d) async { - final env = d.message; - final handler = registry.resolve(env.name); - if (handler == null) { - await broker.deadLetter(d, reason: 'unknown-task'); - return; - } - - // Mark running - await backend.set(env.id, TaskState.running, attempt: env.attempt); - - final ctx = TaskContext( - id: env.id, - attempt: env.attempt, - heartbeat: () {}, // attach to metrics/logs if you like - extendLease: (_) async {}, // no-op for Redis Streams - progress: (_, {data}) async {}, // wire to a channel if you like - ); - - try { - // NOTE: For hard time limits, spawn in a separate isolate and kill if overdue (TODO). - final soft = handler.options.softTimeLimit; - final hard = handler.options.hardTimeLimit; - final exec = handler.call(ctx, env.args); - - final result = await (hard != null - ? exec.timeout(hard, onTimeout: () { throw TimeoutException('hard time limit exceeded'); }) - : exec); - - // If we want soft TL, we could wrap the handler to check deadlines cooperatively (out of scope here). - - await broker.ack(d); - await backend.set(env.id, TaskState.succeeded, result: result, attempt: env.attempt); - } catch (e, _) { - if (env.attempt < (env.maxRetries)) { - final delay = retry.nextDelay(env.attempt, e); - final next = env.copyWith( - attempt: env.attempt + 1, - notBefore: DateTime.now().add(delay), - // Optionally: new id per attempt (choose policy). Keeping id can simplify traceability. - ); - await broker.ack(d); // Finish current delivery - await broker.publish(next); // Re-enqueue retry - await backend.set(env.id, TaskState.retried, error: e, attempt: env.attempt); - } else { - await broker.deadLetter(d, reason: 'max-retries-reached'); - await backend.set(env.id, TaskState.failed, error: e, attempt: env.attempt); - } - } - } -} - - Why ack+re‑enqueue on retry? - In Redis Streams, leaving a message pending until retry makes recovery messy. Acking the current delivery and creating a fresh message is simpler and aligns with typical retry patterns (DLQ when maxed). - -6) Example task(s) (lib/tasks.dart) - -// lib/tasks.dart -import 'core.dart'; - -class SendEmailTask implements TaskHandler { - @override - String get name => 'email.send'; - - @override - TaskOptions get options => const TaskOptions( - queue: 'emails', - maxRetries: 5, - softTimeLimit: Duration(seconds: 20), - hardTimeLimit: Duration(seconds: 30), - acksLate: true, - ); - - @override - Future call(TaskContext ctx, Map args) async { - final to = args['to'] as String; - final subject = args['subject'] as String? ?? '(no subject)'; - final body = args['body'] as String? ?? ''; - // TODO: call your email provider; add heartbeats/progress as you like - // ctx.heartbeat(); - // await ctx.progress(0.5); - print('Sending email to $to: $subject'); - } -} - -7) Wire it together (examples in bin/) - -A) Implement your Redis client adapter (one-time) - -// lib/redis_dartis_adapter.dart (example shape) -- PSEUDO; adapt to your client -import 'redis_broker.dart'; -// import 'package:dartis/dartis.dart' as dartis; - -class DartisRedis implements RedisCommands { - // final dartis.Client client; - // final dartis.Commands cmd; - DartisRedis(/*this.client, this.cmd*/); - - @override - Future send(List args) async { - // If your client supports raw commands: - // return await cmd.send(args.first, args.skip(1).toList()); - throw UnimplementedError('Wire to your Redis client here.'); - } - - @override - Future multi(List> batched) async { - // Wrap batched commands in MULTI/EXEC if your client supports it. - for (final a in batched) { await send(a); } - } -} - -B) Worker entrypoint - -// bin/worker_example.dart -import 'package:stem/core.dart'; -import 'package:stem/redis_broker.dart'; -import 'package:stem/redis_backend.dart'; -import 'package:stem/worker.dart'; -import 'package:stem/tasks.dart'; -// import 'package:stem/redis_dartis_adapter.dart'; - -Future main() async { - // final redis = DartisRedis(...); // TODO: create real client - final redis = throw UnimplementedError('Provide RedisCommands implementation'); - - final broker = RedisBroker(redis, namespace: 'stem'); - final backend = RedisResultBackend(redis, namespace: 'stem'); - - final reg = InMemoryTaskRegistry()..register(SendEmailTask()); - - final w = Worker( - broker: broker, - backend: backend, - registry: reg, - queue: 'emails', - prefetch: 10, - ); - - await w.run(); // runs indefinitely -} - -C) Beat entrypoint - -// bin/beat_example.dart -import 'package:stem/beat.dart'; -import 'package:stem/redis_broker.dart'; -// import 'package:stem/redis_dartis_adapter.dart'; - -Future main() async { - // final redis = DartisRedis(...); - final redis = throw UnimplementedError('Provide RedisCommands implementation'); - - final broker = RedisBroker(redis, namespace: 'stem'); - final beat = Beat(redis, broker, namespace: 'stem'); - - // Example: run a cleanup task every 5 minutes with up to 30s jitter - await beat.upsert(ScheduleEntry( - id: 'cleanup-invoices', - taskName: 'billing.cleanup', - spec: 'every:5m', - queue: 'maintenance', - jitter: Duration(seconds: 30), - )); - - await beat.run(); -} - -D) Enqueue example - -// bin/enqueue_example.dart -import 'package:stem/core.dart'; -import 'package:stem/redis_broker.dart'; -// import 'package:stem/redis_dartis_adapter.dart'; - -Future main() async { - // final redis = DartisRedis(...); - final redis = throw UnimplementedError('Provide RedisCommands implementation'); - - final broker = RedisBroker(redis, namespace: 'stem'); - final stem = Stem(broker); - - await stem.enqueue( - 'email.send', - {'to': 'user@example.com', 'subject': 'Welcome', 'body': 'Hello!'}, - options: const TaskOptions(queue: 'emails', maxRetries: 3), - ); -} - -8) Notes, trade‑offs, and next TODOs (so you can extend confidently) - - Delayed jobs: Implemented via ZSET stem:delayed: → mover periodically XADDs due items. (Workers call _drainDelayed every poll; you can also run a tiny “queue-scheduler” sidecar.) - - Retries: We ack + re-enqueue with a new notBefore, tracked with attempt. When attempt > maxRetries, we DLQ the envelope to LPUSH stem:dead:. - - Visibility/leases: Redis Streams doesn’t have visibility timeouts like SQS. Recovery is via pending entry claim (you can add XAUTOCLAIM loop later to reclaim messages idle for reclaimAfter). extendLease is a no‑op for Redis Streams by design. - - Time limits: The code uses Future.timeout for a hard limit surface. For a true hard kill, run the task in a spawned Isolate and isolate.kill() when the timer expires (recommended for CPU/blocked work). - - Idempotency: Before side effects, check your own idempotency keys (e.g., headers['idempotency-key']) using Redis locks or a DB outbox table. The skeleton leaves this to the task author for now. - - Rate limiting: Add a Redis token‑bucket (INCR + TTL per window) keyed by taskName to enforce "5/s" / "300/m". Wire it in Worker._handle before execution. - - Canvas (chains/groups/chords): Add a tiny “group result” record in the result backend, and a chord aggregator that watches completion and enqueues the callback — easy to layer on this base. - -9) How to try this quickly - - Start Redis - - docker run --rm -p 6379:6379 redis:7 - - Wire RedisCommands to your preferred package (keep the command arrays the same as in the comments). - - Run Beat (bin/beat_example.dart) and Worker (bin/worker_example.dart) in separate terminals. - - Enqueue (bin/enqueue_example.dart) and watch the worker log. - -10) Production‑readiness checklist (applied to this skeleton) - - Correctness - -Add XAUTOCLAIM reclaimer loop (periodically claim idle > reclaimAfter). - -Make retry policy and DLQ configurable per task/queue. - - Provide idempotency helpers (unique job lock + outbox docs). - -Performance - -Prefetch tuning, backpressure (cap concurrent in‑flight per worker). - - Benchmarks: p95 execute time, throughput vs. concurrency. - -Reliability - -Graceful shutdown: stop polling, drain inflight (ack or requeue). - - Chaos tests: kill worker mid‑task; verify retry/claim path. - -Security - -TLS Redis; creds via env/secret. - - Optionally sign/encrypt payloads. - -Observability - -OpenTelemetry spans (enqueue → consume → execute). - -Metrics: queue depth, pending, retries, DLQ size, task durations. - - Admin UI (Flutter Web): queues, workers, schedules, DLQ replay. - -Ops - -CLI: stem worker, stem beat, stem admin (dlq, purge, move). - - Docs: broker trade‑offs, tuning, K8s deployment. - -If you want, I can add the XAUTOCLAIM reclaimer loop and a minimal isolate-based hard timeout runner next, plus a small DLQ replay CLI snippet — just say the word and I’ll drop the code. - - -ChatGPT can make mistakes. Check important info. diff --git a/packages/stem/test/api/observability_api_test.dart b/packages/stem/test/api/observability_api_test.dart new file mode 100644 index 00000000..998034d7 --- /dev/null +++ b/packages/stem/test/api/observability_api_test.dart @@ -0,0 +1,19 @@ +import 'package:stem/observability.dart'; +import 'package:test/test.dart'; + +void main() { + test('observability entrypoint exposes Stem-owned logging configuration', () { + expect(StemLogLevel.warning, isA()); + expect(StemLogFormat.plain, isA()); + + configureStemLogging( + level: StemLogLevel.warning, + format: StemLogFormat.plain, + enableConsole: false, + ); + stemLogger.info( + 'structured message', + fields: const {'component': 'api-test'}, + ); + }); +} diff --git a/packages/stem/test/api/stable_api_test.dart b/packages/stem/test/api/stable_api_test.dart new file mode 100644 index 00000000..e36b4682 --- /dev/null +++ b/packages/stem/test/api/stable_api_test.dart @@ -0,0 +1,42 @@ +import 'package:stem/stable.dart'; +import 'package:test/test.dart'; + +class _StableArgs { + const _StableArgs(this.value); + + final int value; +} + +void main() { + test('stable entrypoint supports typed registration and enqueue', () async { + final definition = TaskDefinition<_StableArgs, int>.codec( + name: 'stable.api.double', + argsCodec: PayloadCodec<_StableArgs>.map( + encode: (args) => {'value': args.value}, + decode: (payload) => _StableArgs(payload['value']! as int), + ), + ); + final handler = definition.handler( + entrypoint: (context, args) async => args.value * 2, + executionMode: TaskExecutionMode.inline, + ); + expect(handler.executionMode, TaskExecutionMode.inline); + final app = await StemApp.inMemory(tasks: [handler]); + + try { + await app.start(); + final taskId = await app.enqueueCall( + definition.buildCall(const _StableArgs(21)), + ); + final result = await app.waitForTask( + taskId, + timeout: const Duration(seconds: 2), + decode: (payload) => payload! as int, + ); + + expect(result?.value, 42); + } finally { + await app.shutdown(); + } + }); +} diff --git a/packages/stem/test/bootstrap/module_bootstrap_test.dart b/packages/stem/test/bootstrap/module_bootstrap_test.dart index 7aa44fd9..acb703bd 100644 --- a/packages/stem/test/bootstrap/module_bootstrap_test.dart +++ b/packages/stem/test/bootstrap/module_bootstrap_test.dart @@ -242,8 +242,8 @@ void main() { final client = await StemClient.inMemory( module: StemModule(tasks: [moduleTask]), ); - final app = await client.createApp(); + await app.start(); try { expect(app.registry.resolve('module.client.task'), same(moduleTask)); diff --git a/packages/stem/test/bootstrap/shortcut_allow_auto_start_test.dart b/packages/stem/test/bootstrap/shortcut_allow_auto_start_test.dart index b282b752..364a4a70 100644 --- a/packages/stem/test/bootstrap/shortcut_allow_auto_start_test.dart +++ b/packages/stem/test/bootstrap/shortcut_allow_auto_start_test.dart @@ -2,10 +2,42 @@ import 'package:stem/stem.dart'; import 'package:test/test.dart'; void main() { - group('shortcut allowWorkerAutoStart', () { + group('explicit worker lifecycle', () { + test('StemApp does not start the worker by default', () async { + final app = await StemApp.inMemory( + tasks: [ + FunctionTaskHandler( + name: 'shortcut.default-stop.echo', + entrypoint: (context, args) async => 'done', + ), + ], + ); + + try { + final taskId = await app.enqueue('shortcut.default-stop.echo'); + expect(app.isStarted, isFalse); + + final pending = await app.waitForTask( + taskId, + timeout: const Duration(milliseconds: 10), + ); + expect(pending?.status.state, TaskState.queued); + expect(app.isStarted, isFalse); + + await app.start(); + final completed = await app.waitForTask( + taskId, + timeout: const Duration(seconds: 1), + ); + expect(completed?.isSucceeded, isTrue); + expect(completed?.value, 'done'); + } finally { + await app.shutdown(); + } + }); + test('StemApp can enqueue without starting the worker', () async { final app = await StemApp.inMemory( - allowWorkerAutoStart: false, tasks: [ FunctionTaskHandler( name: 'shortcut.echo', @@ -18,6 +50,10 @@ void main() { final taskId = await app.enqueue('shortcut.echo'); expect(app.isStarted, isFalse); + final status = await app.getTaskStatus(taskId); + expect(status?.state, TaskState.queued); + expect(app.isStarted, isFalse); + final pending = await app.waitForTask( taskId, timeout: const Duration(milliseconds: 10), @@ -52,7 +88,6 @@ void main() { final app = await StemWorkflowApp.inMemory( flows: [flow], - allowWorkerAutoStart: false, ); try { diff --git a/packages/stem/test/bootstrap/stem_app_test.dart b/packages/stem/test/bootstrap/stem_app_test.dart index 7bda08b8..b6971c67 100644 --- a/packages/stem/test/bootstrap/stem_app_test.dart +++ b/packages/stem/test/bootstrap/stem_app_test.dart @@ -1,3 +1,4 @@ +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; import 'package:test/test.dart'; @@ -5,6 +6,22 @@ import 'test_store_adapter.dart'; void main() { group('StemApp', () { + test('exposes separate producer, observer, and worker roles', () async { + final app = await StemApp.inMemory(); + try { + final StemProducer producer = app; + final StemObserver observer = app; + final StemWorkerHost workerHost = app; + + expect(producer, same(app)); + expect(observer, same(app)); + expect(workerHost, same(app)); + expect(workerHost.isStarted, isFalse); + } finally { + await app.shutdown(); + } + }); + test('inMemory executes tasks', () async { final handler = FunctionTaskHandler( name: 'test.echo', @@ -12,7 +29,10 @@ void main() { metadata: const TaskMetadata(idempotent: true), ); - final app = await StemApp.inMemory(tasks: [handler]); + final app = await StemApp.inMemory( + tasks: [handler], + ); + await app.start(); try { final taskId = await app.enqueue('test.echo'); final completed = await app.backend @@ -25,16 +45,19 @@ void main() { } }); - test('inMemory lazy-starts on first enqueue', () async { + test('inMemory executes after an explicit start', () async { final handler = FunctionTaskHandler( - name: 'test.lazy-start', + name: 'test.explicit-start', entrypoint: (context, args) async => 'started', runInIsolate: false, ); - final app = await StemApp.inMemory(tasks: [handler]); + final app = await StemApp.inMemory( + tasks: [handler], + ); + await app.start(); try { - final taskId = await app.enqueue('test.lazy-start'); + final taskId = await app.enqueue('test.explicit-start'); final completed = await app.waitForTask( taskId, timeout: const Duration(seconds: 2), @@ -52,7 +75,10 @@ void main() { runInIsolate: false, ); - final app = await StemApp.inMemory(tasks: [taskHandler]); + final app = await StemApp.inMemory( + tasks: [taskHandler], + ); + await app.start(); try { final taskId = await app.enqueue('test.status.task'); final taskStatus = await app.waitForTask( @@ -91,6 +117,7 @@ void main() { final app = await StemApp.inMemory( module: StemModule(tasks: [handler]), ); + await app.start(); try { expect(app.registry.resolve('test.module.queue'), same(handler)); expect(app.worker.subscription.queues, ['priority']); @@ -118,7 +145,10 @@ void main() { runInIsolate: false, ); - final app = await StemApp.inMemory(tasks: [handler]); + final app = await StemApp.inMemory( + tasks: [handler], + ); + await app.start(); try { expect(app.worker.subscription.queues, ['priority']); @@ -136,7 +166,7 @@ void main() { } }); - test('inMemory lazy-starts for canvas dispatch', () async { + test('inMemory executes Canvas work after an explicit start', () async { final handler = FunctionTaskHandler( name: 'test.canvas.double', entrypoint: (context, args) async { @@ -146,7 +176,10 @@ void main() { runInIsolate: false, ); - final app = await StemApp.inMemory(tasks: [handler]); + final app = await StemApp.inMemory( + tasks: [handler], + ); + await app.start(); try { final result = await app.canvas.chain([ task('test.canvas.double', args: {'value': 21}), @@ -175,8 +208,9 @@ void main() { entrypoint: (context, args) async => 'extra-ok', runInIsolate: false, ); - final app = await StemApp.inMemory(); + await app.start(); + try { app ..registerTask(directHandler) @@ -324,6 +358,7 @@ void main() { adapters: [adapter], tasks: [handler], ); + await app.start(); try { final taskId = await app.enqueue('test.from-url'); final completed = await app.backend @@ -435,7 +470,10 @@ void main() { }, ); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); + await workflowApp.start(); try { final runId = await workflowApp.startWorkflow('workflow.demo'); final run = await workflowApp @@ -461,7 +499,10 @@ void main() { }, ); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); + await workflowApp.start(); try { final runId = await workflowApp.startWorkflow('workflow.typed'); final run = await workflowApp.waitForCompletion<_DemoPayload>( @@ -494,7 +535,10 @@ void main() { }, ); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); + await workflowApp.start(); try { final runId = await workflowApp.startWorkflow( 'workflow.typed.versioned', @@ -528,7 +572,10 @@ void main() { }, ); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); + await workflowApp.start(); try { final runId = await workflowApp.startWorkflowJson( 'workflow.json.start', @@ -561,7 +608,10 @@ void main() { }, ); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); + await workflowApp.start(); try { final runId = await workflowApp.startWorkflowValue( 'workflow.codec.start', @@ -602,7 +652,10 @@ void main() { }, ); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); + await workflowApp.start(); try { final runId = await workflowApp.startWorkflowVersionedJson( 'workflow.versioned.json.start', @@ -650,7 +703,10 @@ void main() { }, ); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); + await workflowApp.start(); try { final runId = await workflowApp.startWorkflow('workflow.json.emit'); await workflowApp.executeRun(runId); @@ -698,7 +754,10 @@ void main() { }, ); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); + await workflowApp.start(); try { final runId = await workflowApp.startWorkflow( 'workflow.versioned.json.emit', @@ -734,7 +793,9 @@ void main() { }, ); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); try { var decodeInvocations = 0; final runId = await workflowApp.startWorkflow('workflow.cancelled'); @@ -773,7 +834,10 @@ void main() { }, ); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); + await workflowApp.start(); try { final runId = await workflowApp.startWorkflow('workflow.timeout'); final result = await workflowApp.waitForCompletion( @@ -814,6 +878,7 @@ void main() { adapters: [adapter], flows: [flow], ); + await workflowApp.start(); try { final runId = await workflowApp.startWorkflow('workflow.from-url'); final result = await workflowApp.waitForCompletion( @@ -873,7 +938,10 @@ void main() { ); final module = StemModule(flows: [moduleFlow], tasks: [helperTask]); - final workflowApp = await StemWorkflowApp.inMemory(module: module); + final workflowApp = await StemWorkflowApp.inMemory( + module: module, + ); + await workflowApp.start(); try { expect( workflowApp.app.registry.resolve('workflow.module.helper'), @@ -905,6 +973,7 @@ void main() { final workflowApp = await StemWorkflowApp.inMemory( module: StemModule(tasks: [helperTask]), ); + await workflowApp.start(); try { expect( workflowApp.app.worker.subscription.queues, @@ -938,6 +1007,7 @@ void main() { subscription: RoutingSubscription.singleQueue('workflow'), ), ); + await workflowApp.start(); try { expect(workflowApp.app.worker.subscription.queues, ['workflow']); } finally { @@ -961,7 +1031,10 @@ void main() { encodeParams: (params) => params, ); - final workflowApp = await StemWorkflowApp.inMemory(flows: [moduleFlow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [moduleFlow], + ); + await workflowApp.start(); try { final runId = await workflowRef.start( workflowApp, @@ -987,7 +1060,10 @@ void main() { }, ); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); + await workflowApp.start(); try { final runId = await workflowApp.startWorkflow('workflow.detail.helper'); final result = await workflowApp.waitForCompletion( @@ -1012,7 +1088,9 @@ void main() { }, ); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); try { final manifest = workflowApp.workflowManifest(); final entry = manifest.singleWhere( @@ -1041,6 +1119,7 @@ void main() { final module = StemModule(flows: [flow], tasks: [taskHandler]); final workflowApp = await StemWorkflowApp.inMemory(); + await workflowApp.start(); try { workflowApp.registerModule(module); @@ -1074,6 +1153,7 @@ void main() { ); final workflowApp = await StemWorkflowApp.inMemory(); + await workflowApp.start(); try { workflowApp.registerWorkflow(flow.definition); @@ -1108,6 +1188,7 @@ void main() { ); final workflowApp = await StemWorkflowApp.inMemory(); + await workflowApp.start(); try { workflowApp ..registerFlow(flow) @@ -1154,6 +1235,7 @@ void main() { ); final workflowApp = await StemWorkflowApp.inMemory(); + await workflowApp.start(); try { workflowApp ..registerFlows([flow]) @@ -1191,6 +1273,7 @@ void main() { ); final workflowApp = await StemWorkflowApp.inMemory(); + await workflowApp.start(); try { workflowApp.registerWorkflows([definition]); @@ -1215,7 +1298,10 @@ void main() { }, ); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); + await workflowApp.start(); try { final runId = await workflowApp.startWorkflow('workflow.views.helper'); final result = await workflowApp.waitForCompletion( @@ -1249,7 +1335,10 @@ void main() { }, ); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); + await workflowApp.start(); try { final runId = await workflowApp.startWorkflow( 'workflow.execute.helper', @@ -1280,7 +1369,10 @@ void main() { }, ); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); + await workflowApp.start(); try { final runId = await workflowApp.startWorkflow('workflow.rewind.helper'); await workflowApp.executeRun(runId); @@ -1314,7 +1406,10 @@ void main() { }, ); - final workflowApp = await StemWorkflowApp.inMemory(scripts: [script]); + final workflowApp = await StemWorkflowApp.inMemory( + scripts: [script], + ); + await workflowApp.start(); try { final runId = await workflowApp.startWorkflow( 'workflow.watchers.helper', @@ -1348,7 +1443,10 @@ void main() { }, ); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); + await workflowApp.start(); try { final runId = await workflowApp.startWorkflow( 'workflow.resume.due.helper', @@ -1398,7 +1496,10 @@ void main() { ); final workflowRef = flow.ref0(); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); + await workflowApp.start(); try { final runId = await workflowRef.start(workflowApp); final result = await workflowRef.waitFor( @@ -1460,7 +1561,10 @@ void main() { ); final workflowRef = script.ref0(); - final workflowApp = await StemWorkflowApp.inMemory(scripts: [script]); + final workflowApp = await StemWorkflowApp.inMemory( + scripts: [script], + ); + await workflowApp.start(); try { final runId = await workflowRef.start(workflowApp); final result = await workflowRef.waitFor( diff --git a/packages/stem/test/bootstrap/stem_client_test.dart b/packages/stem/test/bootstrap/stem_client_test.dart index a4a96b3b..5c9f3ac8 100644 --- a/packages/stem/test/bootstrap/stem_client_test.dart +++ b/packages/stem/test/bootstrap/stem_client_test.dart @@ -1,3 +1,4 @@ +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; import 'package:test/test.dart'; @@ -72,8 +73,8 @@ void main() { final client = await StemClient.inMemory( module: StemModule(tasks: [moduleTask]), ); - final app = await client.createApp(); + await app.start(); expect( app.registry.resolve('client.default-module.app-task'), @@ -97,20 +98,20 @@ void main() { }, ); - test('StemClient createApp lazy-starts on first enqueue', () async { + test('StemClient createApp executes after an explicit start', () async { final client = await StemClient.inMemory( tasks: [ FunctionTaskHandler( - name: 'client.lazy-start', + name: 'client.explicit-start', entrypoint: (context, args) async => 'task-ok', runInIsolate: false, ), ], ); - final app = await client.createApp(); + await app.start(); - final taskId = await app.enqueue('client.lazy-start'); + final taskId = await app.enqueue('client.explicit-start'); final result = await app.waitForTask( taskId, timeout: const Duration(seconds: 2), @@ -220,6 +221,7 @@ void main() { ), ], ); + await app.start(); expect(app.worker.subscription.queues, ['priority']); @@ -252,6 +254,7 @@ void main() { final app = await client.createApp( module: StemModule(tasks: [moduleTask]), ); + await app.start(); expect(app.registry.resolve('client.module.app-task'), same(moduleTask)); expect(app.worker.subscription.queues, ['priority']); diff --git a/packages/stem/test/bootstrap/stem_stack_test.dart b/packages/stem/test/bootstrap/stem_stack_test.dart index a94c6400..1f2ff101 100644 --- a/packages/stem/test/bootstrap/stem_stack_test.dart +++ b/packages/stem/test/bootstrap/stem_stack_test.dart @@ -1,3 +1,4 @@ +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; import 'package:test/test.dart'; diff --git a/packages/stem/test/performance/throughput_test.dart b/packages/stem/test/performance/throughput_test.dart index d393a72e..e82d78aa 100644 --- a/packages/stem/test/performance/throughput_test.dart +++ b/packages/stem/test/performance/throughput_test.dart @@ -1,5 +1,6 @@ import 'dart:async'; +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; import 'package:test/test.dart'; diff --git a/packages/stem/test/soak/soak_test.dart b/packages/stem/test/soak/soak_test.dart index ba28dba5..a3952cb4 100644 --- a/packages/stem/test/soak/soak_test.dart +++ b/packages/stem/test/soak/soak_test.dart @@ -1,5 +1,6 @@ import 'dart:async'; +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; import 'package:test/test.dart'; diff --git a/packages/stem/test/unit/backend/encoding_result_backend_test.dart b/packages/stem/test/unit/backend/encoding_result_backend_test.dart index 2abd6801..8abbc4d8 100644 --- a/packages/stem/test/unit/backend/encoding_result_backend_test.dart +++ b/packages/stem/test/unit/backend/encoding_result_backend_test.dart @@ -1,5 +1,6 @@ import 'dart:async'; +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; import 'package:test/test.dart'; @@ -72,6 +73,43 @@ void main() { final fetched = await backend.getGroup('grp'); expect(fetched?.results['grp-task']?.payload, 'group-value'); }); + + test('encodes atomic terminal writes before delegation', () async { + final inner = InMemoryResultBackend(); + const encoder = _PrefixTaskPayloadEncoder(); + final registry = TaskPayloadEncoderRegistry( + defaultResultEncoder: encoder, + defaultArgsEncoder: const JsonTaskPayloadEncoder(), + ); + final backend = withTaskPayloadEncoder(inner, registry); + final meta = {stemResultEncoderMetaKey: encoder.id}; + + await backend.set('atomic-task', TaskState.running, meta: meta); + final applied = await (backend as AtomicTerminalResultBackend) + .setTerminalIfAbsent( + TaskStatus( + id: 'atomic-task', + state: TaskState.succeeded, + payload: 'value', + attempt: 1, + meta: meta, + ), + ); + final rejected = await (backend as AtomicTerminalResultBackend) + .setTerminalIfAbsent( + TaskStatus( + id: 'atomic-task', + state: TaskState.failed, + attempt: 1, + meta: meta, + ), + ); + + expect(applied, isTrue); + expect(rejected, isFalse); + expect((await backend.get('atomic-task'))?.payload, 'value'); + expect((await backend.get('atomic-task'))?.state, TaskState.succeeded); + }); }); } diff --git a/packages/stem/test/unit/backend/in_memory_backend_test.dart b/packages/stem/test/unit/backend/in_memory_backend_test.dart index 9af89ff7..e56d2361 100644 --- a/packages/stem/test/unit/backend/in_memory_backend_test.dart +++ b/packages/stem/test/unit/backend/in_memory_backend_test.dart @@ -1,6 +1,8 @@ import 'dart:async'; +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; +import 'package:stem/testing.dart' show FakeStemClock; import 'package:test/test.dart'; void main() { @@ -27,6 +29,30 @@ void main() { expect(await backend.get('task-1'), isNull); }); + test('terminal writes are first-writer-wins', () async { + final backend = InMemoryResultBackend(); + addTearDown(backend.dispose); + + await backend.set('terminal-task', TaskState.running); + final first = TaskStatus( + id: 'terminal-task', + state: TaskState.succeeded, + payload: 'first', + attempt: 1, + ); + final late = TaskStatus( + id: 'terminal-task', + state: TaskState.failed, + error: const TaskError(type: 'LateFailure', message: 'late'), + attempt: 1, + ); + + expect(await backend.setTerminalIfAbsent(first), isTrue); + expect(await backend.setTerminalIfAbsent(late), isFalse); + expect((await backend.get('terminal-task'))?.payload, 'first'); + expect((await backend.get('terminal-task'))?.state, TaskState.succeeded); + }); + test('InMemoryResultBackend lists task statuses with filters', () async { final backend = InMemoryResultBackend(); addTearDown(backend.dispose); @@ -110,23 +136,26 @@ void main() { test('InMemoryResultBackend worker heartbeats expire', () async { final backend = InMemoryResultBackend( - heartbeatTtl: const Duration(milliseconds: 5), + heartbeatTtl: const Duration(seconds: 5), ); addTearDown(backend.dispose); + final clock = FakeStemClock(DateTime.utc(2026)); final heartbeat = WorkerHeartbeat( workerId: 'worker-1', - timestamp: DateTime.now(), + timestamp: clock.now(), isolateCount: 1, inflight: 0, queues: [QueueHeartbeat(name: 'default', inflight: 0)], ); - await backend.setWorkerHeartbeat(heartbeat); - expect(await backend.getWorkerHeartbeat('worker-1'), isNotNull); + await withStemClock(clock, () async { + await backend.setWorkerHeartbeat(heartbeat); + expect(await backend.getWorkerHeartbeat('worker-1'), isNotNull); - await Future.delayed(const Duration(milliseconds: 10)); - expect(await backend.getWorkerHeartbeat('worker-1'), isNull); - expect(await backend.listWorkerHeartbeats(), isEmpty); + clock.advance(const Duration(seconds: 6)); + expect(await backend.getWorkerHeartbeat('worker-1'), isNull); + expect(await backend.listWorkerHeartbeats(), isEmpty); + }); }); } diff --git a/packages/stem/test/unit/canvas/canvas_test.dart b/packages/stem/test/unit/canvas/canvas_test.dart index 42023d26..0fc676f9 100644 --- a/packages/stem/test/unit/canvas/canvas_test.dart +++ b/packages/stem/test/unit/canvas/canvas_test.dart @@ -1,5 +1,6 @@ import 'dart:async'; +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; import 'package:test/test.dart'; @@ -19,12 +20,24 @@ void main() { canvas = Canvas( broker: broker, backend: backend, - tasks: [_EchoTask(), _SumTask()], + tasks: [ + _EchoTask(), + _FailTask(), + _SumTask(), + _StringifyTask(), + _LengthTask(), + ], ); worker = Worker( broker: broker, backend: backend, - tasks: [_EchoTask(), _SumTask()], + tasks: [ + _EchoTask(), + _FailTask(), + _SumTask(), + _StringifyTask(), + _LengthTask(), + ], consumerName: 'canvas-worker', concurrency: 1, prefetchMultiplier: 1, @@ -87,6 +100,66 @@ void main() { expect(result.finalStatus?.state, TaskState.succeeded); }); + test( + 'typedChain enforces and executes heterogeneous transitions', + () async { + final stringify = TaskDefinition( + name: 'stringify', + encodeArgs: (value) => {'value': value}, + decodeResult: (payload) => payload.toString(), + ); + final length = TaskDefinition( + name: 'length', + encodeArgs: (value) => {'value': value}, + decodeResult: (payload) => (payload! as num).toInt(), + ); + + final result = await canvas + .typedChain(stringify, 42) + .then(length) + .run(); + + expect(result.value, equals(2)); + expect(result.finalStatus?.state, TaskState.succeeded); + }, + ); + + test('typedChain completes when a result decoder throws', () async { + final definition = TaskDefinition( + name: 'echo', + encodeArgs: (value) => {'value': value}, + decodeResult: (_) => throw const FormatException('invalid result'), + ); + + await expectLater( + canvas + .typedChain(definition, 42) + .run() + .timeout(const Duration(seconds: 1)), + throwsA(isA()), + ); + }); + + test('typedChain completes when its completion callback throws', () async { + final definition = TaskDefinition( + name: 'echo', + encodeArgs: (value) => {'value': value}, + decodeResult: (payload) => (payload! as num).toInt(), + ); + + await expectLater( + canvas + .typedChain(definition, 42) + .run( + onStepCompleted: (_, _, _) { + throw StateError('completion callback failed'); + }, + ) + .timeout(const Duration(seconds: 1)), + throwsA(isA()), + ); + }); + test('chord returns typed body results', () async { final result = await canvas.chord( body: [ @@ -101,6 +174,50 @@ void main() { expect(status.payload, equals(5)); }); + test( + 'chord policies can collect failures and allow a partial quorum', + () async { + final collected = await canvas.chord( + body: [ + task('echo', args: {'value': 2}), + task('fail'), + ], + callback: task('sum'), + policy: const ChordPolicy.collectTerminalResults(), + ); + + expect(collected.policy, const ChordPolicy.collectTerminalResults()); + expect(collected.values, equals([2, null])); + expect(collected.failures, hasLength(1)); + final collectedCallback = await _waitForSuccess( + backend, + collected.callbackTaskId, + ); + expect(collectedCallback.payload, equals(2)); + + final partial = await canvas.chord( + body: [ + task('echo', args: {'value': 3}), + task('fail'), + ], + callback: task('sum'), + policy: const ChordPolicy.allowPartial(minSuccessful: 1), + ); + + expect( + partial.policy, + const ChordPolicy.allowPartial(minSuccessful: 1), + ); + expect(partial.values, equals([3, null])); + expect(partial.failures, hasLength(1)); + final partialCallback = await _waitForSuccess( + backend, + partial.callbackTaskId, + ); + expect(partialCallback.payload, equals(3)); + }, + ); + test( 'submitBatch returns stable id and terminal lifecycle summary', () async { @@ -249,6 +366,25 @@ class _EchoTask implements TaskHandler { } } +class _FailTask implements TaskHandler { + @override + String get name => 'fail'; + + @override + TaskOptions get options => const TaskOptions(); + + @override + TaskMetadata get metadata => const TaskMetadata(); + + @override + TaskEntrypoint? get isolateEntrypoint => null; + + @override + Future call(TaskContext context, Map args) { + throw StateError('expected chord failure'); + } +} + class _SumTask implements TaskHandler { @override String get name => 'sum'; @@ -277,3 +413,41 @@ class _SumTask implements TaskHandler { return previousValue + addValue; } } + +class _StringifyTask implements TaskHandler { + @override + String get name => 'stringify'; + + @override + TaskOptions get options => const TaskOptions(); + + @override + TaskMetadata get metadata => const TaskMetadata(); + + @override + TaskEntrypoint? get isolateEntrypoint => null; + + @override + Future call(TaskContext context, Map args) async { + return '${args['value']}'; + } +} + +class _LengthTask implements TaskHandler { + @override + String get name => 'length'; + + @override + TaskOptions get options => const TaskOptions(); + + @override + TaskMetadata get metadata => const TaskMetadata(); + + @override + TaskEntrypoint? get isolateEntrypoint => null; + + @override + Future call(TaskContext context, Map args) async { + return (args['value']! as String).length; + } +} diff --git a/packages/stem/test/unit/core/contracts_test.dart b/packages/stem/test/unit/core/contracts_test.dart index 765f4b78..9e31008f 100644 --- a/packages/stem/test/unit/core/contracts_test.dart +++ b/packages/stem/test/unit/core/contracts_test.dart @@ -1,3 +1,4 @@ +import 'package:stem/memory.dart'; import 'package:stem/src/core/contracts.dart'; import 'package:stem/src/core/envelope.dart'; import 'package:stem/src/core/payload_codec.dart'; @@ -5,6 +6,37 @@ import 'package:stem/src/scheduler/schedule_spec.dart'; import 'package:test/test.dart'; void main() { + group('BrokerCapabilities', () { + test('keeps optional adapter behavior in one snapshot', () { + const capabilities = BrokerCapabilities( + supportsDelayedDelivery: true, + supportsPriorityOrdering: false, + supportsBroadcastFanout: true, + supportsQueueInspection: true, + ); + + expect(capabilities.supportsDelayedDelivery, isTrue); + expect(capabilities.supportsPriorityOrdering, isFalse); + expect( + capabilities.deliveryGuarantee, + BrokerDeliveryGuarantee.unknown, + ); + expect(capabilities.supportsBroadcastFanout, isTrue); + expect(capabilities.supportsQueueInspection, isTrue); + expect(capabilities.supportsLeaseExtension, isFalse); + expect(capabilities.supportsDeadLetterReplay, isFalse); + }); + + test('built-in memory broker declares optional capabilities', () { + final broker = InMemoryBroker(); + + expect(broker, isA()); + expect(broker, isA()); + expect(broker, isA()); + expect(broker, isA()); + }); + }); + group('RoutingSubscription', () { test('trims inputs and rejects empty', () { expect( @@ -452,7 +484,7 @@ void main() { expect(options.maxRetries, equals(5)); expect(options.softTimeLimit, equals(const Duration(milliseconds: 1000))); expect(options.hardTimeLimit, equals(const Duration(milliseconds: 2000))); - expect(options.groupRateLimit, equals('25/m')); + expect(options.groupRateLimit, equals(const RateLimit.perMinute(25))); expect(options.groupRateKey, equals('tenant:acme')); expect(options.groupRateKeyHeader, equals('x-tenant')); expect( @@ -492,6 +524,18 @@ void main() { expect(decoded.autoRetryFor, equals(['Timeout'])); expect(decoded.dontAutoRetryFor, equals(['StateError'])); }); + + test('RateLimit provides typed constructors and legacy parsing', () { + const perSecond = RateLimit.perSecond(10); + const perMinute = RateLimit.perMinute(25); + + expect(perSecond.tokens, equals(10)); + expect(perSecond.interval, const Duration(seconds: 1)); + expect(perSecond.toString(), equals('10/s')); + expect(RateLimit.parse('25/m'), equals(perMinute)); + expect(RateLimit.parse(perSecond), same(perSecond)); + expect(RateLimit.parse('invalid'), isNull); + }); }); group('ScheduleEntry', () { diff --git a/packages/stem/test/unit/core/fake_stem_test.dart b/packages/stem/test/unit/core/fake_stem_test.dart index d170b467..6a534652 100644 --- a/packages/stem/test/unit/core/fake_stem_test.dart +++ b/packages/stem/test/unit/core/fake_stem_test.dart @@ -1,4 +1,5 @@ import 'package:stem/stem.dart'; +import 'package:stem/testing.dart'; import 'package:test/test.dart'; class _Args { diff --git a/packages/stem/test/unit/core/function_task_handler_test.dart b/packages/stem/test/unit/core/function_task_handler_test.dart index f62db6ef..01315293 100644 --- a/packages/stem/test/unit/core/function_task_handler_test.dart +++ b/packages/stem/test/unit/core/function_task_handler_test.dart @@ -47,10 +47,50 @@ void main() { expect(result, equals(5)); expect(argValue, equals('stem')); expect(handler.isolateEntrypoint, isNotNull); + expect(handler.executionMode, TaskExecutionMode.isolate); expect(heartbeats, equals(1)); expect(extended, equals(const Duration(seconds: 3))); expect(progressValue, equals(0.5)); expect(progressData, equals({'stage': 'halfway'})); }, ); + + test('task code can observe cooperative cancellation', () async { + final token = TaskCancellationToken(); + final handler = FunctionTaskHandler.inline( + name: 'cancelled.task', + entrypoint: (invocation, args) async { + invocation.cancellation.throwIfCancelled(); + return null; + }, + ); + token.cancel(); + + expect( + () => handler( + TaskContext( + id: 'task-2', + attempt: 1, + headers: const {}, + meta: const {}, + cancellation: token, + heartbeat: () {}, + extendLease: (_) async {}, + progress: (_, {data}) async {}, + ), + const {}, + ), + throwsA(isA()), + ); + }); + + test('inline handlers declare coordinator-isolate execution', () { + final handler = FunctionTaskHandler.inline( + name: 'inline.task', + entrypoint: (invocation, args) async => null, + ); + + expect(handler.executionMode, TaskExecutionMode.inline); + expect(handler.isolateEntrypoint, isNull); + }); } diff --git a/packages/stem/test/unit/core/queue_broker_test.dart b/packages/stem/test/unit/core/queue_broker_test.dart new file mode 100644 index 00000000..7f155a49 --- /dev/null +++ b/packages/stem/test/unit/core/queue_broker_test.dart @@ -0,0 +1,132 @@ +import 'package:stem/memory.dart'; +import 'package:stem/stem.dart'; +import 'package:test/test.dart'; + +void main() { + test( + 'a queue-only transport can run Stem without optional broker APIs', + () async { + final broker = _QueueOnlyBroker(); + expect(broker.capabilities.supportsLeaseExtension, isFalse); + expect( + broker.capabilities.deliveryGuarantee, + BrokerDeliveryGuarantee.unknown, + ); + expect(broker.capabilities.supportsDeadLetterReplay, isFalse); + final backend = InMemoryResultBackend(); + final task = FunctionTaskHandler.inline( + name: 'queue-only.echo', + entrypoint: (context, args) async => 'ok', + ); + final worker = Worker( + broker: broker, + backend: backend, + tasks: [task], + consumerName: 'queue-only-worker', + concurrency: 1, + prefetchMultiplier: 1, + lifecycle: const WorkerLifecycleConfig(installSignalHandlers: false), + ); + + await worker.start(); + try { + final producer = Stem( + broker: broker, + backend: backend, + tasks: [task], + ); + final taskId = await producer.enqueue(task.name); + final status = await backend + .watch(taskId) + .firstWhere( + (value) => value.state == TaskState.succeeded, + ); + + expect(status.payload, 'ok'); + } finally { + await worker.shutdown(); + await broker.close(); + await backend.close(); + } + }, + ); + + test( + 'queue-only failures are terminal without dead-letter support', + () async { + final broker = _QueueOnlyBroker(); + final backend = InMemoryResultBackend(); + final task = FunctionTaskHandler.inline( + name: 'queue-only.failure', + entrypoint: (context, args) async { + throw StateError('expected failure'); + }, + ); + final worker = Worker( + broker: broker, + backend: backend, + tasks: [task], + consumerName: 'queue-only-failure-worker', + concurrency: 1, + prefetchMultiplier: 1, + lifecycle: const WorkerLifecycleConfig(installSignalHandlers: false), + ); + + await worker.start(); + try { + final producer = Stem( + broker: broker, + backend: backend, + tasks: [task], + ); + final taskId = await producer.enqueue(task.name); + final status = await backend + .watch(taskId) + .firstWhere( + (value) => value.state == TaskState.failed, + ); + + expect(status.error?.message, contains('expected failure')); + } finally { + await worker.shutdown(); + await broker.close(); + await backend.close(); + } + }, + ); +} + +/// Deliberately implements only [QueueBroker], not the compatibility [Broker] +/// facade or any optional capability interface. +final class _QueueOnlyBroker implements QueueBroker { + _QueueOnlyBroker() : _delegate = InMemoryBroker(); + + final InMemoryBroker _delegate; + + @override + Future publish(Envelope envelope, {RoutingInfo? routing}) => + _delegate.publish(envelope, routing: routing); + + @override + Stream consume( + RoutingSubscription subscription, { + int prefetch = 1, + String? consumerGroup, + String? consumerName, + }) => _delegate.consume( + subscription, + prefetch: prefetch, + consumerGroup: consumerGroup, + consumerName: consumerName, + ); + + @override + Future ack(Delivery delivery) => _delegate.ack(delivery); + + @override + Future nack(Delivery delivery, {bool requeue = true}) => + _delegate.nack(delivery, requeue: requeue); + + @override + Future close() => _delegate.close(); +} diff --git a/packages/stem/test/unit/core/queue_events_test.dart b/packages/stem/test/unit/core/queue_events_test.dart index f6b57c6f..6539a361 100644 --- a/packages/stem/test/unit/core/queue_events_test.dart +++ b/packages/stem/test/unit/core/queue_events_test.dart @@ -1,5 +1,6 @@ import 'dart:async'; +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; import 'package:test/test.dart'; diff --git a/packages/stem/test/unit/core/stem_core_test.dart b/packages/stem/test/unit/core/stem_core_test.dart index 7fe54382..790deabb 100644 --- a/packages/stem/test/unit/core/stem_core_test.dart +++ b/packages/stem/test/unit/core/stem_core_test.dart @@ -1,5 +1,6 @@ import 'dart:async'; +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; import 'package:test/test.dart'; @@ -1186,6 +1187,12 @@ class _RecordingBroker implements Broker { @override bool get supportsPriority => false; + BrokerCapabilities get capabilities => const BrokerCapabilities( + supportsDelayedDelivery: true, + supportsPriorityOrdering: false, + supportsQueueInspection: true, + ); + @override Future listDeadLetters( String queue, { diff --git a/packages/stem/test/unit/core/stem_enqueue_options_test.dart b/packages/stem/test/unit/core/stem_enqueue_options_test.dart index 1c91b647..fd15818c 100644 --- a/packages/stem/test/unit/core/stem_enqueue_options_test.dart +++ b/packages/stem/test/unit/core/stem_enqueue_options_test.dart @@ -233,6 +233,11 @@ class _RecordingBroker implements Broker { @override bool get supportsPriority => true; + BrokerCapabilities get capabilities => const BrokerCapabilities( + supportsDelayedDelivery: true, + supportsPriorityOrdering: true, + ); + @override Future close() async {} } diff --git a/packages/stem/test/unit/core/stem_unique_task_test.dart b/packages/stem/test/unit/core/stem_unique_task_test.dart index 5aa51620..8d7eff26 100644 --- a/packages/stem/test/unit/core/stem_unique_task_test.dart +++ b/packages/stem/test/unit/core/stem_unique_task_test.dart @@ -1,3 +1,4 @@ +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; import 'package:test/test.dart'; diff --git a/packages/stem/test/unit/core/task_invocation_test.dart b/packages/stem/test/unit/core/task_invocation_test.dart index 5505a70e..25d00cb0 100644 --- a/packages/stem/test/unit/core/task_invocation_test.dart +++ b/packages/stem/test/unit/core/task_invocation_test.dart @@ -178,7 +178,7 @@ void main() { attempt: 0, heartbeat: () {}, extendLease: (_) async {}, - progress: (_, {Map? data}) async {}, + progress: (_, {data}) async {}, ); expect(context.requiredArg('customerId'), equals('cus-42')); @@ -201,7 +201,7 @@ void main() { attempt: 0, heartbeat: () {}, extendLease: (_) async {}, - progress: (_, {Map? data}) async {}, + progress: (_, {data}) async {}, ); expect( @@ -244,7 +244,7 @@ void main() { attempt: 0, heartbeat: () {}, extendLease: (_) async {}, - progress: (percent, {Map? data}) async { + progress: (percent, {data}) async { progressSignal = ProgressSignal(percent, data: data); }, ); @@ -266,7 +266,7 @@ void main() { attempt: 0, heartbeat: () {}, extendLease: (_) async {}, - progress: (percent, {Map? data}) async { + progress: (percent, {data}) async { progressSignal = ProgressSignal(percent, data: data); }, ); @@ -342,7 +342,7 @@ void main() { attempt: 2, heartbeat: () {}, extendLease: (_) async {}, - progress: (_, {Map? data}) async {}, + progress: (_, {data}) async {}, enqueuer: enqueuer, ); @@ -369,7 +369,7 @@ void main() { attempt: 0, heartbeat: () {}, extendLease: (_) async {}, - progress: (_, {Map? data}) async {}, + progress: (_, {data}) async {}, enqueuer: enqueuer, ); final scheduledAt = DateTime.now().add(const Duration(minutes: 5)); @@ -388,7 +388,7 @@ void main() { attempt: 0, heartbeat: () {}, extendLease: (_) async {}, - progress: (_, {Map? data}) async {}, + progress: (_, {data}) async {}, enqueuer: enqueuer, ); final scheduledAt = DateTime.now().add(const Duration(minutes: 5)); @@ -406,7 +406,7 @@ void main() { attempt: 0, heartbeat: () {}, extendLease: (_) async {}, - progress: (_, {Map? data}) async {}, + progress: (_, {data}) async {}, ); expect( @@ -435,7 +435,7 @@ void main() { attempt: 1, heartbeat: () {}, extendLease: (_) async {}, - progress: (_, {Map? data}) async {}, + progress: (_, {data}) async {}, enqueuer: enqueuer, ); @@ -468,7 +468,7 @@ void main() { attempt: 1, heartbeat: () {}, extendLease: (_) async {}, - progress: (_, {Map? data}) async {}, + progress: (_, {data}) async {}, workflows: workflows, ); final definition = WorkflowRef, String>( @@ -498,7 +498,7 @@ void main() { attempt: 1, heartbeat: () {}, extendLease: (_) async {}, - progress: (_, {Map? data}) async {}, + progress: (_, {data}) async {}, workflowEvents: workflowEvents, ); @@ -790,7 +790,7 @@ void main() { attempt: 0, heartbeat: () {}, extendLease: (_) async {}, - progress: (_, {Map? data}) async {}, + progress: (_, {data}) async {}, enqueuer: _CapturingEnqueuer('noop'), ); diff --git a/packages/stem/test/unit/core/task_registry_test.dart b/packages/stem/test/unit/core/task_registry_test.dart index 572ae603..d5290c30 100644 --- a/packages/stem/test/unit/core/task_registry_test.dart +++ b/packages/stem/test/unit/core/task_registry_test.dart @@ -105,6 +105,11 @@ class _FakeBroker implements Broker { @override bool get supportsPriority => true; + BrokerCapabilities get capabilities => const BrokerCapabilities( + supportsDelayedDelivery: true, + supportsPriorityOrdering: true, + ); + @override Future close() async {} } @@ -206,6 +211,54 @@ void main() { expect(call.resolveOptions(), const TaskOptions()); }); + test( + 'typed handlers decode arguments before invoking application code', + () async { + final definition = TaskDefinition<_Args, int>.codec( + name: 'demo.typed', + argsCodec: PayloadCodec<_Args>.map( + encode: (args) => {'value': args.value}, + decode: (payload) => _Args(payload['value'] as int), + ), + ); + final handler = definition.handler( + entrypoint: (context, args) async => args.value * 2, + ); + final context = TaskContext( + id: 'typed-task', + attempt: 1, + headers: const {}, + meta: const {}, + heartbeat: () {}, + extendLease: (_) async {}, + progress: (_, {data}) async {}, + ); + + expect(await handler.call(context, {'value': 21}), 42); + }, + ); + + test('json definitions can decode arguments for typed handlers', () async { + final definition = TaskDefinition<_Args, int>.json( + name: 'demo.typed.json', + decodeArgsJson: (payload) => _Args(payload['value'] as int), + ); + final handler = definition.handler( + entrypoint: (context, args) async => args.value + 1, + ); + final context = TaskContext( + id: 'typed-json-task', + attempt: 1, + headers: const {}, + meta: const {}, + heartbeat: () {}, + extendLease: (_) async {}, + progress: (_, {data}) async {}, + ); + + expect(await handler.call(context, {'value': 41}), 42); + }); + test('enqueues via Stem.enqueueCall', () async { final broker = _FakeBroker(); final stem = Stem( diff --git a/packages/stem/test/unit/core/unique_task_coordinator_test.dart b/packages/stem/test/unit/core/unique_task_coordinator_test.dart index e58b6e7c..ac764d37 100644 --- a/packages/stem/test/unit/core/unique_task_coordinator_test.dart +++ b/packages/stem/test/unit/core/unique_task_coordinator_test.dart @@ -1,3 +1,4 @@ +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; import 'package:test/test.dart'; diff --git a/packages/stem/test/unit/observability/logging_test.dart b/packages/stem/test/unit/observability/logging_test.dart index 0d4c319b..a65b275e 100644 --- a/packages/stem/test/unit/observability/logging_test.dart +++ b/packages/stem/test/unit/observability/logging_test.dart @@ -1,17 +1,21 @@ import 'package:ansicolor/ansicolor.dart' show ansiColorDisabled; import 'package:contextual/contextual.dart' show + Context, + Level, LogDriver, LogEntry, LogRecord, + Logger, LoggerChannelSelection, PlainTextLogFormatter, PrettyLogFormatter; -import 'package:stem/stem.dart'; +import 'package:stem/src/observability/logging.dart'; +import 'package:stem/src/observability/logging_types.dart'; import 'package:test/test.dart'; void main() { - test('package:stem exports logging types used by the public API', () { + test('internal logging implementation supports contextual integration', () { void acceptsStemLogger(Logger logger, Level level) { logger.setLevel(level); } diff --git a/packages/stem/test/unit/observability/metrics_integration_test.dart b/packages/stem/test/unit/observability/metrics_integration_test.dart index 29129e35..5e0acbaa 100644 --- a/packages/stem/test/unit/observability/metrics_integration_test.dart +++ b/packages/stem/test/unit/observability/metrics_integration_test.dart @@ -1,5 +1,6 @@ import 'dart:async'; +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; import 'package:test/test.dart'; diff --git a/packages/stem/test/unit/redis_components_test.dart b/packages/stem/test/unit/redis_components_test.dart index 7ada06d0..f633cd41 100644 --- a/packages/stem/test/unit/redis_components_test.dart +++ b/packages/stem/test/unit/redis_components_test.dart @@ -1,5 +1,6 @@ import 'dart:async'; +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; import 'package:test/test.dart'; diff --git a/packages/stem/test/unit/scheduler/beat_test.dart b/packages/stem/test/unit/scheduler/beat_test.dart index db62d90a..5b6e8be0 100644 --- a/packages/stem/test/unit/scheduler/beat_test.dart +++ b/packages/stem/test/unit/scheduler/beat_test.dart @@ -1,6 +1,7 @@ import 'dart:async'; import 'dart:convert'; +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; import 'package:test/test.dart'; @@ -124,6 +125,43 @@ void main() { broker.dispose(); }); + test( + 'propagates the scheduler fencing token to scheduled envelopes', + () async { + final broker = InMemoryBroker(); + final store = InMemoryScheduleStore(); + final beat = Beat( + store: store, + broker: broker, + lockStore: InMemoryLockStore(), + tickInterval: const Duration(milliseconds: 10), + ); + final deliveryFuture = broker + .consume( + RoutingSubscription.singleQueue('default'), + consumerName: 'beat-fencing-test', + ) + .first + .timeout(const Duration(seconds: 1)); + + await store.upsert( + ScheduleEntry( + id: 'fenced-schedule', + taskName: 'fenced.task', + queue: 'default', + spec: ClockedScheduleSpec(runAt: DateTime.now()), + ), + ); + await beat.start(); + + final delivery = await deliveryFuture; + expect(delivery.envelope.headers['stem-lock-fencing-token'], '1'); + await broker.ack(delivery); + await beat.stop(); + broker.dispose(); + }, + ); + test('disables one-shot schedules after execution', () async { final broker = InMemoryBroker(); final registry = InMemoryTaskRegistry()..register(_NoopTask()); @@ -224,6 +262,32 @@ void main() { broker.dispose(); }); + test('does not publish after losing the schedule lease', () async { + final broker = InMemoryBroker(); + final store = InMemoryScheduleStore(); + final beat = Beat( + store: store, + broker: broker, + lockStore: _LeaseLossLockStore(), + ); + await store.upsert( + ScheduleEntry( + id: 'lease-loss', + taskName: 'noop', + queue: 'default', + spec: IntervalScheduleSpec(every: const Duration(seconds: 1)), + nextRunAt: DateTime.now().subtract(const Duration(seconds: 1)), + ), + ); + + await beat.runOnce(); + + expect(await broker.pendingCount('default'), equals(0)); + expect((await store.get('lease-loss'))?.lastError, contains('lock')); + await beat.stop(); + broker.dispose(); + }); + test('emits scheduler signals on successful dispatch', () async { StemSignals.configure(configuration: const StemSignalConfiguration()); final broker = InMemoryBroker(); @@ -349,3 +413,39 @@ class _ThrowingBroker extends InMemoryBroker { return super.publish(envelope, routing: routing); } } + +class _LeaseLossLockStore implements LockStore { + @override + Future acquire( + String key, { + Duration ttl = const Duration(seconds: 30), + String? owner, + }) async { + return _LeaseLossLock(key, owner ?? 'lease-loss-owner'); + } + + @override + Future ownerOf(String key) async => 'lease-loss-owner'; + + @override + Future release(String key, String owner) async => true; + + @override + Future renew(String key, String owner, Duration ttl) async => false; +} + +class _LeaseLossLock implements Lock { + _LeaseLossLock(this.key, this.owner); + + @override + final String key; + + @override + final String owner; + + @override + Future renew(Duration ttl) async => false; + + @override + Future release() async {} +} diff --git a/packages/stem/test/unit/scheduler/lock_store_test.dart b/packages/stem/test/unit/scheduler/lock_store_test.dart new file mode 100644 index 00000000..48542900 --- /dev/null +++ b/packages/stem/test/unit/scheduler/lock_store_test.dart @@ -0,0 +1,38 @@ +import 'package:stem/memory.dart'; +import 'package:stem/stem.dart'; +import 'package:test/test.dart'; + +void main() { + test('in-memory lock fencing tokens increase for each acquisition', () async { + final store = InMemoryLockStore(); + + final first = await store.acquire('fenced', owner: 'first'); + expect(first, isA()); + expect(first!.fencingToken, equals(1)); + await first.release(); + + final second = await store.acquire('fenced', owner: 'second'); + expect(second, isA()); + expect(second!.fencingToken, equals(2)); + await second.release(); + }); + + test('legacy locks expose no fencing token', () async { + final lock = _LegacyLock(); + expect(lock.fencingToken, isNull); + }); +} + +class _LegacyLock implements Lock { + @override + String get key => 'legacy'; + + @override + String get owner => 'legacy-owner'; + + @override + Future renew(Duration ttl) async => true; + + @override + Future release() async {} +} diff --git a/packages/stem/test/unit/scheduler/schedule_store_test.dart b/packages/stem/test/unit/scheduler/schedule_store_test.dart index 5ee27b01..eb33fe96 100644 --- a/packages/stem/test/unit/scheduler/schedule_store_test.dart +++ b/packages/stem/test/unit/scheduler/schedule_store_test.dart @@ -1,3 +1,4 @@ +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; import 'package:test/test.dart'; diff --git a/packages/stem/test/unit/security/signing_test.dart b/packages/stem/test/unit/security/signing_test.dart index 5faf9eeb..9ad02025 100644 --- a/packages/stem/test/unit/security/signing_test.dart +++ b/packages/stem/test/unit/security/signing_test.dart @@ -2,6 +2,7 @@ import 'dart:convert'; import 'package:contextual/contextual.dart'; import 'package:cryptography/cryptography.dart'; +import 'package:stem/src/observability/logging.dart' show stemLogger; import 'package:stem/stem.dart'; import 'package:test/test.dart'; diff --git a/packages/stem/test/unit/signals/middleware_test.dart b/packages/stem/test/unit/signals/middleware_test.dart index b18d4cd1..30752989 100644 --- a/packages/stem/test/unit/signals/middleware_test.dart +++ b/packages/stem/test/unit/signals/middleware_test.dart @@ -68,7 +68,7 @@ void main() { meta: envelope.meta, heartbeat: () {}, extendLease: (_) async {}, - progress: (_, {Map? data}) async {}, + progress: (_, {data}) async {}, ); await middleware.onExecute(context, () async {}); @@ -105,7 +105,7 @@ void main() { meta: envelope.meta, heartbeat: () {}, extendLease: (_) async {}, - progress: (_, {Map? data}) async {}, + progress: (_, {data}) async {}, ); await expectLater( diff --git a/packages/stem/test/unit/signals/payloads_test.dart b/packages/stem/test/unit/signals/payloads_test.dart index 66432480..b73599f1 100644 --- a/packages/stem/test/unit/signals/payloads_test.dart +++ b/packages/stem/test/unit/signals/payloads_test.dart @@ -31,7 +31,7 @@ void main() { meta: const {}, heartbeat: () {}, extendLease: (_) async {}, - progress: (_, {Map? data}) async {}, + progress: (_, {data}) async {}, ); final prerun = TaskPrerunPayload( envelope: envelope, diff --git a/packages/stem/test/unit/tracing/tracing_test.dart b/packages/stem/test/unit/tracing/tracing_test.dart index 76c5218a..c6a4f7d0 100644 --- a/packages/stem/test/unit/tracing/tracing_test.dart +++ b/packages/stem/test/unit/tracing/tracing_test.dart @@ -1,6 +1,7 @@ import 'dart:async'; import 'package:dartastic_opentelemetry/dartastic_opentelemetry.dart' as dotel; +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; import 'package:test/test.dart'; @@ -259,6 +260,65 @@ void main() { expect(consumeTraceIds.length, equals(2)); }); + test('Canvas group links body consumption to its composition span', () async { + final broker = InMemoryBroker(); + final backend = InMemoryResultBackend(); + final registry = InMemoryTaskRegistry() + ..register( + FunctionTaskHandler.inline( + name: 'trace.group', + entrypoint: (context, args) async => null, + ), + ); + final canvas = Canvas( + broker: broker, + backend: backend, + registry: registry, + ); + final worker = Worker( + broker: broker, + registry: registry, + backend: backend, + consumerName: 'trace-group-worker', + heartbeatTransport: const NoopHeartbeatTransport(), + concurrency: 2, + ); + await worker.start(); + + final dispatch = await canvas.group([ + task('trace.group'), + task('trace.group'), + ]); + final results = await dispatch.results.toList(); + await dispatch.dispose(); + await worker.shutdown(); + broker.dispose(); + + expect(results, hasLength(2)); + final compositionSpan = exporter.spans.firstWhere( + (span) => span.name == 'stem.canvas.group', + ); + final consumeSpans = exporter.spans + .where( + (span) => + span.name == 'stem.consume' && + span.attributes.getString('stem.task') == 'trace.group', + ) + .toList(growable: false); + expect(consumeSpans, hasLength(2)); + for (final span in consumeSpans) { + expect( + span.spanLinks?.any( + (link) => + link.spanContext.spanId.hexString == + compositionSpan.spanContext.spanId.hexString, + ) ?? + false, + isTrue, + ); + } + }); + test( 'enqueue span uses ambient parent context when headers are absent', () async { diff --git a/packages/stem/test/unit/worker/task_context_enqueue_integration_test.dart b/packages/stem/test/unit/worker/task_context_enqueue_integration_test.dart index e8b2f532..04710b3c 100644 --- a/packages/stem/test/unit/worker/task_context_enqueue_integration_test.dart +++ b/packages/stem/test/unit/worker/task_context_enqueue_integration_test.dart @@ -1,6 +1,7 @@ import 'dart:async'; import 'package:property_testing/property_testing.dart'; +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; import 'package:test/test.dart'; @@ -101,6 +102,7 @@ void main() { tasks: [_IsolateStartWorkflowTask()], flows: [_childWorkflow], ); + await app.start(); final taskId = await app.enqueue('tasks.isolate.start.workflow'); final result = await app.waitForTask( @@ -119,6 +121,7 @@ void main() { tasks: [_IsolateEmitWorkflowEventTask()], flows: [_waitingWorkflow], ); + await app.start(); final runId = await _waitingWorkflowRef.start(app); final taskId = await app.enqueue('tasks.isolate.emit.workflow.event'); diff --git a/packages/stem/test/unit/worker/task_retry_policy_test.dart b/packages/stem/test/unit/worker/task_retry_policy_test.dart index 12542303..6152dd02 100644 --- a/packages/stem/test/unit/worker/task_retry_policy_test.dart +++ b/packages/stem/test/unit/worker/task_retry_policy_test.dart @@ -1,6 +1,7 @@ import 'dart:async'; import 'package:property_testing/property_testing.dart'; +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; import 'package:test/test.dart'; diff --git a/packages/stem/test/unit/worker/worker_delivery_tracker_test.dart b/packages/stem/test/unit/worker/worker_delivery_tracker_test.dart new file mode 100644 index 00000000..564b5d63 --- /dev/null +++ b/packages/stem/test/unit/worker/worker_delivery_tracker_test.dart @@ -0,0 +1,62 @@ +import 'package:stem/src/worker/worker_delivery_tracker.dart'; +import 'package:stem/stem.dart'; +import 'package:test/test.dart'; + +void main() { + test('tracks and releases deliveries with queue accounting', () { + final tracker = WorkerDeliveryTracker(); + final first = _delivery('first', 'orders'); + final second = _delivery('second', 'orders'); + final startedAt = DateTime.utc(2026, 8, 19); + + tracker + ..track(first, startedAt: startedAt) + ..track(second, startedAt: startedAt); + + expect(tracker.inflight, 2); + expect(tracker.inflightPerQueue, {'orders': 2}); + expect( + tracker.active.values + .firstWhere((active) => identical(active.delivery, first)) + .startedAt, + startedAt, + ); + + expect(tracker.release(first)?.delivery, same(first)); + expect(tracker.inflight, 1); + expect(tracker.inflightPerQueue, {'orders': 1}); + expect(tracker.release(first), isNull); + + tracker.release(second); + + final duplicate = _delivery('second', 'orders'); + tracker + ..track(second, startedAt: startedAt) + ..track(duplicate, startedAt: startedAt); + expect(tracker.inflight, 2); + expect(tracker.containsEnvelopeId('second'), isTrue); + expect(tracker.forEnvelopeId('second')?.delivery, same(second)); + expect(tracker.forDelivery(duplicate)?.delivery, same(duplicate)); + expect(tracker.release(duplicate)?.delivery, same(duplicate)); + expect(tracker.release(second)?.delivery, same(second)); + + tracker.clear(); + expect(tracker.inflight, 0); + expect(tracker.active, isEmpty); + expect(tracker.inflightPerQueue, isEmpty); + }); +} + +Delivery _delivery(String id, String queue, {String? receipt}) { + final envelope = Envelope( + id: id, + name: 'test.task', + args: const {}, + queue: queue, + ); + return Delivery( + envelope: envelope, + receipt: receipt ?? 'receipt-$id', + leaseExpiresAt: null, + ); +} diff --git a/packages/stem/test/unit/worker/worker_execution_supervisor_test.dart b/packages/stem/test/unit/worker/worker_execution_supervisor_test.dart new file mode 100644 index 00000000..567c006b --- /dev/null +++ b/packages/stem/test/unit/worker/worker_execution_supervisor_test.dart @@ -0,0 +1,70 @@ +import 'package:stem/src/core/contracts.dart'; +import 'package:stem/src/core/envelope.dart'; +import 'package:stem/src/core/task_invocation.dart'; +import 'package:stem/src/worker/worker_config.dart'; +import 'package:stem/src/worker/worker_execution_supervisor.dart'; +import 'package:test/test.dart'; + +void main() { + test('rejects isolate mode without an entrypoint', () async { + final supervisor = WorkerExecutionSupervisor( + concurrency: 1, + lifecycle: const WorkerLifecycleConfig(installSignalHandlers: false), + onRecycle: (_) {}, + onSpawned: (_) {}, + onDisposed: (_) {}, + ); + addTearDown(supervisor.dispose); + + final handler = _InvalidIsolateHandler(); + final context = TaskContext( + id: 'task-1', + attempt: 0, + headers: const {}, + meta: const {}, + heartbeat: () {}, + extendLease: (_) async {}, + progress: (_, {data}) async {}, + ); + + await expectLater( + supervisor.execute( + handler: handler, + context: context, + envelope: Envelope(name: handler.name, args: const {}), + args: const {}, + controlHandler: (_) {}, + ), + throwsA( + isA().having( + (error) => error.toString(), + 'message', + contains('declares isolate execution'), + ), + ), + ); + }); +} + +class _InvalidIsolateHandler + implements TaskHandler, TaskExecutionModeProvider { + @override + TaskEntrypoint? get isolateEntrypoint => null; + + @override + Future call(TaskContext context, Map args) async { + return null; + } + + @override + TaskExecutionMode get executionMode => TaskExecutionMode.isolate; + + @override + TaskMetadata get metadata => const TaskMetadata(); + + @override + String get name => 'invalid.isolate'; + + @override + TaskOptions get options => const TaskOptions(); +} diff --git a/packages/stem/test/unit/worker/worker_lease_coordinator_test.dart b/packages/stem/test/unit/worker/worker_lease_coordinator_test.dart new file mode 100644 index 00000000..01386ac3 --- /dev/null +++ b/packages/stem/test/unit/worker/worker_lease_coordinator_test.dart @@ -0,0 +1,327 @@ +import 'dart:async'; + +import 'package:stem/memory.dart'; +import 'package:stem/src/worker/worker_lease_coordinator.dart'; +import 'package:stem/stem.dart'; +import 'package:test/test.dart'; + +void main() { + test('renews a short lease before the default minimum interval', () async { + final broker = _RecordingLeaseBroker(); + final coordinator = WorkerLeaseCoordinator( + broker: broker, + onLeaseUpdated: (_) {}, + onLeaseRenewed: (_) {}, + onRenewalFailure: (_, error, _) { + throw StateError('Unexpected lease renewal failure: $error'); + }, + ); + final delivery = Delivery( + envelope: Envelope(name: 'short-lease.test', args: const {}), + receipt: 'short-lease-receipt', + leaseExpiresAt: DateTime.now().add(const Duration(milliseconds: 300)), + ); + + coordinator.schedule(delivery); + try { + await _waitFor( + () => broker.attempts >= 1, + timeout: const Duration(milliseconds: 600), + ); + expect(broker.attempts, greaterThanOrEqualTo(1)); + } finally { + coordinator.cancelAll(); + broker.dispose(); + } + }); + + test('passes the full lease duration instead of the timer cadence', () async { + final broker = _RecordingLeaseBroker(); + final coordinator = WorkerLeaseCoordinator( + broker: broker, + minimumInterval: const Duration(milliseconds: 5), + maximumInterval: const Duration(seconds: 1), + onLeaseUpdated: (_) {}, + onLeaseRenewed: (_) {}, + onRenewalFailure: (_, error, _) { + throw StateError('Unexpected lease renewal failure: $error'); + }, + ); + final delivery = Delivery( + envelope: Envelope( + name: 'full-lease.test', + args: const {}, + visibilityTimeout: const Duration(milliseconds: 200), + ), + receipt: 'full-lease-receipt', + leaseExpiresAt: DateTime.now().add(const Duration(milliseconds: 200)), + ); + + coordinator.schedule(delivery); + try { + await _waitFor(() => broker.attempts >= 1); + expect(broker.durations.first, const Duration(milliseconds: 200)); + } finally { + coordinator.cancelAll(); + broker.dispose(); + } + }); + + test( + 'restarts a short manually extended lease before the default minimum', + () async { + final broker = _RecordingLeaseBroker(); + final coordinator = WorkerLeaseCoordinator( + broker: broker, + onLeaseUpdated: (_) {}, + onLeaseRenewed: (_) {}, + onRenewalFailure: (_, error, _) { + throw StateError('Unexpected lease renewal failure: $error'); + }, + ); + final delivery = Delivery( + envelope: Envelope(name: 'short-lease-restart.test', args: const {}), + receipt: 'short-lease-restart-receipt', + leaseExpiresAt: DateTime.now().add(const Duration(seconds: 10)), + ); + + coordinator.restart(delivery, const Duration(milliseconds: 300)); + try { + await _waitFor( + () => broker.attempts >= 1, + timeout: const Duration(milliseconds: 600), + ); + expect(broker.attempts, greaterThanOrEqualTo(1)); + } finally { + coordinator.cancelAll(); + broker.dispose(); + } + }, + ); + + test('keeps same-receipt redeliveries on independent timers', () async { + final broker = _RecordingLeaseBroker(); + final coordinator = WorkerLeaseCoordinator( + broker: broker, + minimumInterval: const Duration(milliseconds: 5), + maximumInterval: const Duration(milliseconds: 20), + onLeaseUpdated: (_) {}, + onLeaseRenewed: (_) {}, + onRenewalFailure: (_, error, _) { + throw StateError('Unexpected lease renewal failure: $error'); + }, + ); + final first = Delivery( + envelope: Envelope(name: 'same-receipt.test', args: const {}), + receipt: 'reused-receipt', + leaseExpiresAt: DateTime.now().add(const Duration(milliseconds: 40)), + ); + final redelivery = Delivery( + envelope: first.envelope, + receipt: first.receipt, + leaseExpiresAt: DateTime.now().add(const Duration(milliseconds: 40)), + ); + + coordinator + ..schedule(first) + ..schedule(redelivery); + try { + await _waitFor( + () => broker.attempts >= 2, + timeout: const Duration(milliseconds: 600), + ); + expect(broker.attempts, greaterThanOrEqualTo(2)); + } finally { + coordinator + ..cancel(first) + ..cancel(redelivery) + ..cancelAll(); + broker.dispose(); + } + }); + + test('contains renewal failures and recovers on a later attempt', () async { + final broker = _FlakyLeaseBroker(); + final updated = []; + final renewed = []; + final failures = []; + final coordinator = WorkerLeaseCoordinator( + broker: broker, + minimumInterval: const Duration(milliseconds: 5), + maximumInterval: const Duration(milliseconds: 20), + onLeaseUpdated: updated.add, + onLeaseRenewed: renewed.add, + onRenewalFailure: (_, error, _) => failures.add(error), + ); + final delivery = Delivery( + envelope: Envelope(name: 'lease.test', args: const {}), + receipt: 'lease-receipt', + leaseExpiresAt: DateTime.now().add(const Duration(milliseconds: 30)), + ); + + coordinator.schedule(delivery); + try { + await _waitFor(() => broker.attempts >= 2); + expect(failures, hasLength(1)); + expect(updated, isNotEmpty); + expect(renewed, isNotEmpty); + } finally { + coordinator.cancelAll(); + broker.dispose(); + } + }); + + test( + 'retries a failed renewal before the original interval elapses', + () async { + final broker = _TimedFlakyLeaseBroker(); + final coordinator = WorkerLeaseCoordinator( + broker: broker, + minimumInterval: const Duration(milliseconds: 5), + maximumInterval: const Duration(seconds: 1), + onLeaseUpdated: (_) {}, + onLeaseRenewed: (_) {}, + onRenewalFailure: (_, error, _) => broker.failures.add(error), + ); + final delivery = Delivery( + envelope: Envelope(name: 'lease-retry.test', args: const {}), + receipt: 'lease-retry-receipt', + leaseExpiresAt: DateTime.now().add(const Duration(milliseconds: 100)), + ); + + coordinator.schedule(delivery); + try { + await _waitFor(() => broker.attempts >= 2); + expect(broker.failures, hasLength(1)); + expect( + broker.attemptTimes[1].difference(broker.attemptTimes[0]), + lessThan(const Duration(milliseconds: 50)), + ); + } finally { + coordinator.cancelAll(); + broker.dispose(); + } + }, + ); + + test('does not restart after an in-flight renewal is cancelled', () async { + final broker = _BlockingLeaseBroker(); + final failures = []; + final coordinator = WorkerLeaseCoordinator( + broker: broker, + minimumInterval: const Duration(milliseconds: 5), + maximumInterval: const Duration(milliseconds: 20), + onLeaseUpdated: (_) {}, + onLeaseRenewed: (_) {}, + onRenewalFailure: (_, error, _) => failures.add(error), + ); + final delivery = Delivery( + envelope: Envelope(name: 'lease-cancel.test', args: const {}), + receipt: 'lease-cancel-receipt', + leaseExpiresAt: DateTime.now().add(const Duration(milliseconds: 30)), + ); + + coordinator.schedule(delivery); + try { + await broker.started.future.timeout(const Duration(seconds: 1)); + coordinator.cancel(delivery); + broker.release.complete(); + await Future.delayed(const Duration(milliseconds: 50)); + + expect(broker.attempts, 1); + expect(failures, isEmpty); + } finally { + coordinator.cancelAll(); + broker.dispose(); + } + }); +} + +class _FlakyLeaseBroker extends InMemoryBroker { + _FlakyLeaseBroker() + : super( + delayedInterval: const Duration(milliseconds: 5), + claimInterval: const Duration(milliseconds: 20), + ); + + int attempts = 0; + + @override + Future extendLease(Delivery delivery, Duration by) async { + attempts += 1; + if (attempts == 1) { + throw StateError('simulated lease renewal failure'); + } + } +} + +class _RecordingLeaseBroker extends InMemoryBroker { + _RecordingLeaseBroker() + : super( + delayedInterval: const Duration(milliseconds: 5), + claimInterval: const Duration(milliseconds: 20), + ); + + int attempts = 0; + final durations = []; + + @override + Future extendLease(Delivery delivery, Duration by) async { + attempts += 1; + durations.add(by); + } +} + +class _TimedFlakyLeaseBroker extends InMemoryBroker { + _TimedFlakyLeaseBroker() + : super( + delayedInterval: const Duration(milliseconds: 5), + claimInterval: const Duration(milliseconds: 20), + ); + + final attemptTimes = []; + final failures = []; + + int get attempts => attemptTimes.length; + + @override + Future extendLease(Delivery delivery, Duration by) async { + attemptTimes.add(DateTime.now()); + if (attemptTimes.length == 1) { + throw StateError('simulated lease renewal failure'); + } + } +} + +class _BlockingLeaseBroker extends InMemoryBroker { + _BlockingLeaseBroker() + : super( + delayedInterval: const Duration(milliseconds: 5), + claimInterval: const Duration(milliseconds: 20), + ); + + final started = Completer(); + final release = Completer(); + int attempts = 0; + + @override + Future extendLease(Delivery delivery, Duration by) async { + attempts += 1; + if (!started.isCompleted) started.complete(); + await release.future; + throw StateError('simulated late lease renewal failure'); + } +} + +Future _waitFor( + bool Function() predicate, { + Duration timeout = const Duration(seconds: 2), +}) async { + final deadline = DateTime.now().add(timeout); + while (!predicate()) { + if (DateTime.now().isAfter(deadline)) { + throw TimeoutException('condition was not met'); + } + await Future.delayed(const Duration(milliseconds: 10)); + } +} diff --git a/packages/stem/test/unit/worker/worker_test.dart b/packages/stem/test/unit/worker/worker_test.dart index 683afd03..f9cd19c6 100644 --- a/packages/stem/test/unit/worker/worker_test.dart +++ b/packages/stem/test/unit/worker/worker_test.dart @@ -3,11 +3,298 @@ import 'dart:convert'; import 'dart:isolate'; import 'package:contextual/contextual.dart' show Level, LogDriver, LogEntry; +import 'package:stem/memory.dart'; +import 'package:stem/src/observability/logging.dart' show stemLogger; import 'package:stem/stem.dart'; import 'package:test/test.dart'; void main() { group('Worker', () { + test( + 'recovers an acknowledgement failure without re-executing', + () async { + final broker = _AckFailingBroker(); + final backend = InMemoryResultBackend(); + var executions = 0; + final task = _CountingSuccessTask(() => executions += 1); + final worker = Worker( + broker: broker, + backend: backend, + tasks: [task], + consumerName: 'ack-recovery-worker', + concurrency: 1, + prefetchMultiplier: 1, + ); + + await worker.start(); + final stem = Stem( + broker: broker, + backend: backend, + tasks: [task], + ); + final taskId = await stem.enqueue(task.name); + + await _waitForTaskState(backend, taskId, TaskState.succeeded); + await _waitFor(() => broker.ackAttempts >= 2); + expect(executions, equals(1)); + + await worker.shutdown(); + broker.dispose(); + }, + ); + + test( + 'keeps the lease alive until terminal acknowledgement completes', + () async { + final broker = _BlockingAckBroker(); + final backend = InMemoryResultBackend(); + final task = _CountingSuccessTask(() {}); + final worker = Worker( + broker: broker, + backend: backend, + tasks: [task], + consumerName: 'terminal-ack-lease-worker', + concurrency: 1, + prefetchMultiplier: 1, + ); + + await worker.start(); + try { + final stem = Stem( + broker: broker, + backend: backend, + tasks: [task], + ); + final taskId = await stem.enqueue( + task.name, + options: const TaskOptions( + visibilityTimeout: Duration(milliseconds: 80), + ), + ); + + await broker.ackStarted.future.timeout(const Duration(seconds: 2)); + await _waitFor( + () => broker.leaseExtensions > 0, + ); + expect((await backend.get(taskId))?.state, TaskState.succeeded); + + broker.releaseAcknowledgement(); + await broker.ackCompleted.future.timeout(const Duration(seconds: 2)); + } finally { + broker.releaseAcknowledgement(); + await worker.shutdown(); + broker.dispose(); + } + }, + ); + + test( + 'starts lease renewal before consume middleware completes', + () async { + final broker = _BlockingConsumeBroker(); + final backend = InMemoryResultBackend(); + final middleware = _BlockingConsumeMiddleware(); + final task = _CountingSuccessTask(() {}); + final worker = Worker( + broker: broker, + backend: backend, + tasks: [task], + middleware: [middleware], + consumerName: 'consume-lease-worker', + concurrency: 1, + prefetchMultiplier: 1, + ); + + await worker.start(); + try { + final stem = Stem( + broker: broker, + backend: backend, + tasks: [task], + ); + final taskId = await stem.enqueue( + task.name, + options: const TaskOptions( + visibilityTimeout: Duration(milliseconds: 80), + ), + ); + + await middleware.started.future.timeout(const Duration(seconds: 2)); + await _waitFor(() => broker.leaseExtensions > 0); + middleware.release(); + await _waitForTaskState(backend, taskId, TaskState.succeeded); + } finally { + middleware.release(); + await worker.shutdown(); + broker.dispose(); + } + }, + ); + + test( + 'suppresses a duplicate delivery while the original is active', + () async { + final broker = InMemoryBroker( + delayedInterval: const Duration(milliseconds: 5), + claimInterval: const Duration(milliseconds: 5), + ); + final backend = InMemoryResultBackend(); + final task = _BlockingSuccessTask(); + final worker = Worker( + broker: broker, + backend: backend, + tasks: [task], + consumerName: 'duplicate-suppression-worker', + concurrency: 2, + prefetchMultiplier: 1, + ); + + await worker.start(); + try { + final envelope = Envelope( + id: 'duplicate-delivery-task', + name: task.name, + args: const {}, + ); + await broker.publish(envelope); + await broker.publish(envelope); + + await task.started.future.timeout(const Duration(seconds: 2)); + await Future.delayed(const Duration(milliseconds: 40)); + expect(task.calls, equals(1)); + + task.release(); + await _waitForTaskState( + backend, + envelope.id, + TaskState.succeeded, + ); + } finally { + task.release(); + await worker.shutdown(); + broker.dispose(); + } + }, + ); + + test( + 'does not let a late cross-worker terminal write overwrite the winner', + () async { + final broker = InMemoryBroker( + delayedInterval: const Duration(milliseconds: 5), + claimInterval: const Duration(milliseconds: 5), + ); + final backend = _DelayedTerminalBackend(); + final task = _SequencedSuccessTask(); + final firstWorker = Worker( + broker: broker, + backend: backend, + tasks: [task], + consumerName: 'terminal-race-worker-1', + concurrency: 1, + prefetchMultiplier: 1, + ); + final secondWorker = Worker( + broker: broker, + backend: backend, + tasks: [task], + consumerName: 'terminal-race-worker-2', + concurrency: 1, + prefetchMultiplier: 1, + ); + + await firstWorker.start(); + await secondWorker.start(); + try { + final envelope = Envelope( + id: 'cross-worker-terminal-race', + name: task.name, + args: const {}, + ); + await backend.set(envelope.id, TaskState.queued); + await broker.publish(envelope); + await broker.publish(envelope); + + await backend.firstTerminalEntered.future.timeout( + const Duration(seconds: 2), + ); + await _waitFor(() => backend.terminalCalls >= 2); + backend.releaseFirstTerminal(); + + await _waitForTaskState( + backend, + envelope.id, + TaskState.succeeded, + ); + expect((await backend.get(envelope.id))?.payload, 'result-2'); + expect(task.calls, equals(2)); + } finally { + backend.releaseFirstTerminal(); + await firstWorker.shutdown(); + await secondWorker.shutdown(); + broker.dispose(); + } + }, + ); + + test( + 'redelivers after lease loss and preserves the later terminal result', + () async { + final broker = _LeaseLossBroker(); + final backend = InMemoryResultBackend(); + final task = _LeaseLossTask(); + final firstWorker = Worker( + broker: broker, + backend: backend, + tasks: [task], + consumerName: 'lease-loss-worker-1', + concurrency: 1, + prefetchMultiplier: 1, + ); + final secondWorker = Worker( + broker: broker, + backend: backend, + tasks: [task], + consumerName: 'lease-loss-worker-2', + concurrency: 1, + prefetchMultiplier: 1, + ); + + await firstWorker.start(); + await secondWorker.start(); + try { + final stem = Stem( + broker: broker, + backend: backend, + tasks: [task], + ); + final taskId = await stem.enqueue( + task.name, + options: const TaskOptions( + visibilityTimeout: Duration(milliseconds: 80), + ), + ); + + await task.firstStarted.future.timeout(const Duration(seconds: 2)); + await task.secondStarted.future.timeout(const Duration(seconds: 3)); + await _waitFor( + () async => (await backend.get(taskId))?.payload == 'result-2', + ); + + task.releaseFirst(); + await _waitForTaskState(backend, taskId, TaskState.succeeded); + + expect((await backend.get(taskId))?.payload, 'result-2'); + expect(task.calls, equals(2)); + } finally { + task.releaseFirst(); + await firstWorker.shutdown(); + await secondWorker.shutdown(); + broker.dispose(); + } + }, + ); + test('executes task and records success', () async { final broker = InMemoryBroker( delayedInterval: const Duration(milliseconds: 10), @@ -1240,6 +1527,105 @@ void main() { broker.dispose(); }); + test('bounds a retry storm at each task retry budget', () async { + final broker = InMemoryBroker( + delayedInterval: const Duration(milliseconds: 2), + claimInterval: const Duration(milliseconds: 5), + ); + final backend = InMemoryResultBackend(); + final task = _RetryStormTask(); + final registry = InMemoryTaskRegistry()..register(task); + final worker = Worker( + broker: broker, + registry: registry, + backend: backend, + consumerName: 'retry-storm-worker', + concurrency: 4, + prefetchMultiplier: 1, + retryStrategy: ExponentialJitterRetryStrategy( + base: const Duration(milliseconds: 1), + max: const Duration(milliseconds: 2), + seed: 1, + ), + ); + const totalTasks = 12; + final taskIds = []; + + await worker.start(); + try { + final stem = Stem(broker: broker, registry: registry, backend: backend); + for (var index = 0; index < totalTasks; index++) { + taskIds.add( + await stem.enqueue( + task.name, + args: {'job': index}, + ), + ); + } + + await _waitFor( + () async { + final dead = await broker.listDeadLetters('default'); + return dead.entries.length == totalTasks; + }, + timeout: const Duration(seconds: 5), + ); + + for (final taskId in taskIds) { + expect((await backend.get(taskId))?.state, TaskState.failed); + } + expect(task.calls, equals(totalTasks * 4)); + expect(await broker.pendingCount('default'), equals(0)); + expect(await broker.inflightCount('default'), equals(0)); + } finally { + await worker.shutdown(); + broker.dispose(); + } + }); + + test('dead-letters malformed payloads without retrying them', () async { + final broker = InMemoryBroker( + delayedInterval: const Duration(milliseconds: 5), + claimInterval: const Duration(milliseconds: 5), + ); + final backend = InMemoryResultBackend(); + final task = _PoisonPayloadTask(); + final registry = InMemoryTaskRegistry()..register(task); + final worker = Worker( + broker: broker, + registry: registry, + backend: backend, + consumerName: 'poison-payload-worker', + concurrency: 1, + prefetchMultiplier: 1, + ); + + await worker.start(); + try { + final stem = Stem(broker: broker, registry: registry, backend: backend); + final taskId = await stem.enqueue( + task.name, + args: const {'value': 'poison'}, + ); + + await _waitForTaskState(backend, taskId, TaskState.failed); + final dead = await broker.listDeadLetters('default'); + expect(dead.entries, hasLength(1)); + expect(dead.entries.single.reason, equals('invalid-payload')); + expect(task.calls, equals(0)); + + await Future.delayed(const Duration(milliseconds: 80)); + expect(task.calls, equals(0)); + expect( + (await broker.listDeadLetters('default')).entries, + hasLength(1), + ); + } finally { + await worker.shutdown(); + broker.dispose(); + } + }); + test('executes handler inside isolate when entrypoint provided', () async { final broker = InMemoryBroker( delayedInterval: const Duration(milliseconds: 10), @@ -1358,57 +1744,319 @@ void main() { broker.dispose(); }); - test('skips revoked tasks from persistent store', () async { - StemSignals.configure(configuration: const StemSignalConfiguration()); - - final broker = InMemoryBroker( - delayedInterval: const Duration(milliseconds: 10), - claimInterval: const Duration(milliseconds: 40), - ); - final backend = InMemoryResultBackend(); - final registry = InMemoryTaskRegistry()..register(_SuccessTask()); - final revokeStore = InMemoryRevokeStore(); + test( + 'inline hard time limits stop awaiting but do not cancel handler work', + () async { + final broker = InMemoryBroker( + delayedInterval: const Duration(milliseconds: 10), + claimInterval: const Duration(milliseconds: 40), + ); + final backend = InMemoryResultBackend(); + final handlerFinished = Completer(); + var underlyingCompletions = 0; + final registry = InMemoryTaskRegistry() + ..register( + FunctionTaskHandler.inline( + name: 'tasks.inline-hard-limit', + options: const TaskOptions( + hardTimeLimit: Duration(milliseconds: 20), + ), + entrypoint: (context, args) async { + await Future.delayed(const Duration(milliseconds: 100)); + underlyingCompletions++; + handlerFinished.complete(); + return null; + }, + ), + ); + final worker = Worker( + broker: broker, + registry: registry, + backend: backend, + consumerName: 'worker-inline-timeout', + concurrency: 1, + prefetchMultiplier: 1, + ); - final stem = Stem(broker: broker, registry: registry, backend: backend); + await worker.start(); + final stem = Stem(broker: broker, registry: registry, backend: backend); + final taskId = await stem.enqueue('tasks.inline-hard-limit'); - final taskId = await stem.enqueue('tasks.success'); - await revokeStore.upsertAll([ - RevokeEntry( - namespace: 'stem', - taskId: taskId, - version: generateRevokeVersion(), - issuedAt: DateTime.now().toUtc(), - terminate: true, - ), - ]); + await _waitForTaskState(backend, taskId, TaskState.failed); + expect(underlyingCompletions, 0); - final worker = Worker( - broker: broker, - registry: registry, - backend: backend, - consumerName: 'worker-revoked', - concurrency: 1, - prefetchMultiplier: 1, - revokeStore: revokeStore, - ); + await handlerFinished.future.timeout(const Duration(seconds: 1)); + expect(underlyingCompletions, 1); - final events = []; - final sub = worker.events.listen(events.add); + await worker.shutdown(); + broker.dispose(); + }, + ); - final revokedSignal = Completer(); - final postrunStates = []; - final subscriptions = [ - StemSignals.taskRevoked.connect((payload, _) { - if (payload.envelope.id == taskId && !revokedSignal.isCompleted) { - revokedSignal.complete(payload); - } - }), - StemSignals.taskPostrun.connect((payload, _) { - if (payload.envelope.id == taskId) { - postrunStates.add(payload.state.name); - } - }), - ]; + test( + 'hard shutdown requeues an active isolate delivery ' + 'for a replacement worker', + () async { + final broker = InMemoryBroker( + delayedInterval: const Duration(milliseconds: 5), + claimInterval: const Duration(milliseconds: 20), + ); + final backend = InMemoryResultBackend(); + final registry = InMemoryTaskRegistry() + ..register( + FunctionTaskHandler( + name: 'tasks.shutdown-requeue', + entrypoint: _shutdownRequeueEntrypoint, + options: const TaskOptions(maxRetries: 1), + ), + ); + final workerA = Worker( + broker: broker, + registry: registry, + backend: backend, + consumerName: 'worker-shutdown-a', + concurrency: 1, + prefetchMultiplier: 1, + lifecycle: const WorkerLifecycleConfig( + installSignalHandlers: false, + ), + ); + final workerB = Worker( + broker: broker, + registry: registry, + backend: backend, + consumerName: 'worker-shutdown-b', + concurrency: 1, + prefetchMultiplier: 1, + lifecycle: const WorkerLifecycleConfig( + installSignalHandlers: false, + ), + ); + + try { + await workerA.start(); + final stem = Stem( + broker: broker, + registry: registry, + backend: backend, + ); + final taskId = await stem.enqueue('tasks.shutdown-requeue'); + + await _waitForTaskState(backend, taskId, TaskState.running); + await workerA.shutdown(); + + await workerB.start(); + await _waitForTaskState(backend, taskId, TaskState.succeeded); + expect((await backend.get(taskId))?.payload, isA()); + } finally { + await workerA.shutdown(); + await workerB.shutdown(); + broker.dispose(); + } + }, + ); + + test( + 'hard shutdown requeues the full prefetched batch for a replacement', + () async { + final broker = InMemoryBroker( + delayedInterval: const Duration(milliseconds: 5), + claimInterval: const Duration(milliseconds: 5), + ); + final backend = InMemoryResultBackend(); + final registry = InMemoryTaskRegistry() + ..register( + FunctionTaskHandler( + name: 'tasks.prefetch-shutdown', + entrypoint: _prefetchShutdownEntrypoint, + options: const TaskOptions(maxRetries: 1), + ), + ); + final workerA = Worker( + broker: broker, + registry: registry, + backend: backend, + consumerName: 'prefetch-shutdown-a', + concurrency: 2, + prefetch: 6, + lifecycle: const WorkerLifecycleConfig( + installSignalHandlers: false, + ), + ); + final workerB = Worker( + broker: broker, + registry: registry, + backend: backend, + consumerName: 'prefetch-shutdown-b', + concurrency: 6, + prefetch: 6, + lifecycle: const WorkerLifecycleConfig( + installSignalHandlers: false, + ), + ); + const totalTasks = 12; + final taskIds = []; + + try { + await workerA.start(); + final stem = Stem( + broker: broker, + registry: registry, + backend: backend, + ); + for (var index = 0; index < totalTasks; index++) { + taskIds.add( + await stem.enqueue( + 'tasks.prefetch-shutdown', + args: {'index': index}, + ), + ); + } + + await _waitFor( + () async => (await broker.inflightCount('default')) == 6, + timeout: const Duration(seconds: 3), + ); + await workerA.shutdown(); + + expect(await broker.inflightCount('default'), equals(0)); + expect(await broker.pendingCount('default'), equals(totalTasks)); + + await workerB.start(); + await _waitFor( + () async { + for (final taskId in taskIds) { + if ((await backend.get(taskId))?.state != TaskState.succeeded) { + return false; + } + } + return true; + }, + timeout: const Duration(seconds: 8), + ); + } finally { + await workerA.shutdown(); + await workerB.shutdown(); + broker.dispose(); + } + }, + ); + + test( + 'late inline completion cannot write to the closed event stream', + () async { + final broker = InMemoryBroker( + delayedInterval: const Duration(milliseconds: 5), + claimInterval: const Duration(milliseconds: 20), + ); + final backend = InMemoryResultBackend(); + final entered = Completer(); + final release = Completer(); + final registry = InMemoryTaskRegistry() + ..register( + FunctionTaskHandler.inline( + name: 'tasks.late-inline-completion', + options: const TaskOptions( + softTimeLimit: Duration(milliseconds: 10), + ), + entrypoint: (context, args) async { + if (!entered.isCompleted) entered.complete(); + await release.future; + return null; + }, + ), + ); + final worker = Worker( + broker: broker, + registry: registry, + backend: backend, + consumerName: 'worker-late-inline-completion', + concurrency: 1, + prefetchMultiplier: 1, + heartbeatInterval: const Duration(milliseconds: 5), + lifecycle: const WorkerLifecycleConfig( + installSignalHandlers: false, + ), + ); + final events = []; + final subscription = worker.events.listen(events.add); + + try { + await worker.start(); + final stem = Stem( + broker: broker, + registry: registry, + backend: backend, + ); + await stem.enqueue('tasks.late-inline-completion'); + await entered.future.timeout(const Duration(seconds: 1)); + await Future.delayed(const Duration(milliseconds: 25)); + + await worker.shutdown(); + release.complete(); + await Future.delayed(const Duration(milliseconds: 100)); + + expect(events, isNotEmpty); + } finally { + if (!release.isCompleted) release.complete(); + await subscription.cancel(); + await worker.shutdown(); + broker.dispose(); + } + }, + ); + + test('skips revoked tasks from persistent store', () async { + StemSignals.configure(configuration: const StemSignalConfiguration()); + + final broker = InMemoryBroker( + delayedInterval: const Duration(milliseconds: 10), + claimInterval: const Duration(milliseconds: 40), + ); + final backend = InMemoryResultBackend(); + final registry = InMemoryTaskRegistry()..register(_SuccessTask()); + final revokeStore = InMemoryRevokeStore(); + + final stem = Stem(broker: broker, registry: registry, backend: backend); + + final taskId = await stem.enqueue('tasks.success'); + await revokeStore.upsertAll([ + RevokeEntry( + namespace: 'stem', + taskId: taskId, + version: generateRevokeVersion(), + issuedAt: DateTime.now().toUtc(), + terminate: true, + ), + ]); + + final worker = Worker( + broker: broker, + registry: registry, + backend: backend, + consumerName: 'worker-revoked', + concurrency: 1, + prefetchMultiplier: 1, + revokeStore: revokeStore, + ); + + final events = []; + final sub = worker.events.listen(events.add); + + final revokedSignal = Completer(); + final postrunStates = []; + final subscriptions = [ + StemSignals.taskRevoked.connect((payload, _) { + if (payload.envelope.id == taskId && !revokedSignal.isCompleted) { + revokedSignal.complete(payload); + } + }), + StemSignals.taskPostrun.connect((payload, _) { + if (payload.envelope.id == taskId) { + postrunStates.add(payload.state.name); + } + }), + ]; await worker.start(); @@ -1452,7 +2100,7 @@ void main() { FunctionTaskHandler( name: 'tasks.group.a', options: const TaskOptions( - groupRateLimit: '1/s', + groupRateLimit: RateLimit.perSecond(1), ), entrypoint: (context, args) async => null, ), @@ -1461,7 +2109,7 @@ void main() { FunctionTaskHandler( name: 'tasks.group.b', options: const TaskOptions( - groupRateLimit: '1/s', + groupRateLimit: RateLimit.perSecond(1), ), entrypoint: (context, args) async => null, ), @@ -1534,7 +2182,7 @@ void main() { FunctionTaskHandler( name: 'tasks.group.failopen', options: const TaskOptions( - groupRateLimit: '10/m', + groupRateLimit: RateLimit.perMinute(10), ), entrypoint: (context, args) async => null, ), @@ -1581,7 +2229,7 @@ void main() { FunctionTaskHandler( name: 'tasks.group.failclosed', options: const TaskOptions( - groupRateLimit: '10/m', + groupRateLimit: RateLimit.perMinute(10), groupRateLimiterFailureMode: RateLimiterFailureMode.failClosed, maxRetries: 5, ), @@ -1632,6 +2280,9 @@ void main() { broker: broker, registry: registry, backend: backend, + subscription: RoutingSubscription( + queues: const ['default', 'priority'], + ), consumerName: 'pause-worker-a', concurrency: 1, prefetchMultiplier: 1, @@ -1648,7 +2299,7 @@ void main() { ), type: 'queue_pause', payload: const { - 'queues': ['default'], + 'queues': ['default', 'priority'], }, ); expect(pauseReply.status, equals('ok')); @@ -1658,6 +2309,9 @@ void main() { broker: broker, registry: registry, backend: backend, + subscription: RoutingSubscription( + queues: const ['default', 'priority'], + ), consumerName: 'pause-worker-b', concurrency: 1, prefetchMultiplier: 1, @@ -1680,7 +2334,7 @@ void main() { ), type: 'queue_resume', payload: const { - 'queues': ['default'], + 'queues': ['default', 'priority'], }, ); expect(resumeReply.status, equals('ok')); @@ -1830,6 +2484,326 @@ class _ChordCallbackTask implements TaskHandler { } } +class _AckFailingBroker extends InMemoryBroker { + _AckFailingBroker() + : super( + delayedInterval: const Duration(milliseconds: 5), + claimInterval: const Duration(milliseconds: 5), + defaultVisibilityTimeout: const Duration(milliseconds: 30), + ); + + int ackAttempts = 0; + bool failNextAck = true; + + @override + Future ack(Delivery delivery) async { + ackAttempts += 1; + if (failNextAck) { + failNextAck = false; + throw StateError('simulated acknowledgement disconnect'); + } + await super.ack(delivery); + } +} + +class _LeaseLossBroker extends InMemoryBroker { + _LeaseLossBroker() + : super( + delayedInterval: const Duration(milliseconds: 5), + claimInterval: const Duration(milliseconds: 5), + defaultVisibilityTimeout: const Duration(milliseconds: 80), + ); + + @override + Stream consume( + RoutingSubscription subscription, { + int prefetch = 1, + String? consumerGroup, + String? consumerName, + }) { + if (consumerName != 'lease-loss-worker-1') { + return super.consume( + subscription, + prefetch: prefetch, + consumerGroup: consumerGroup, + consumerName: consumerName, + ); + } + + // Model a worker process that stops consuming after receiving its active + // delivery. The delivery remains leased until the broker's visibility + // timeout, allowing the replacement worker to receive the redelivery. + late StreamController controller; + controller = StreamController( + onListen: () async { + try { + final delivery = await super + .consume( + subscription, + prefetch: prefetch, + consumerGroup: consumerGroup, + consumerName: consumerName, + ) + .first; + if (!controller.isClosed) { + controller.add(delivery); + } + } on Object catch (error, stackTrace) { + if (!controller.isClosed) { + controller.addError(error, stackTrace); + await controller.close(); + } + } + }, + ); + return controller.stream; + } + + @override + Future extendLease(Delivery delivery, Duration by) async { + throw StateError('simulated lease renewal disconnect'); + } +} + +class _BlockingAckBroker extends InMemoryBroker { + _BlockingAckBroker() + : super( + delayedInterval: const Duration(milliseconds: 5), + claimInterval: const Duration(milliseconds: 5), + defaultVisibilityTimeout: const Duration(milliseconds: 80), + ); + + final Completer ackStarted = Completer(); + final Completer ackCompleted = Completer(); + final Completer _ackRelease = Completer(); + int leaseExtensions = 0; + + @override + Future ack(Delivery delivery) async { + if (!ackStarted.isCompleted) { + ackStarted.complete(); + } + await _ackRelease.future; + await super.ack(delivery); + if (!ackCompleted.isCompleted) { + ackCompleted.complete(); + } + } + + @override + Future extendLease(Delivery delivery, Duration by) async { + leaseExtensions += 1; + await super.extendLease(delivery, by); + } + + void releaseAcknowledgement() { + if (!_ackRelease.isCompleted) { + _ackRelease.complete(); + } + } +} + +class _BlockingConsumeBroker extends InMemoryBroker { + _BlockingConsumeBroker() + : super( + delayedInterval: const Duration(milliseconds: 5), + claimInterval: const Duration(milliseconds: 5), + defaultVisibilityTimeout: const Duration(milliseconds: 80), + ); + + int leaseExtensions = 0; + + @override + Future extendLease(Delivery delivery, Duration by) async { + leaseExtensions += 1; + await super.extendLease(delivery, by); + } +} + +class _BlockingConsumeMiddleware implements Middleware { + final Completer started = Completer(); + final Completer _release = Completer(); + + @override + Future onConsume( + Delivery delivery, + Future Function() next, + ) async { + if (!started.isCompleted) { + started.complete(); + } + await _release.future; + await next(); + } + + @override + Future onEnqueue( + Envelope envelope, + Future Function() next, + ) => next(); + + @override + Future onExecute( + TaskContext context, + Future Function() next, + ) => next(); + + @override + Future onError( + TaskContext context, + Object error, + StackTrace stackTrace, + ) async {} + + void release() { + if (!_release.isCompleted) { + _release.complete(); + } + } +} + +class _DelayedTerminalBackend extends InMemoryResultBackend { + final Completer firstTerminalEntered = Completer(); + final Completer _firstTerminalRelease = Completer(); + int terminalCalls = 0; + + @override + Future setTerminalIfAbsent( + TaskStatus status, { + Duration? ttl, + }) async { + terminalCalls += 1; + if (terminalCalls == 1) { + firstTerminalEntered.complete(); + await _firstTerminalRelease.future; + } + return super.setTerminalIfAbsent(status, ttl: ttl); + } + + void releaseFirstTerminal() { + if (!_firstTerminalRelease.isCompleted) { + _firstTerminalRelease.complete(); + } + } +} + +class _SequencedSuccessTask implements TaskHandler { + int calls = 0; + + @override + String get name => 'tasks.sequenced-success'; + + @override + TaskOptions get options => const TaskOptions(); + + @override + TaskMetadata get metadata => const TaskMetadata(); + + @override + TaskEntrypoint? get isolateEntrypoint => null; + + @override + Future call(TaskContext context, Map args) async { + calls += 1; + return 'result-$calls'; + } +} + +class _LeaseLossTask implements TaskHandler { + final Completer firstStarted = Completer(); + final Completer secondStarted = Completer(); + final Completer _firstRelease = Completer(); + int calls = 0; + + @override + String get name => 'tasks.lease-loss'; + + @override + TaskOptions get options => const TaskOptions(); + + @override + TaskMetadata get metadata => const TaskMetadata(); + + @override + TaskEntrypoint? get isolateEntrypoint => null; + + @override + Future call(TaskContext context, Map args) async { + calls += 1; + if (calls == 1) { + firstStarted.complete(); + await _firstRelease.future; + return 'result-1'; + } + secondStarted.complete(); + return 'result-2'; + } + + void releaseFirst() { + if (!_firstRelease.isCompleted) { + _firstRelease.complete(); + } + } +} + +class _CountingSuccessTask implements TaskHandler { + _CountingSuccessTask(this.onCall); + + final void Function() onCall; + + @override + String get name => 'tasks.counting-success'; + + @override + TaskOptions get options => const TaskOptions(); + + @override + TaskMetadata get metadata => const TaskMetadata(); + + @override + TaskEntrypoint? get isolateEntrypoint => null; + + @override + Future call(TaskContext context, Map args) async { + onCall(); + return 'ok'; + } +} + +class _BlockingSuccessTask implements TaskHandler { + final Completer started = Completer(); + final Completer _release = Completer(); + int calls = 0; + + @override + String get name => 'tasks.blocking-success'; + + @override + TaskOptions get options => const TaskOptions(); + + @override + TaskMetadata get metadata => const TaskMetadata(); + + @override + TaskEntrypoint? get isolateEntrypoint => null; + + @override + Future call(TaskContext context, Map args) async { + calls += 1; + if (!started.isCompleted) { + started.complete(); + } + await _release.future; + return 'ok'; + } + + void release() { + if (!_release.isCompleted) { + _release.complete(); + } + } +} + Future _waitFor( FutureOr Function() predicate, { Duration timeout = const Duration(seconds: 2), @@ -2027,6 +3001,69 @@ class _AlwaysFailTask implements TaskHandler { } } +class _RetryStormTask implements TaskHandler { + final Map _callsByJob = {}; + + int get calls => _callsByJob.values.fold(0, (sum, count) => sum + count); + + @override + String get name => 'tasks.retry-storm'; + + @override + TaskOptions get options => const TaskOptions(maxRetries: 3); + + @override + TaskMetadata get metadata => const TaskMetadata(); + + @override + TaskEntrypoint? get isolateEntrypoint => null; + + @override + Future call(TaskContext context, Map args) async { + final job = args['job']! as int; + _callsByJob[job] = (_callsByJob[job] ?? 0) + 1; + throw StateError('retry storm failure for job $job'); + } +} + +class _PoisonPayloadTask implements TaskHandler { + int calls = 0; + + @override + String get name => 'tasks.poison-payload'; + + @override + TaskOptions get options => const TaskOptions(maxRetries: 10); + + @override + TaskMetadata get metadata => const TaskMetadata( + argsEncoder: _ThrowingArgsEncoder(), + ); + + @override + TaskEntrypoint? get isolateEntrypoint => null; + + @override + Future call(TaskContext context, Map args) async { + calls += 1; + } +} + +class _ThrowingArgsEncoder extends TaskPayloadEncoder { + const _ThrowingArgsEncoder(); + + @override + String get id => 'poison-test'; + + @override + Object? encode(Object? value) => value; + + @override + Object? decode(Object? stored) { + throw const FormatException('payload is intentionally malformed'); + } +} + FutureOr _isolateEntrypoint( TaskInvocationContext context, Map args, @@ -2062,6 +3099,22 @@ FutureOr _sleepyEntrypoint( return null; } +FutureOr _shutdownRequeueEntrypoint( + TaskInvocationContext context, + Map args, +) async { + await Future.delayed(const Duration(seconds: 2)); + return Isolate.current.hashCode; +} + +FutureOr _prefetchShutdownEntrypoint( + TaskInvocationContext context, + Map args, +) async { + await Future.delayed(const Duration(milliseconds: 150)); + return 'completed:${args['index']}'; +} + FutureOr _isolateHashEntrypoint( TaskInvocationContext context, Map args, diff --git a/packages/stem/test/unit/workflow/in_memory_event_bus_test.dart b/packages/stem/test/unit/workflow/in_memory_event_bus_test.dart index d3a4b8a4..fe555295 100644 --- a/packages/stem/test/unit/workflow/in_memory_event_bus_test.dart +++ b/packages/stem/test/unit/workflow/in_memory_event_bus_test.dart @@ -1,3 +1,4 @@ +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; import 'package:test/test.dart'; diff --git a/packages/stem/test/workflow/workflow_runtime_call_extensions_test.dart b/packages/stem/test/workflow/workflow_runtime_call_extensions_test.dart index c818a9bc..a8c677d6 100644 --- a/packages/stem/test/workflow/workflow_runtime_call_extensions_test.dart +++ b/packages/stem/test/workflow/workflow_runtime_call_extensions_test.dart @@ -20,7 +20,9 @@ void main() { encodeParams: (params) => params, ); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); try { await workflowApp.start(); @@ -71,7 +73,9 @@ void main() { encodeParams: (params) => params, ); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); try { await workflowApp.start(); @@ -117,7 +121,9 @@ void main() { encodeParams: (params) => params, ); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); try { await workflowApp.start(); diff --git a/packages/stem/test/workflow/workflow_runtime_ref_test.dart b/packages/stem/test/workflow/workflow_runtime_ref_test.dart index e6ecbe7f..f49f1f2a 100644 --- a/packages/stem/test/workflow/workflow_runtime_ref_test.dart +++ b/packages/stem/test/workflow/workflow_runtime_ref_test.dart @@ -104,7 +104,9 @@ void main() { encodeParams: (params) => params, ); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); try { await workflowApp.start(); @@ -148,7 +150,9 @@ void main() { encodeParams: (params) => params, ); - final workflowApp = await StemWorkflowApp.inMemory(scripts: [script]); + final workflowApp = await StemWorkflowApp.inMemory( + scripts: [script], + ); try { await workflowApp.start(); @@ -182,7 +186,9 @@ void main() { paramsCodec: _greetingParamsCodec, ); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); try { await workflowApp.start(); @@ -210,7 +216,9 @@ void main() { ); final workflowRef = flow.refJson<_GreetingParams>(); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); try { await workflowApp.start(); @@ -239,7 +247,9 @@ void main() { ); final workflowRef = flow.refVersionedJson<_GreetingParams>(version: 2); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); try { await workflowApp.start(); @@ -272,7 +282,9 @@ void main() { encodeParams: (params) => {'display_name': params.name}, ); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); try { await workflowApp.start(); @@ -304,7 +316,9 @@ void main() { decodeResultJson: _GreetingResult.fromJson, ); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); try { await workflowApp.start(); @@ -345,7 +359,9 @@ void main() { defaultDecodeVersion: 2, ); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); try { await workflowApp.start(); @@ -380,7 +396,9 @@ void main() { paramsCodec: _greetingParamsCodec, ); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); try { await workflowApp.start(); @@ -637,7 +655,9 @@ void main() { decodeResultVersionedJson: _GreetingResult.fromVersionedJson, ); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); try { await workflowApp.start(); @@ -677,7 +697,9 @@ void main() { resultRegistry: _greetingResultRegistry, ); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); try { await workflowApp.start(); @@ -890,7 +912,9 @@ void main() { }, ); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); try { await workflowApp.start(); @@ -927,7 +951,9 @@ void main() { }, ); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); try { await workflowApp.start(); @@ -966,7 +992,9 @@ void main() { }, ); - final workflowApp = await StemWorkflowApp.inMemory(flows: [flow]); + final workflowApp = await StemWorkflowApp.inMemory( + flows: [flow], + ); try { await workflowApp.start(); diff --git a/packages/stem/test/workflow/workflow_runtime_test.dart b/packages/stem/test/workflow/workflow_runtime_test.dart index 562a53cc..5b5577ba 100644 --- a/packages/stem/test/workflow/workflow_runtime_test.dart +++ b/packages/stem/test/workflow/workflow_runtime_test.dart @@ -1,4 +1,6 @@ import 'package:contextual/contextual.dart' show Level, LogDriver, LogEntry; +import 'package:stem/memory.dart'; +import 'package:stem/src/observability/logging.dart' show stemLogger; import 'package:stem/stem.dart'; import 'package:test/test.dart'; diff --git a/packages/stem/test/workflow/workflow_store_contract_test.dart b/packages/stem/test/workflow/workflow_store_contract_test.dart index a68a2cb2..2862d1fa 100644 --- a/packages/stem/test/workflow/workflow_store_contract_test.dart +++ b/packages/stem/test/workflow/workflow_store_contract_test.dart @@ -1,4 +1,4 @@ -import 'package:stem/stem.dart'; +import 'package:stem/memory.dart'; import 'package:stem_adapter_tests/stem_adapter_tests.dart'; void main() { diff --git a/packages/stem/tool/proxy_runtime_check.dart b/packages/stem/tool/proxy_runtime_check.dart index 9c0ef764..c7ff7c36 100644 --- a/packages/stem/tool/proxy_runtime_check.dart +++ b/packages/stem/tool/proxy_runtime_check.dart @@ -1,5 +1,6 @@ import 'dart:io'; +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; class ScriptDef { diff --git a/packages/stem_adapter_tests/CHANGELOG.md b/packages/stem_adapter_tests/CHANGELOG.md index 6e9dfcfe..a1c5e9bb 100644 --- a/packages/stem_adapter_tests/CHANGELOG.md +++ b/packages/stem_adapter_tests/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## 0.2.0 + +- Updated shared adapter contracts for the Stem 0.3.0 capability and lifecycle + semantics. + +## 0.1.3 + +- Expanded contract fixtures and capability assertions used by the Redis, + Postgres, SQLite, and in-memory adapters. + ## 0.1.2 - Updated the `stem` dependency range for the 0.2.1 core release. diff --git a/packages/stem_adapter_tests/lib/src/workflow_script_facade_suite.dart b/packages/stem_adapter_tests/lib/src/workflow_script_facade_suite.dart index 35ce0656..d028081e 100644 --- a/packages/stem_adapter_tests/lib/src/workflow_script_facade_suite.dart +++ b/packages/stem_adapter_tests/lib/src/workflow_script_facade_suite.dart @@ -1,5 +1,6 @@ import 'dart:async'; +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; import 'package:stem_adapter_tests/src/workflow_store_contract_suite.dart'; import 'package:test/test.dart'; diff --git a/packages/stem_adapter_tests/lib/src/workflow_store_contract_suite.dart b/packages/stem_adapter_tests/lib/src/workflow_store_contract_suite.dart index 142c87e8..d230d0de 100644 --- a/packages/stem_adapter_tests/lib/src/workflow_store_contract_suite.dart +++ b/packages/stem_adapter_tests/lib/src/workflow_store_contract_suite.dart @@ -128,10 +128,7 @@ void runWorkflowStoreContractTests({ orchestrationQueue: 'workflow', continuationQueue: 'workflow', executionQueue: 'workflow-step', - serializationFormat: 'json', - serializationVersion: '1', frameFormat: 'stem-envelope', - frameVersion: '1', encryptionScope: 'signed-envelope', encryptionEnabled: true, streamId: 'contract_stream_01', diff --git a/packages/stem_adapter_tests/pubspec.yaml b/packages/stem_adapter_tests/pubspec.yaml index 49ef57de..526ff407 100644 --- a/packages/stem_adapter_tests/pubspec.yaml +++ b/packages/stem_adapter_tests/pubspec.yaml @@ -1,13 +1,13 @@ name: stem_adapter_tests description: Shared contract test suites for Stem broker and result backend adapters. -version: 0.1.2 +version: 0.2.0 repository: https://github.com/kingwill101/stem resolution: workspace environment: sdk: ">=3.9.2 <4.0.0" dependencies: - stem: ">=0.2.1 <0.3.0" + stem: ">=0.3.0 <0.4.0" test: ^1.29.0 dev_dependencies: diff --git a/packages/stem_adapter_tests/test/queue_events_contract_suite_test.dart b/packages/stem_adapter_tests/test/queue_events_contract_suite_test.dart index 42c3b829..56c03619 100644 --- a/packages/stem_adapter_tests/test/queue_events_contract_suite_test.dart +++ b/packages/stem_adapter_tests/test/queue_events_contract_suite_test.dart @@ -1,4 +1,4 @@ -import 'package:stem/stem.dart'; +import 'package:stem/memory.dart'; import 'package:stem_adapter_tests/stem_adapter_tests.dart'; void main() { diff --git a/packages/stem_adapter_tests/test/revoke_store_contract_suite_test.dart b/packages/stem_adapter_tests/test/revoke_store_contract_suite_test.dart index 6e082c5e..4ba39121 100644 --- a/packages/stem_adapter_tests/test/revoke_store_contract_suite_test.dart +++ b/packages/stem_adapter_tests/test/revoke_store_contract_suite_test.dart @@ -1,4 +1,4 @@ -import 'package:stem/stem.dart'; +import 'package:stem/memory.dart'; import 'package:stem_adapter_tests/stem_adapter_tests.dart'; void main() { diff --git a/packages/stem_builder/CHANGELOG.md b/packages/stem_builder/CHANGELOG.md index 001006de..3bfaad13 100644 --- a/packages/stem_builder/CHANGELOG.md +++ b/packages/stem_builder/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +## 0.3.0 + +- Updated generated task adapters and workflow definitions for Stem 0.3.0. +- Generated tasks now preserve typed handler boundaries and isolate entrypoints + without exposing raw map transport code to application code. +- Generated registrations now declare `TaskExecutionMode` explicitly, making + inline versus isolate execution visible in generated source. + +## 0.2.1 + +- Updated the `stem` dependency range to admit the in-progress `0.2.1-wip` + core prerelease during workspace resolution. +- Generated task registrations now use `TypedTaskHandler` adapters backed by + the generated task definition, including generated isolate entrypoints. + ## 0.2.0 - Generated output now centers on `stemModule`, `StemWorkflowDefinitions`, and diff --git a/packages/stem_builder/README.md b/packages/stem_builder/README.md index 6f99afa2..acaa108f 100644 --- a/packages/stem_builder/README.md +++ b/packages/stem_builder/README.md @@ -246,8 +246,8 @@ final taskApp = await StemApp.fromUrl( Plain `StemApp` bootstrap also infers task queue subscriptions from the bundled or explicitly supplied task handlers when -`workerConfig.subscription` is omitted, and it lazy-starts on the first -enqueue or wait call. +`workerConfig.subscription` is omitted. Start the app explicitly with +`await taskApp.start()` before enqueueing or waiting for work. If you already centralize wiring in a `StemClient`, prefer the shared-client path: diff --git a/packages/stem_builder/example/bin/main.dart b/packages/stem_builder/example/bin/main.dart index f523c634..bcbc3fac 100644 --- a/packages/stem_builder/example/bin/main.dart +++ b/packages/stem_builder/example/bin/main.dart @@ -1,3 +1,6 @@ +// This executable prints generated manifests so users can inspect the output. +// ignore_for_file: avoid_print + import 'dart:convert'; import 'package:stem/stem.dart'; @@ -16,8 +19,11 @@ Future main() async { ), ); - final app = await StemWorkflowApp.inMemory(module: stemModule); + final app = await StemWorkflowApp.inMemory( + module: stemModule, + ); try { + await app.start(); final runtimeManifest = app .workflowManifest() .map((entry) => entry.toJson()) @@ -40,8 +46,11 @@ Future main() async { await app.close(); } - final taskApp = await StemApp.inMemory(module: stemModule); + final taskApp = await StemApp.inMemory( + module: stemModule, + ); try { + await taskApp.start(); final taskResult = await StemTaskDefinitions.builderExamplePing .enqueueAndWait(taskApp, timeout: const Duration(seconds: 2)); print('\nNo-arg task result: ${taskResult?.value}'); diff --git a/packages/stem_builder/example/bin/runtime_metadata_views.dart b/packages/stem_builder/example/bin/runtime_metadata_views.dart index f8a71cde..550662bc 100644 --- a/packages/stem_builder/example/bin/runtime_metadata_views.dart +++ b/packages/stem_builder/example/bin/runtime_metadata_views.dart @@ -1,13 +1,19 @@ +// This executable prints generated runtime metadata so users can inspect it. +// ignore_for_file: avoid_print + import 'dart:convert'; import 'package:stem/stem.dart'; import 'package:stem_builder_example/definitions.dart'; Future main() async { - final app = await StemWorkflowApp.inMemory(module: stemModule); + final app = await StemWorkflowApp.inMemory( + module: stemModule, + ); final runtime = app.runtime; try { + await app.start(); print('--- Generated manifest (builder output) ---'); print( const JsonEncoder.withIndent(' ').convert( diff --git a/packages/stem_builder/example/lib/definitions.dart b/packages/stem_builder/example/lib/definitions.dart index ac0f464e..bd6d92f1 100644 --- a/packages/stem_builder/example/lib/definitions.dart +++ b/packages/stem_builder/example/lib/definitions.dart @@ -1,3 +1,6 @@ +// These sample definitions keep the focus on generated APIs, not doc noise. +// ignore_for_file: public_member_api_docs + import 'package:stem/stem.dart'; part 'definitions.stem.g.dart'; diff --git a/packages/stem_builder/example/lib/definitions.stem.g.dart b/packages/stem_builder/example/lib/definitions.stem.g.dart index ede891a6..9f7d9707 100644 --- a/packages/stem_builder/example/lib/definitions.stem.g.dart +++ b/packages/stem_builder/example/lib/definitions.stem.g.dart @@ -91,6 +91,29 @@ Object? _stemRequireArg(Map args, String name) { return args[name]; } +TaskInvocationContext _stemTaskInvocationContext( + TaskExecutionContext context, + Map args, +) { + if (context case final TaskInvocationContext value) { + return value; + } + return TaskInvocationContext.local( + id: context.id, + args: args, + headers: context.headers, + meta: context.meta, + attempt: context.attempt, + heartbeat: context.heartbeat, + extendLease: context.extendLease, + progress: (percent, {data}) => context.progress(percent, data: data), + cancellation: context.cancellation, + enqueuer: context, + workflows: context, + workflowEvents: context, + ); +} + Future _stemTaskAdapter0( TaskInvocationContext context, Map args, @@ -103,6 +126,7 @@ abstract final class StemTaskDefinitions { builderExampleTask = TaskDefinition, Object?>( name: "builder.example.task", encodeArgs: (args) => args, + decodeArgs: (args) => args, defaultOptions: const TaskOptions(), metadata: const TaskMetadata(), ); @@ -114,20 +138,25 @@ abstract final class StemTaskDefinitions { ); } -final List> _stemTasks = >[ - FunctionTaskHandler( - name: "builder.example.task", - entrypoint: builderExampleTask, - options: const TaskOptions(), - metadata: const TaskMetadata(), - ), - FunctionTaskHandler( - name: "builder.example.ping", - entrypoint: _stemTaskAdapter0, - options: const TaskOptions(), - metadata: const TaskMetadata(), - ), -]; +final List> _stemTasks = + >[ + StemTaskDefinitions.builderExampleTask.handler( + entrypoint: (context, args) => builderExampleTask( + _stemTaskInvocationContext( + context, + StemTaskDefinitions.builderExampleTask.encodeArgs(args), + ), + args, + ), + executionMode: TaskExecutionMode.isolate, + isolateEntrypoint: builderExampleTask, + ), + StemTaskDefinitions.builderExamplePing.asDefinition.handler( + entrypoint: (context, args) => builderPingTask(), + executionMode: TaskExecutionMode.isolate, + isolateEntrypoint: _stemTaskAdapter0, + ), + ]; final List _stemWorkflowManifest = [ diff --git a/packages/stem_builder/example/pubspec.yaml b/packages/stem_builder/example/pubspec.yaml index 6e202d58..e8288218 100644 --- a/packages/stem_builder/example/pubspec.yaml +++ b/packages/stem_builder/example/pubspec.yaml @@ -11,9 +11,9 @@ dependencies: path: ../../stem dev_dependencies: + build_runner: ^2.10.5 stem_builder: path: .. - build_runner: ^2.10.5 dependency_overrides: stem: diff --git a/packages/stem_builder/lib/src/stem_registry_builder.dart b/packages/stem_builder/lib/src/stem_registry_builder.dart index c0e10e3e..b4f15433 100644 --- a/packages/stem_builder/lib/src/stem_registry_builder.dart +++ b/packages/stem_builder/lib/src/stem_registry_builder.dart @@ -1910,23 +1910,33 @@ class _RegistryEmitter { void _emitTasks(StringBuffer buffer) { buffer.writeln( - 'final List> _stemTasks = >[', + 'final List> _stemTasks = >[', ); + final symbolNames = _symbolNamesForTasks(tasks); for (final task in tasks) { - final entrypoint = - task.adapterName ?? _qualify(task.importAlias, task.function); - final metadataCode = _taskMetadataCode(task); - buffer.writeln(' FunctionTaskHandler('); - buffer.writeln(' name: ${_string(task.name)},'); - buffer.writeln(' entrypoint: $entrypoint,'); - if (task.options != null) { - buffer.writeln(' options: ${_dartObjectToCode(task.options!)},'); - } - if (metadataCode != null) { - buffer.writeln(' metadata: $metadataCode,'); - } - if (!task.runInIsolate) { - buffer.writeln(' runInIsolate: false,'); + final symbol = _lowerCamel(symbolNames[task]!); + final definition = task.valueParameters.isEmpty && !task.usesLegacyMapArgs + ? 'StemTaskDefinitions.$symbol.asDefinition' + : 'StemTaskDefinitions.$symbol'; + final applicationContext = task.acceptsTaskContext + ? '_stemTaskInvocationContext(context, $definition.encodeArgs(args))' + : null; + final callArgs = _taskApplicationInvocation( + task, + context: applicationContext, + args: 'args', + ); + buffer.writeln(' $definition.handler('); + buffer.writeln( + ' entrypoint: (context, args) => ${_qualify(task.importAlias, task.function)}($callArgs),', + ); + buffer.writeln( + ' executionMode: TaskExecutionMode.${task.runInIsolate ? 'isolate' : 'inline'},', + ); + if (task.runInIsolate) { + final isolateEntrypoint = + task.adapterName ?? _qualify(task.importAlias, task.function); + buffer.writeln(' isolateEntrypoint: $isolateEntrypoint,'); } buffer.writeln(' ),'); } @@ -1958,6 +1968,7 @@ class _RegistryEmitter { buffer.writeln(' name: ${_string(task.name)},'); if (task.usesLegacyMapArgs) { buffer.writeln(' encodeArgs: (args) => args,'); + buffer.writeln(' decodeArgs: (args) => args,'); } else if (task.valueParameters.isNotEmpty) { buffer.writeln(' encodeArgs: (args) => {'); if (singleParameter != null) { @@ -1974,14 +1985,18 @@ class _RegistryEmitter { } } buffer.writeln(' },'); + buffer.writeln( + ' decodeArgs: (args) => ${_taskArgsDecodeExpression(task)},', + ); } if (task.options != null) { buffer.writeln( ' defaultOptions: ${_dartObjectToCode(task.options!)},', ); } - if (task.metadata != null) { - buffer.writeln(' metadata: ${_dartObjectToCode(task.metadata!)},'); + final metadataCode = _taskMetadataCode(task); + if (metadataCode != null) { + buffer.writeln(' metadata: $metadataCode,'); } if (task.resultPayloadCodecTypeCode != null) { final codecField = @@ -2029,55 +2044,6 @@ class _RegistryEmitter { } } - void _emitGeneratedHelpers(StringBuffer buffer) { - final needsArgHelper = - tasks.any((task) => !task.usesLegacyMapArgs) || - workflows.any( - (workflow) => - workflow.runValueParameters.isNotEmpty || - workflow.steps.any((step) => step.valueParameters.isNotEmpty), - ); - if (!needsArgHelper) { - return; - } - buffer.writeln('Object? _stemRequireArg('); - buffer.writeln(' Map args,'); - buffer.writeln(' String name,'); - buffer.writeln(') {'); - buffer.writeln(' if (!args.containsKey(name)) {'); - buffer.writeln( - " throw ArgumentError('Missing required argument \"\$name\".');", - ); - buffer.writeln(' }'); - buffer.writeln(' return args[name];'); - buffer.writeln('}'); - buffer.writeln(); - } - - void _emitManifest(StringBuffer buffer) { - buffer.writeln( - 'final List _stemWorkflowManifest = [', - ); - buffer.writeln( - ' ..._stemFlows.map((flow) => flow.definition.toManifestEntry()),', - ); - buffer.writeln( - ' ..._stemScripts.map((script) => script.definition.toManifestEntry()),', - ); - buffer.writeln('];'); - buffer.writeln(); - } - - void _emitModule(StringBuffer buffer) { - buffer.writeln('final StemModule stemModule = StemModule('); - buffer.writeln(' flows: _stemFlows,'); - buffer.writeln(' scripts: _stemScripts,'); - buffer.writeln(' tasks: _stemTasks,'); - buffer.writeln(' workflowManifest: _stemWorkflowManifest,'); - buffer.writeln(');'); - buffer.writeln(); - } - String? _taskMetadataCode(_TaskInfo task) { final resultCodecTypeCode = task.resultPayloadCodecTypeCode; if (task.metadata == null && resultCodecTypeCode == null) { @@ -2137,6 +2103,120 @@ class _RegistryEmitter { return 'TaskMetadata(${fields.join(', ')})'; } + String _taskApplicationInvocation( + _TaskInfo task, { + required String? context, + required String args, + }) { + final positional = []; + if (context != null && !task.taskContextIsNamed) { + positional.add(context); + } + if (task.usesLegacyMapArgs) { + positional.add(args); + } else { + final singleParameter = _singleValueParameter(task.valueParameters); + if (singleParameter != null) { + positional.add(args); + } else { + positional.addAll( + task.valueParameters.map((parameter) => '$args.${parameter.name}'), + ); + } + } + return _invocationArgs( + positional: positional, + named: { + if (context != null && task.taskContextIsNamed) + task.taskContextParameterName!: context, + }, + ); + } + + String _taskArgsDecodeExpression(_TaskInfo task) { + final singleParameter = _singleValueParameter(task.valueParameters); + if (singleParameter != null) { + return _decodeArg('args', singleParameter); + } + return '(${task.valueParameters.map((parameter) => '${parameter.name}: ${_decodeArg('args', parameter)}').join(', ')})'; + } + + void _emitGeneratedHelpers(StringBuffer buffer) { + final needsArgHelper = + tasks.any((task) => !task.usesLegacyMapArgs) || + workflows.any( + (workflow) => + workflow.runValueParameters.isNotEmpty || + workflow.steps.any((step) => step.valueParameters.isNotEmpty), + ); + if (needsArgHelper) { + buffer.writeln('Object? _stemRequireArg('); + buffer.writeln(' Map args,'); + buffer.writeln(' String name,'); + buffer.writeln(') {'); + buffer.writeln(' if (!args.containsKey(name)) {'); + buffer.writeln( + " throw ArgumentError('Missing required argument \"\$name\".');", + ); + buffer.writeln(' }'); + buffer.writeln(' return args[name];'); + buffer.writeln('}'); + buffer.writeln(); + } + + if (tasks.any((task) => task.acceptsTaskContext)) { + buffer.writeln( + 'TaskInvocationContext _stemTaskInvocationContext(TaskExecutionContext ' + 'context, Map args) {', + ); + buffer.writeln(' if (context case final TaskInvocationContext value) {'); + buffer.writeln(' return value;'); + buffer.writeln(' }'); + buffer.writeln(' return TaskInvocationContext.local('); + buffer.writeln(' id: context.id,'); + buffer.writeln(' args: args,'); + buffer.writeln(' headers: context.headers,'); + buffer.writeln(' meta: context.meta,'); + buffer.writeln(' attempt: context.attempt,'); + buffer.writeln(' heartbeat: context.heartbeat,'); + buffer.writeln(' extendLease: context.extendLease,'); + buffer.writeln( + ' progress: (percent, {data}) => context.progress(percent, data: data),', + ); + buffer.writeln(' cancellation: context.cancellation,'); + buffer.writeln(' enqueuer: context,'); + buffer.writeln(' workflows: context,'); + buffer.writeln(' workflowEvents: context,'); + buffer.writeln(' );'); + buffer.writeln('}'); + buffer.writeln(); + } + } + + void _emitManifest(StringBuffer buffer) { + buffer.writeln( + 'final List _stemWorkflowManifest = [', + ); + buffer.writeln( + ' ..._stemFlows.map((flow) => flow.definition.toManifestEntry()),', + ); + buffer.writeln( + ' ..._stemScripts.map((script) => script.definition.toManifestEntry()),', + ); + buffer.writeln('];'); + buffer.writeln(); + } + + void _emitModule(StringBuffer buffer) { + buffer.writeln('final StemModule stemModule = StemModule('); + buffer.writeln(' flows: _stemFlows,'); + buffer.writeln(' scripts: _stemScripts,'); + buffer.writeln(' tasks: _stemTasks,'); + buffer.writeln(' workflowManifest: _stemWorkflowManifest,'); + buffer.writeln(');'); + buffer.writeln(); + } + String _decodeArg(String sourceMap, _ValueParameterInfo parameter) { final codecTypeCode = parameter.payloadCodecTypeCode; if (codecTypeCode != null) { diff --git a/packages/stem_builder/pubspec.yaml b/packages/stem_builder/pubspec.yaml index 54c99d56..59140c38 100644 --- a/packages/stem_builder/pubspec.yaml +++ b/packages/stem_builder/pubspec.yaml @@ -1,6 +1,6 @@ name: stem_builder description: Build-time registry generator for annotated Stem workflows and tasks. -version: 0.2.0 +version: 0.3.0 repository: https://github.com/kingwill101/stem resolution: workspace @@ -13,7 +13,7 @@ dependencies: dart_style: ^3.1.4 glob: ^2.1.3 source_gen: ^4.1.2 - stem: ">=0.2.0-dev <0.3.0" + stem: ">=0.3.0 <0.4.0" dev_dependencies: build_runner: ^2.10.5 diff --git a/packages/stem_builder/test/stem_registry_builder_test.dart b/packages/stem_builder/test/stem_registry_builder_test.dart index 4f2cb112..46dc2cb5 100644 --- a/packages/stem_builder/test/stem_registry_builder_test.dart +++ b/packages/stem_builder/test/stem_registry_builder_test.dart @@ -232,7 +232,7 @@ Future sendEmail( contains('Flow('), contains('WorkflowScript('), contains('stemModule = StemModule('), - contains('FunctionTaskHandler'), + contains('TypedTaskHandler'), contains("part of 'workflows.dart';"), isNot(contains('StemGeneratedTaskEnqueuer')), isNot(contains('StemGeneratedTaskResults')), @@ -1234,7 +1234,8 @@ Future typedTask( contains('Future _stemTaskAdapter0('), contains('_stemRequireArg(args, "email") as String'), contains('_stemRequireArg(args, "retries") as int'), - contains('entrypoint: _stemTaskAdapter0'), + contains('executionMode: TaskExecutionMode.isolate'), + contains('isolateEntrypoint: _stemTaskAdapter0'), ]), ), }, diff --git a/packages/stem_cli/.pubignore b/packages/stem_cli/.pubignore new file mode 100644 index 00000000..db0ca9f9 --- /dev/null +++ b/packages/stem_cli/.pubignore @@ -0,0 +1,3 @@ +# Disposable integration assets are generated by the test environment and +# must never be included in the published CLI package. +docker/testing/postgres/certs/ diff --git a/packages/stem_cli/CHANGELOG.md b/packages/stem_cli/CHANGELOG.md index c858777a..44a8367d 100644 --- a/packages/stem_cli/CHANGELOG.md +++ b/packages/stem_cli/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +## 0.2.0 + +- Updated CLI adapter dependencies for the Stem 0.3.0 release train. +- Kept worker startup explicit and retained `schedule trigger` support. +- Made Docker-backed integration setup reuse disposable TLS assets and wait for + Redis/Postgres health before exporting test endpoints. + +## 0.1.2 + +- Added `stem schedule trigger` for one-off execution without mutating the + persisted recurring schedule. +- Hardened CLI integration discovery and documented explicit worker startup. +- Removed the prerelease core dependency range from the published-package + manifest. + ## 0.1.0 - Updated CLI adapter wiring and docker test stack to the Ormed-backed diff --git a/packages/stem_cli/_init_test_env b/packages/stem_cli/_init_test_env index 2c13181f..4665cf22 100755 --- a/packages/stem_cli/_init_test_env +++ b/packages/stem_cli/_init_test_env @@ -20,15 +20,18 @@ if ! command -v docker >/dev/null 2>&1; then if [ "$SOURCED" -eq 1 ]; then return 1; else exit 1; fi fi +"$SCRIPT_DIR/docker/testing/generate_certs.sh" + echo "Starting Redis/Postgres test services (docker compose)..." STEM_TEST_REDIS_TLS_PORT="${STEM_TEST_REDIS_TLS_PORT:-56380}" STEM_TEST_REDIS_MTLS_PORT="${STEM_TEST_REDIS_MTLS_PORT:-56381}" export STEM_TEST_REDIS_TLS_PORT STEM_TEST_REDIS_MTLS_PORT -docker compose -f "$COMPOSE_FILE" up -d postgres postgres_tls redis redis_tls redis_mtls +docker compose -f "$COMPOSE_FILE" up -d --build --force-recreate --wait --wait-timeout 60 \ + postgres postgres_tls redis redis_tls redis_mtls REDIS_URL="redis://127.0.0.1:56379" POSTGRES_URL="postgresql://postgres:postgres@127.0.0.1:65432/stem_test" -CA_CERT_PATH="$SCRIPT_DIR/docker/testing/certs/postgres-root.crt" +CA_CERT_PATH="$SCRIPT_DIR/docker/testing/postgres/certs/root.crt" REDIS_TLS_CA_CERT_PATH="$SCRIPT_DIR/../stem/example/microservice/certs/ca.crt" REDIS_MTLS_CLIENT_CERT_PATH="$SCRIPT_DIR/../stem/example/microservice/certs/client.crt" REDIS_MTLS_CLIENT_KEY_PATH="$SCRIPT_DIR/../stem/example/microservice/certs/client.key" diff --git a/packages/stem_cli/docker/testing/README.md b/packages/stem_cli/docker/testing/README.md index 2719b99a..7e1dc95b 100644 --- a/packages/stem_cli/docker/testing/README.md +++ b/packages/stem_cli/docker/testing/README.md @@ -5,6 +5,7 @@ Complete Docker Compose setup for testing Stem with PostgreSQL, Redis, Prometheu ## Quick Start (30 seconds) ```bash +./generate_certs.sh # Start all services docker compose up -d --wait --wait-timeout 60 @@ -37,6 +38,9 @@ dart test -r expanded -j 1 | `grafana-datasources.yml` | Pre-configured datasources | | `TESTING.md` | Comprehensive testing guide | +TLS certificates and private keys are generated by `generate_certs.sh` and are +ignored by Git. Do not commit them or add them to package archives. + ## Environment Variables Automatically exported by `_init_test_env`: diff --git a/packages/stem_cli/docker/testing/TESTING.md b/packages/stem_cli/docker/testing/TESTING.md index d6a08293..00032afc 100644 --- a/packages/stem_cli/docker/testing/TESTING.md +++ b/packages/stem_cli/docker/testing/TESTING.md @@ -26,7 +26,7 @@ source ../../_init_test_env export STEM_TEST_REDIS_URL="redis://127.0.0.1:56379" export STEM_TEST_POSTGRES_URL="postgresql://postgres:postgres@127.0.0.1:65432/stem_test" export STEM_TEST_POSTGRES_TLS_URL="postgresql://postgres:postgres@127.0.0.1:65432/stem_test" -export STEM_TEST_POSTGRES_TLS_CA_CERT="$(pwd)/certs/postgres-root.crt" +export STEM_TEST_POSTGRES_TLS_CA_CERT="$(pwd)/postgres/certs/root.crt" ``` ### 3. Run Tests @@ -191,11 +191,11 @@ The PostgreSQL service is configured with TLS support using self-signed certific ```bash # TLS certificate location -packages/stem_cli/docker/testing/certs/postgres-root.crt +packages/stem_cli/docker/testing/postgres/certs/root.crt # TLS environment variables (exported by _init_test_env) export STEM_TEST_POSTGRES_TLS_URL="postgresql://postgres:postgres@127.0.0.1:65432/stem_test" -export STEM_TEST_POSTGRES_TLS_CA_CERT="packages/stem_cli/docker/testing/certs/postgres-root.crt" +export STEM_TEST_POSTGRES_TLS_CA_CERT="packages/stem_cli/docker/testing/postgres/certs/root.crt" # Run TLS integration tests dart test packages/stem_postgres/test/integration -r expanded @@ -310,7 +310,7 @@ dart test packages/stem_postgres -r expanded | `STEM_TEST_REDIS_URL` | `redis://127.0.0.1:56379` | Redis broker connection | | `STEM_TEST_POSTGRES_URL` | `postgresql://postgres:postgres@127.0.0.1:65432/stem_test` | PostgreSQL connection | | `STEM_TEST_POSTGRES_TLS_URL` | Same as above (TLS enabled) | PostgreSQL TLS connection | -| `STEM_TEST_POSTGRES_TLS_CA_CERT` | `./certs/postgres-root.crt` | PostgreSQL TLS CA certificate | +| `STEM_TEST_POSTGRES_TLS_CA_CERT` | `./postgres/certs/root.crt` | PostgreSQL TLS CA certificate | | `STEM_BROKER_URL` | Depends on broker choice | Broker connection string | | `STEM_RESULT_BACKEND_URL` | Depends on backend choice | Result backend connection string | diff --git a/packages/stem_cli/docker/testing/certs/postgres-root.crt b/packages/stem_cli/docker/testing/certs/postgres-root.crt deleted file mode 100644 index 1aa6d571..00000000 --- a/packages/stem_cli/docker/testing/certs/postgres-root.crt +++ /dev/null @@ -1,19 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDEzCCAfugAwIBAgIURYRu4D/FqIumdTVae9JoVdsm/x8wDQYJKoZIhvcNAQEL -BQAwGTEXMBUGA1UEAwwOU3RlbVRlc3RSb290Q0EwHhcNMjUxMDI0MTYzNzQzWhcN -MzUxMDIyMTYzNzQzWjAZMRcwFQYDVQQDDA5TdGVtVGVzdFJvb3RDQTCCASIwDQYJ -KoZIhvcNAQEBBQADggEPADCCAQoCggEBAOGAMzMKfWV+0DJHM0anvY2lnDpRH9bI -qDiehQSPeemZzXy+ndldFSz6g9bnIoxW5ToqNtgriMwK+q496XcJVqVpYHQJ8IQp -zh6L3y3cUV3OXNFPjOKffqocF92hXKTc7Z9UyO2H7kayDpzzXtnURUqvOpgmSLIH -7hp05yXPE1jrH1zSRKDr3lXE53c9K4bZDPmEMvgfkO5iVqOtAKO1wAyeBy5vCtg8 -sQK/lX54n+zzrV77sDr4MY0mewI3k101H6gT1rratIpw9W28IaqeRIy/W/Fl3Kd/ -LaWYrXt2G+byBNe1uGTwCWt9bKSIp1lZVJCzca7QgWGe8UeSuqlYDOcCAwEAAaNT -MFEwHQYDVR0OBBYEFDdubWJNz9GmOANZlNVfx1nE8q+NMB8GA1UdIwQYMBaAFDdu -bWJNz9GmOANZlNVfx1nE8q+NMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEL -BQADggEBAL03dfjbfqdCBr2VQzL+Ar36on6qMRBtuPrflsLEQUpMzs2z8lPpbxcI -f1ngzd2kE7zRSozWLDUyvm30Bz9eCfyEcp8IPxrnA3CYgQLv8kYhXLV1NDbxX5KD -XPTlJlM1TIHPTjRwlVRSOTB+rHijbu5qAf6caS8CjuyGdRUdSdfDWyxKhcPKjEeB -a1JyCjN4NpnjyIlEM0oSKB6CBxWMMUKXcZJ9Y5Fu4UeiKL5Kdg8MFEpXyaFJvZOO -xctAvKEYOxV8h7oYDbm/JaCtDS/mZX6onRNIh5nTGQBd9Heuk/c9ZaFFElRRK1ai -BclHPsA48Fhzw3IMFOdwgknfTuIDzj4= ------END CERTIFICATE----- diff --git a/packages/stem_cli/docker/testing/generate_certs.sh b/packages/stem_cli/docker/testing/generate_certs.sh new file mode 100755 index 00000000..a7ff4b2d --- /dev/null +++ b/packages/stem_cli/docker/testing/generate_certs.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Integration TLS assets are disposable fixtures. Keep them out of Git and +# regenerate them before Docker Compose starts. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../../../.." && pwd)" +TLS_GENERATOR="$REPO_ROOT/packages/stem/scripts/security/generate_tls_assets.sh" +REDIS_CERT_DIR="$REPO_ROOT/packages/stem/example/microservice/certs" +POSTGRES_CERT_DIR="$SCRIPT_DIR/postgres/certs" + +redis_assets=( + "$REDIS_CERT_DIR/ca.crt" + "$REDIS_CERT_DIR/server.crt" + "$REDIS_CERT_DIR/server.key" + "$REDIS_CERT_DIR/client.crt" + "$REDIS_CERT_DIR/client.key" +) +postgres_assets=( + "$POSTGRES_CERT_DIR/root.crt" + "$POSTGRES_CERT_DIR/server.crt" + "$POSTGRES_CERT_DIR/server.key" +) + +if ! command -v openssl >/dev/null 2>&1; then + echo "openssl is required to generate integration TLS assets." >&2 + exit 1 +fi + +needs_generation=0 +for asset in "${redis_assets[@]}" "${postgres_assets[@]}"; do + if [[ ! -f "$asset" ]]; then + needs_generation=1 + break + fi +done + +if [[ "$needs_generation" -eq 1 ]]; then + "$TLS_GENERATOR" "$REDIS_CERT_DIR" "redis" "redis,localhost,127.0.0.1" >/dev/null + + temporary_dir="$(mktemp -d)" + trap 'rm -rf "$temporary_dir"' EXIT + "$TLS_GENERATOR" "$temporary_dir" "postgres" "postgres,localhost,127.0.0.1" \ + >/dev/null + mkdir -p "$POSTGRES_CERT_DIR" + cp "$temporary_dir/ca.crt" "$POSTGRES_CERT_DIR/root.crt" + cp "$temporary_dir/server.crt" "$POSTGRES_CERT_DIR/server.crt" + cp "$temporary_dir/server.key" "$POSTGRES_CERT_DIR/server.key" + chmod 600 "$POSTGRES_CERT_DIR/server.key" +else + echo "Reusing existing disposable TLS assets." +fi + +# Redis runs as the unprivileged `redis` user inside the test container. These +# assets are disposable, ignored fixtures, so make the mounted Redis files +# readable by that user while leaving the general TLS generator restrictive. +chmod 644 \ + "$REDIS_CERT_DIR/ca.crt" \ + "$REDIS_CERT_DIR/server.crt" \ + "$REDIS_CERT_DIR/server.key" \ + "$REDIS_CERT_DIR/client.crt" \ + "$REDIS_CERT_DIR/client.key" + +echo "Disposable Redis and PostgreSQL TLS assets are ready." diff --git a/packages/stem_cli/docker/testing/postgres/certs/root.crt b/packages/stem_cli/docker/testing/postgres/certs/root.crt deleted file mode 100644 index 1aa6d571..00000000 --- a/packages/stem_cli/docker/testing/postgres/certs/root.crt +++ /dev/null @@ -1,19 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDEzCCAfugAwIBAgIURYRu4D/FqIumdTVae9JoVdsm/x8wDQYJKoZIhvcNAQEL -BQAwGTEXMBUGA1UEAwwOU3RlbVRlc3RSb290Q0EwHhcNMjUxMDI0MTYzNzQzWhcN -MzUxMDIyMTYzNzQzWjAZMRcwFQYDVQQDDA5TdGVtVGVzdFJvb3RDQTCCASIwDQYJ -KoZIhvcNAQEBBQADggEPADCCAQoCggEBAOGAMzMKfWV+0DJHM0anvY2lnDpRH9bI -qDiehQSPeemZzXy+ndldFSz6g9bnIoxW5ToqNtgriMwK+q496XcJVqVpYHQJ8IQp -zh6L3y3cUV3OXNFPjOKffqocF92hXKTc7Z9UyO2H7kayDpzzXtnURUqvOpgmSLIH -7hp05yXPE1jrH1zSRKDr3lXE53c9K4bZDPmEMvgfkO5iVqOtAKO1wAyeBy5vCtg8 -sQK/lX54n+zzrV77sDr4MY0mewI3k101H6gT1rratIpw9W28IaqeRIy/W/Fl3Kd/ -LaWYrXt2G+byBNe1uGTwCWt9bKSIp1lZVJCzca7QgWGe8UeSuqlYDOcCAwEAAaNT -MFEwHQYDVR0OBBYEFDdubWJNz9GmOANZlNVfx1nE8q+NMB8GA1UdIwQYMBaAFDdu -bWJNz9GmOANZlNVfx1nE8q+NMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEL -BQADggEBAL03dfjbfqdCBr2VQzL+Ar36on6qMRBtuPrflsLEQUpMzs2z8lPpbxcI -f1ngzd2kE7zRSozWLDUyvm30Bz9eCfyEcp8IPxrnA3CYgQLv8kYhXLV1NDbxX5KD -XPTlJlM1TIHPTjRwlVRSOTB+rHijbu5qAf6caS8CjuyGdRUdSdfDWyxKhcPKjEeB -a1JyCjN4NpnjyIlEM0oSKB6CBxWMMUKXcZJ9Y5Fu4UeiKL5Kdg8MFEpXyaFJvZOO -xctAvKEYOxV8h7oYDbm/JaCtDS/mZX6onRNIh5nTGQBd9Heuk/c9ZaFFElRRK1ai -BclHPsA48Fhzw3IMFOdwgknfTuIDzj4= ------END CERTIFICATE----- diff --git a/packages/stem_cli/docker/testing/postgres/certs/root.srl b/packages/stem_cli/docker/testing/postgres/certs/root.srl deleted file mode 100644 index 83c56d4f..00000000 --- a/packages/stem_cli/docker/testing/postgres/certs/root.srl +++ /dev/null @@ -1 +0,0 @@ -5C41C2C63114A512E70A6ACB948473E0B2304D3F diff --git a/packages/stem_cli/docker/testing/postgres/certs/server.cnf b/packages/stem_cli/docker/testing/postgres/certs/server.cnf deleted file mode 100644 index 4d6c0c33..00000000 --- a/packages/stem_cli/docker/testing/postgres/certs/server.cnf +++ /dev/null @@ -1,18 +0,0 @@ -[req] -default_bits = 2048 -distinguished_name = req_distinguished_name -req_extensions = v3_req -prompt = no - -[req_distinguished_name] -CN = localhost - -[v3_req] -keyUsage = nonRepudiation, digitalSignature, keyEncipherment -extendedKeyUsage = serverAuth -subjectAltName = @alt_names - -[alt_names] -DNS.1 = localhost -DNS.2 = postgres -IP.1 = 127.0.0.1 diff --git a/packages/stem_cli/docker/testing/postgres/certs/server.crt b/packages/stem_cli/docker/testing/postgres/certs/server.crt deleted file mode 100644 index 661d2de8..00000000 --- a/packages/stem_cli/docker/testing/postgres/certs/server.crt +++ /dev/null @@ -1,20 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDRzCCAi+gAwIBAgIUXEHCxjEUpRLnCmrLlIRz4LIwTT8wDQYJKoZIhvcNAQEL -BQAwGTEXMBUGA1UEAwwOU3RlbVRlc3RSb290Q0EwHhcNMjUxMDI0MTYzODEyWhcN -MzAxMDIzMTYzODEyWjAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3 -DQEBAQUAA4IBDwAwggEKAoIBAQDbJ71OoPXMIa715ePWLr1Qy8oUGw+BJyF8MIsp -sRWc8wNduv7+xJuCKzlDP1+OB8kij2dxYAN7Eyp3nGyQxjC5Hiq/AUBJZf1TzOSX -l3/9Ab3AEDcoYUN5K6RrhOfdvCzp594hYD/P+jdZm9Kv5jJv1/rxU3ZGLVqsD+Jr -VPDBWUy4YjYVXQh6pBGT/QREDUCqyqzcQbzGixZAq0/7e8QTC4W6kn1kB4Ie/EY1 -xuNerc6N1LS0CeUOa0NDj3Mm5yQDr/w6ugrAGATWUq4As90J9YyPRT2/k3FwnyIN -QSjjAwrNxncIOCu8bHR29QQJLQGZ8BVIQESGJxXnKa5EwlgLAgMBAAGjgYswgYgw -CwYDVR0PBAQDAgXgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMCQGA1UdEQQdMBuCCWxv -Y2FsaG9zdIIIcG9zdGdyZXOHBH8AAAEwHQYDVR0OBBYEFKvokmsZdnK4mqyM2KFy -3/v0zhADMB8GA1UdIwQYMBaAFDdubWJNz9GmOANZlNVfx1nE8q+NMA0GCSqGSIb3 -DQEBCwUAA4IBAQAZXCKquWjyqzjOgJ8dOYjb/A7FCSJR88ej+Tbck9VVFM9XrNzG -XaPpyPWq9FVsWwXNcjtg11V34bUMmzcJBnr+gV4bLkL31ceULjbCUM8tdpX3GDba -xgahg14aGW2XW1nXCR+3huc6l1IbInK+AY2BC+/ixdPGPnLSvr0wciysTcO29UpC -vjeAFeOklrYx6LVuDFwsl/w9z/ByWsbVitOPbml0dx+5wcp4qqTQKTciuvgKJTEF -KeSfXewdVXlBw4GfzLka/7EftVNC+U09DDblLH0GdBzvw9GGY2va8Lj8Xpsb3JFX -MukTc5StQWVzsE9uY71xYy3scqLU18SPnqMa ------END CERTIFICATE----- diff --git a/packages/stem_cli/docker/testing/postgres/certs/server.key b/packages/stem_cli/docker/testing/postgres/certs/server.key deleted file mode 100644 index eeb5354e..00000000 --- a/packages/stem_cli/docker/testing/postgres/certs/server.key +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDbJ71OoPXMIa71 -5ePWLr1Qy8oUGw+BJyF8MIspsRWc8wNduv7+xJuCKzlDP1+OB8kij2dxYAN7Eyp3 -nGyQxjC5Hiq/AUBJZf1TzOSXl3/9Ab3AEDcoYUN5K6RrhOfdvCzp594hYD/P+jdZ -m9Kv5jJv1/rxU3ZGLVqsD+JrVPDBWUy4YjYVXQh6pBGT/QREDUCqyqzcQbzGixZA -q0/7e8QTC4W6kn1kB4Ie/EY1xuNerc6N1LS0CeUOa0NDj3Mm5yQDr/w6ugrAGATW -Uq4As90J9YyPRT2/k3FwnyINQSjjAwrNxncIOCu8bHR29QQJLQGZ8BVIQESGJxXn -Ka5EwlgLAgMBAAECggEASttgmMZ3HuJUgk6ji6nMZZgubr6thZaJpANqzXNc6A7D -1BTG+9vJKGW+Of+gEoILTGBCW6IJnlT2GQQXuz3PL7QjsrlpLYkGRkUCwdrwT0n0 -d/Gx/Xt3kiB7mXQ8xOr9Rmo6YBkhwtb7aALHLOkKylQyNZMmH6NnSMVmN/Y8BDZr -0NjPMiDkRf23vElEsEGUnasQS5rY795UrxqCkOZ7yLBTXbzlxRjbZd5repMWEg1N -nF6eDuGHBKlPjewuO9vGqsFBW0T5wC6WF5XDVMO/+VhbLa1hbZJxf+pqgPm+XmY5 -Nnatin478iShq2grB5WJWOmm5eaqUyXAD7LQsSH6iQKBgQD8UsVXGVaQvBhA3F2f -3yXOw9uLbwUSVJfZrNR3FDz6yfliQEGCx8MsrXkIqBIq6tvO08s70jEzdd3FFJGR -7bbfBD2c1Y+ISKy9Jp75Ca5ZHcDcpjEtyAay+6uoeK2S7YmaUAkpRw7yjWeBaWT7 -9UjnlVBfX/n036IOgIH42enj2QKBgQDeWT5Lh5ycnYCDk2wIkw59IpUD+K+ETb2k -LrVt7qZEru9/lo9m8YbBA2jAMUSp2NvMNh1dq5rKvdAlH4zrTcMCiwSb38GCj9eq -xAIZ+Qztx1gJFQPD0LzQc/UFCxckK1esZXk8nCyHcQJ+KlrDdX8N3LlImJCwKSnR -pB0LArDAgwKBgGSpHKMHr7oj38UYZJ1f53RaPF6e1rYeW0nyofwzbbRbGQsyOTBU -Qy+ysONcMYs/kJstcRmXXBt8fRGfiAdgfyDvTykfJZin7Ym0O518WDqjHQqpnlr2 -Pn3u2udl5VEFRGz0WwNFntfyw4UkJ65Ah2oj7iFuZ4hiIwOIDtHZFW7ZAoGBALUR -xBNKGpXIBbIgyeepqjDpiqSzTmvVy0v65bDimfBw7YTfnWRd6Dkih5mMBbizaztA -DmYLq4DTVsAEk4PDzb0ybMxrSXEDpsPf3z47fPXeUqkCvhAm2rx+0dgLkgrYaQtx -j2t2WyTpOPh5kXR/KLqnNxQRejKbMv0QQr6GsF31AoGBAKfowwqByPqO7ep3zVda -oO0uhPyeAazwWQl3qszWL/6dA4PAKVvQ9oCMRy5ei5G+Mhy3JA3bV4PhwjDBOLnH -MC6paNyw+OPJxcvVqVEaPUAFJqh4ENr3AyVNmgJ9l3qYTxYb31bpavHktT92hMi6 -USzohv9gDTq5eCSb5qicFBit ------END PRIVATE KEY----- diff --git a/packages/stem_cli/lib/src/cli/dependencies.dart b/packages/stem_cli/lib/src/cli/dependencies.dart index 0aac2f32..eb599537 100644 --- a/packages/stem_cli/lib/src/cli/dependencies.dart +++ b/packages/stem_cli/lib/src/cli/dependencies.dart @@ -2,6 +2,7 @@ import 'dart:async'; import 'dart:io'; import 'package:artisanal/artisanal.dart'; +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; import 'package:stem_cli/src/cli/cli_runner.dart'; import 'package:stem_cli/src/cli/schedule.dart'; diff --git a/packages/stem_cli/lib/src/cli/revoke_store_factory.dart b/packages/stem_cli/lib/src/cli/revoke_store_factory.dart index cb8f5b3e..33f7fb89 100644 --- a/packages/stem_cli/lib/src/cli/revoke_store_factory.dart +++ b/packages/stem_cli/lib/src/cli/revoke_store_factory.dart @@ -1,5 +1,6 @@ import 'dart:io'; +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; // import 'package:stem_cloud_worker/stem_cloud_worker.dart'; import 'package:stem_postgres/stem_postgres.dart'; diff --git a/packages/stem_cli/lib/src/cli/schedule.dart b/packages/stem_cli/lib/src/cli/schedule.dart index 4355e545..6fd7aa8d 100644 --- a/packages/stem_cli/lib/src/cli/schedule.dart +++ b/packages/stem_cli/lib/src/cli/schedule.dart @@ -15,6 +15,7 @@ class ScheduleCommand extends Command { ScheduleCommand(this.dependencies) { addSubcommand(ScheduleListCommand(dependencies)); addSubcommand(ScheduleShowCommand(dependencies)); + addSubcommand(ScheduleTriggerCommand(dependencies)); addSubcommand(ScheduleApplyCommand(dependencies)); addSubcommand(ScheduleDeleteCommand(dependencies)); addSubcommand(ScheduleEnableCommand(dependencies)); @@ -145,6 +146,91 @@ class ScheduleShowCommand extends Command { } } +class ScheduleTriggerCommand extends Command { + ScheduleTriggerCommand(this.dependencies) { + argParser.addOption('id', abbr: 'i', help: 'Schedule identifier'); + } + + final StemCommandDependencies dependencies; + + @override + final String name = 'trigger'; + + @override + final String description = 'Enqueue one immediate execution.'; + + @override + Future run() async { + final args = argResults!; + final id = + args['id'] as String? ?? + (args.rest.isNotEmpty ? args.rest.first : null); + if (id == null || id.isEmpty) { + dependencies.err.writeln( + 'Missing schedule identifier (use --id or positional argument).', + ); + return 64; + } + + final scheduleCtx = await dependencies.createScheduleContext(); + try { + ScheduleEntry? entry; + if (scheduleCtx.store != null) { + entry = await scheduleCtx.store!.get(id); + } else { + final entries = await scheduleCtx.repo!.load(); + for (final candidate in entries) { + if (candidate.id == id) { + entry = candidate; + break; + } + } + } + if (entry == null) { + dependencies.err.writeln('Schedule "$id" not found.'); + return 64; + } + + final cliCtx = await dependencies.createCliContext(); + try { + final envelope = Envelope( + name: entry.taskName, + args: entry.args, + queue: entry.queue, + headers: {'scheduled-from': 'cli-trigger', 'schedule-id': entry.id}, + meta: { + ...entry.meta, + if (entry.kwargs.isNotEmpty) 'kwargs': entry.kwargs, + 'stem.schedule.trigger': true, + }, + ); + await cliCtx.broker.publish(envelope); + final backend = cliCtx.backend; + if (backend != null) { + await backend.set( + envelope.id, + TaskState.queued, + attempt: envelope.attempt, + meta: { + 'queue': envelope.queue, + 'scheduleId': entry.id, + 'stem.schedule.trigger': true, + }, + ); + } + dependencies.out.writeln( + 'Triggered schedule "${entry.id}" as task ${envelope.id}.', + ); + return 0; + } finally { + await cliCtx.dispose(); + } + } finally { + await scheduleCtx.dispose(); + } + } +} + class ScheduleApplyCommand extends Command { ScheduleApplyCommand(this.dependencies) { argParser diff --git a/packages/stem_cli/lib/src/cli/utilities.dart b/packages/stem_cli/lib/src/cli/utilities.dart index 043f2815..d8689398 100644 --- a/packages/stem_cli/lib/src/cli/utilities.dart +++ b/packages/stem_cli/lib/src/cli/utilities.dart @@ -1,6 +1,7 @@ import 'dart:io'; import 'package:artisanal/args.dart'; +import 'package:stem/memory.dart'; import 'package:stem/stem.dart' hide RevokeStoreFactory; import 'package:stem_cli/src/cli/cli_runner.dart'; import 'package:stem_cli/src/cli/cloud_config.dart'; diff --git a/packages/stem_cli/lib/src/cli/worker.dart b/packages/stem_cli/lib/src/cli/worker.dart index 37d72559..f716f884 100644 --- a/packages/stem_cli/lib/src/cli/worker.dart +++ b/packages/stem_cli/lib/src/cli/worker.dart @@ -5,6 +5,7 @@ import 'dart:io'; import 'package:artisanal/args.dart'; import 'package:redis/redis.dart' as redis; import 'package:path/path.dart' as p; +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; import 'package:stem_cli/src/cli/cli_runner.dart'; import 'package:stem_cli/src/cli/dependencies.dart'; @@ -1202,7 +1203,7 @@ class WorkerStatusCommand extends Command { } try { - return _workerStatus( + return await _workerStatus( argResults!, dependencies.out, dependencies.err, diff --git a/packages/stem_cli/pubspec.yaml b/packages/stem_cli/pubspec.yaml index e59df733..20bb6f3c 100644 --- a/packages/stem_cli/pubspec.yaml +++ b/packages/stem_cli/pubspec.yaml @@ -1,6 +1,6 @@ name: stem_cli description: Command-line tooling for Stem that orchestrates Redis and Postgres adapters. -version: 0.1.0 +version: 0.2.0 repository: https://github.com/kingwill101/stem resolution: workspace environment: @@ -8,10 +8,10 @@ environment: dependencies: artisanal: ^0.2.0 - stem: ">=0.2.0-dev <0.3.0" - stem_redis: ^0.1.0 - stem_postgres: ^0.1.0 - stem_sqlite: ^0.1.0 + stem: ">=0.3.0 <0.4.0" + stem_redis: ^0.2.0 + stem_postgres: ^0.2.0 + stem_sqlite: ^0.2.0 redis: ^4.0.0 postgres: ^3.5.9 async: ^2.13.0 @@ -24,6 +24,3 @@ dev_dependencies: lints: ^6.0.0 test: ^1.29.0 very_good_analysis: ^10.0.0 - -false_secrets: - - docker/testing/postgres/certs/server.key diff --git a/packages/stem_cli/test/integration/cli/cli_health_integration_test.dart b/packages/stem_cli/test/integration/cli/cli_health_integration_test.dart index 155415e8..75f01e2e 100644 --- a/packages/stem_cli/test/integration/cli/cli_health_integration_test.dart +++ b/packages/stem_cli/test/integration/cli/cli_health_integration_test.dart @@ -22,7 +22,8 @@ void main() { // Postgres adapters resolve ormed.yaml relative to cwd; point at the // package config so health checks can open connections. final originalDir = Directory.current; - Directory.current = Directory('../stem_postgres').absolute; + final postgresDir = _findPostgresPackageDirectory(); + Directory.current = postgresDir; addTearDown(() { Directory.current = originalDir; }); @@ -52,6 +53,21 @@ void main() { }); } +Directory _findPostgresPackageDirectory() { + final candidates = [ + Directory('../stem_postgres'), + Directory('../packages/stem_postgres'), + Directory('packages/stem_postgres'), + ]; + for (final candidate in candidates) { + final absolute = candidate.absolute; + if (absolute.existsSync()) return absolute; + } + throw StateError( + 'Could not locate packages/stem_postgres from ${Directory.current.path}.', + ); +} + Future _canConnect(String host, int port) async { try { final socket = await Socket.connect( diff --git a/packages/stem_cli/test/unit/cli/cli_schedule_test.dart b/packages/stem_cli/test/unit/cli/cli_schedule_test.dart index b77fb513..c3e61d92 100644 --- a/packages/stem_cli/test/unit/cli/cli_schedule_test.dart +++ b/packages/stem_cli/test/unit/cli/cli_schedule_test.dart @@ -1,7 +1,9 @@ +import 'dart:async'; import 'dart:convert'; import 'dart:io'; import 'package:test/test.dart'; +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; import 'package:stem_cli/src/cli/cli_runner.dart'; import 'package:stem_cli/src/cli/file_schedule_repository.dart'; @@ -72,6 +74,58 @@ void main() { expect(lines, hasLength(3)); }); + test('trigger publishes once without changing the schedule', () async { + final repo = FileScheduleRepository(path: scheduleFile); + final entry = ScheduleEntry( + id: 'cleanup', + taskName: 'noop', + queue: 'priority', + args: const {'limit': 3}, + spec: IntervalScheduleSpec(every: const Duration(minutes: 1)), + ); + await repo.save([entry]); + + final broker = InMemoryBroker(); + final backend = InMemoryResultBackend(); + final received = []; + final subscription = broker + .consume( + RoutingSubscription.singleQueue('priority'), + consumerName: 'schedule-trigger-test', + ) + .listen((delivery) async { + received.add(delivery.envelope); + await broker.ack(delivery); + }); + + final out = StringBuffer(); + final code = await runStemCli( + ['schedule', 'trigger', 'cleanup'], + out: out, + scheduleFilePath: scheduleFile, + contextBuilder: () async => CliContext( + broker: broker, + backend: backend, + routing: RoutingRegistry(RoutingConfig.legacy()), + dispose: () async {}, + ), + ); + + await _waitUntil(() => received.isNotEmpty); + expect(code, equals(0)); + expect(received.single.name, equals('noop')); + expect(received.single.args, equals({'limit': 3})); + expect(received.single.meta['stem.schedule.trigger'], isTrue); + expect(out.toString(), contains('Triggered schedule "cleanup"')); + final unchanged = (await repo.load()).single; + expect(unchanged.totalRunCount, equals(entry.totalRunCount)); + expect(unchanged.lastRunAt, isNull); + expect(await backend.get(received.single.id), isNotNull); + + await subscription.cancel(); + broker.dispose(); + }); + test('delete removes schedule', () async { final defs = File('${tempDir.path}/definitions.json') ..writeAsStringSync( @@ -363,6 +417,18 @@ void main() { }); } +Future _waitUntil( + FutureOr Function() predicate, { + Duration timeout = const Duration(seconds: 2), +}) async { + final deadline = DateTime.now().add(timeout); + while (DateTime.now().isBefore(deadline)) { + if (await predicate()) return; + await Future.delayed(const Duration(milliseconds: 10)); + } + throw TimeoutException('Condition was not met within $timeout.'); +} + class FlakyScheduleStore implements ScheduleStore { FlakyScheduleStore(this.delegate, {this.failuresBeforeSuccess = 1}); diff --git a/packages/stem_cli/test/unit/cli/cli_tasks_test.dart b/packages/stem_cli/test/unit/cli/cli_tasks_test.dart index dcf5821c..a3c44376 100644 --- a/packages/stem_cli/test/unit/cli/cli_tasks_test.dart +++ b/packages/stem_cli/test/unit/cli/cli_tasks_test.dart @@ -1,5 +1,6 @@ import 'dart:convert'; +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; import 'package:stem_cli/src/cli/cli_runner.dart'; import 'package:test/test.dart'; diff --git a/packages/stem_cli/test/unit/cli/cli_worker_multi_test.dart b/packages/stem_cli/test/unit/cli/cli_worker_multi_test.dart index 08378014..c2e0536b 100644 --- a/packages/stem_cli/test/unit/cli/cli_worker_multi_test.dart +++ b/packages/stem_cli/test/unit/cli/cli_worker_multi_test.dart @@ -55,7 +55,10 @@ void main() { Future expectLogNotEmpty(String node) async { final file = File(logTemplate.replaceAll('%n', node)); - final deadline = DateTime.now().add(const Duration(seconds: 2)); + // Detached Dart processes can take a few seconds to start on a fresh + // container or a loaded CI runner. Keep the assertion strict, but give + // the child enough time to execute its first log write. + final deadline = DateTime.now().add(const Duration(seconds: 10)); while (file.existsSync() && file.lengthSync() == 0) { if (DateTime.now().isAfter(deadline)) { break; diff --git a/packages/stem_cli/test/unit/cli/cli_worker_stats_test.dart b/packages/stem_cli/test/unit/cli/cli_worker_stats_test.dart index e0c16460..b79a1f03 100644 --- a/packages/stem_cli/test/unit/cli/cli_worker_stats_test.dart +++ b/packages/stem_cli/test/unit/cli/cli_worker_stats_test.dart @@ -1,6 +1,7 @@ import 'dart:async'; import 'package:test/test.dart'; +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; import 'package:stem_cli/src/cli/cli_runner.dart'; diff --git a/packages/stem_cli/test/unit/cli/cli_worker_status_test.dart b/packages/stem_cli/test/unit/cli/cli_worker_status_test.dart index 3061e291..f93a9239 100644 --- a/packages/stem_cli/test/unit/cli/cli_worker_status_test.dart +++ b/packages/stem_cli/test/unit/cli/cli_worker_status_test.dart @@ -1,4 +1,5 @@ import 'package:test/test.dart'; +import 'package:stem/memory.dart'; import 'package:stem_cli/src/cli/cli_runner.dart'; import 'package:stem/stem.dart'; diff --git a/packages/stem_cli/test/unit/cli/cli_workflow_agent_help_test.dart b/packages/stem_cli/test/unit/cli/cli_workflow_agent_help_test.dart index 3d8da3bd..a7c8692b 100644 --- a/packages/stem_cli/test/unit/cli/cli_workflow_agent_help_test.dart +++ b/packages/stem_cli/test/unit/cli/cli_workflow_agent_help_test.dart @@ -1,4 +1,5 @@ import 'package:test/test.dart'; +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; import 'package:stem_cli/src/cli/cli_runner.dart'; import 'package:stem_cli/src/cli/dependencies.dart'; diff --git a/packages/stem_cli/test/unit/cli/cli_workflow_test.dart b/packages/stem_cli/test/unit/cli/cli_workflow_test.dart index 6e9bc055..bb01a751 100644 --- a/packages/stem_cli/test/unit/cli/cli_workflow_test.dart +++ b/packages/stem_cli/test/unit/cli/cli_workflow_test.dart @@ -1,5 +1,6 @@ import 'dart:convert'; +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; import 'package:stem_cli/src/cli/cli_runner.dart'; import 'package:stem_cli/src/cli/workflow_context.dart'; diff --git a/packages/stem_cli/test/unit/cli/dlq_cli_test.dart b/packages/stem_cli/test/unit/cli/dlq_cli_test.dart index 2bfd6c8b..0d9c9557 100644 --- a/packages/stem_cli/test/unit/cli/dlq_cli_test.dart +++ b/packages/stem_cli/test/unit/cli/dlq_cli_test.dart @@ -1,6 +1,7 @@ import 'dart:async'; import 'package:test/test.dart'; +import 'package:stem/memory.dart'; import 'package:stem_cli/src/cli/cli_runner.dart'; import 'package:stem/stem.dart'; diff --git a/packages/stem_cli/test/unit/cli/schedule_context_test.dart b/packages/stem_cli/test/unit/cli/schedule_context_test.dart index 51a4d061..037cac3c 100644 --- a/packages/stem_cli/test/unit/cli/schedule_context_test.dart +++ b/packages/stem_cli/test/unit/cli/schedule_context_test.dart @@ -1,3 +1,4 @@ +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; import 'package:stem_cli/src/cli/cli_runner.dart'; import 'package:stem_cli/src/cli/dependencies.dart'; diff --git a/packages/stem_flutter/CHANGELOG.md b/packages/stem_flutter/CHANGELOG.md index 703c9b53..fed0c9f5 100644 --- a/packages/stem_flutter/CHANGELOG.md +++ b/packages/stem_flutter/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.2.0 + +- Updated the Flutter integration for Stem 0.3.0 and explicit worker startup. + ## 0.1.0 - Initial Flutter integration package for Stem. diff --git a/packages/stem_flutter/README.md b/packages/stem_flutter/README.md index 782e0450..9b2d1cf2 100644 --- a/packages/stem_flutter/README.md +++ b/packages/stem_flutter/README.md @@ -36,7 +36,7 @@ Those decisions are intentionally left to the app and the adapter package. ```yaml dependencies: - stem_flutter: ^0.1.0 + stem_flutter: ^0.2.0 ``` ## Minimal Usage diff --git a/packages/stem_flutter/analysis_options.yaml b/packages/stem_flutter/analysis_options.yaml index 8086495a..fe71c58d 100644 --- a/packages/stem_flutter/analysis_options.yaml +++ b/packages/stem_flutter/analysis_options.yaml @@ -4,3 +4,10 @@ analyzer: exclude: - "**/*.g.dart" - "**/*.orm.dart" + - build/** + - android/** + - ios/** + - web/** + - windows/** + - macos/** + - linux/** diff --git a/packages/stem_flutter/pubspec.yaml b/packages/stem_flutter/pubspec.yaml index 3a85de75..eabe034d 100644 --- a/packages/stem_flutter/pubspec.yaml +++ b/packages/stem_flutter/pubspec.yaml @@ -1,6 +1,6 @@ name: stem_flutter description: Flutter integration helpers for mobile-friendly Stem runtimes. -version: 0.1.0 +version: 0.2.0 repository: https://github.com/kingwill101/stem/tree/master/packages/stem_flutter issue_tracker: https://github.com/kingwill101/stem/issues resolution: workspace @@ -11,7 +11,7 @@ environment: dependencies: flutter: sdk: flutter - stem: ">=0.2.1 <0.3.0" + stem: ">=0.3.0 <0.4.0" time_machine2: ^0.13.1 dev_dependencies: diff --git a/packages/stem_flutter/test/stem_flutter_queue_monitor_test.dart b/packages/stem_flutter/test/stem_flutter_queue_monitor_test.dart index 85dfcf1b..ebac23b8 100644 --- a/packages/stem_flutter/test/stem_flutter_queue_monitor_test.dart +++ b/packages/stem_flutter/test/stem_flutter_queue_monitor_test.dart @@ -25,6 +25,12 @@ class _FakeBroker implements Broker { @override bool get supportsPriority => false; + BrokerCapabilities get capabilities => const BrokerCapabilities( + supportsDelayedDelivery: false, + supportsPriorityOrdering: false, + supportsQueueInspection: true, + ); + @override dynamic noSuchMethod(Invocation invocation) => super.noSuchMethod(invocation); } diff --git a/packages/stem_flutter_sqlite/CHANGELOG.md b/packages/stem_flutter_sqlite/CHANGELOG.md index 4d899bff..eee4869c 100644 --- a/packages/stem_flutter_sqlite/CHANGELOG.md +++ b/packages/stem_flutter_sqlite/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.2.0 + +- Updated the Flutter SQLite integration for the Stem 0.3.0 adapter line. + ## 0.1.0 - Initial Flutter SQLite adapter helpers for Stem. diff --git a/packages/stem_flutter_sqlite/README.md b/packages/stem_flutter_sqlite/README.md index 6e962926..0b82c8b3 100644 --- a/packages/stem_flutter_sqlite/README.md +++ b/packages/stem_flutter_sqlite/README.md @@ -18,7 +18,7 @@ with Stem and SQLite. ```yaml dependencies: - stem_flutter_sqlite: ^0.1.0 + stem_flutter_sqlite: ^0.2.0 ``` `stem_flutter_sqlite` depends on `stem_flutter`, `stem_sqlite`, and diff --git a/packages/stem_flutter_sqlite/analysis_options.yaml b/packages/stem_flutter_sqlite/analysis_options.yaml index 9df80aa4..defeac56 100644 --- a/packages/stem_flutter_sqlite/analysis_options.yaml +++ b/packages/stem_flutter_sqlite/analysis_options.yaml @@ -1 +1,10 @@ +analyzer: + exclude: + - build/** + - android/** + - ios/** + - web/** + - windows/** + - macos/** + - linux/** include: package:very_good_analysis/analysis_options.yaml diff --git a/packages/stem_flutter_sqlite/pubspec.yaml b/packages/stem_flutter_sqlite/pubspec.yaml index 10226658..6a7c7b4d 100644 --- a/packages/stem_flutter_sqlite/pubspec.yaml +++ b/packages/stem_flutter_sqlite/pubspec.yaml @@ -1,6 +1,6 @@ name: stem_flutter_sqlite description: SQLite adapter helpers for Flutter-hosted Stem runtimes. -version: 0.1.0 +version: 0.2.0 repository: https://github.com/kingwill101/stem/tree/master/packages/stem_flutter_sqlite issue_tracker: https://github.com/kingwill101/stem/issues resolution: workspace @@ -12,9 +12,9 @@ dependencies: flutter: sdk: flutter path_provider: ^2.1.5 - stem: ">=0.2.1 <0.3.0" - stem_flutter: ">=0.1.0 <0.2.0" - stem_sqlite: ">=0.1.3 <0.2.0" + stem: ">=0.3.0 <0.4.0" + stem_flutter: ">=0.2.0 <0.3.0" + stem_sqlite: ">=0.2.0 <0.3.0" dev_dependencies: flutter_test: diff --git a/packages/stem_memory/CHANGELOG.md b/packages/stem_memory/CHANGELOG.md index d696229b..1d669cbd 100644 --- a/packages/stem_memory/CHANGELOG.md +++ b/packages/stem_memory/CHANGELOG.md @@ -1,5 +1,22 @@ # Changelog +## 0.2.0 + +- Updated the compatibility package for Stem 0.3.0 and the core-owned memory + entrypoint. + +## 0.1.2 + +- Kept the compatibility package aligned with the core-owned + `package:stem/memory.dart` implementation after the adapter layering fix. + +## 0.1.1 + +- Made the package an explicit compatibility export for + `package:stem/memory.dart`. +- Raised the minimum Stem version so the package cannot resolve against a + published core artifact that does not expose the memory library. + ## 0.1.0 - Updated the in-memory workflow store to honor caller-provided run ids, diff --git a/packages/stem_memory/lib/stem_memory.dart b/packages/stem_memory/lib/stem_memory.dart index 418a7a2f..fdf2fded 100644 --- a/packages/stem_memory/lib/stem_memory.dart +++ b/packages/stem_memory/lib/stem_memory.dart @@ -1,21 +1,4 @@ -/// In-memory adapters and factories for Stem. +/// Compatibility package for Stem's in-memory adapters and factories. library; -export 'package:stem/src/bootstrap/stem_stack.dart' show StemMemoryAdapter; -export 'src/backend/in_memory_backend.dart' show InMemoryResultBackend; -export 'src/brokers/in_memory_broker.dart' show InMemoryBroker; -export 'src/control/in_memory_revoke_store.dart' show InMemoryRevokeStore; -export 'src/memory_factories.dart' - show - memoryBrokerFactory, - memoryEventBusFactory, - memoryLockStoreFactory, - memoryResultBackendFactory, - memoryRevokeStoreFactory, - memoryScheduleStoreFactory, - memoryWorkflowStoreFactory; -export 'src/scheduler/in_memory_lock_store.dart' show InMemoryLockStore; -export 'src/scheduler/in_memory_schedule_store.dart' show InMemoryScheduleStore; -export 'src/workflow/event_bus/in_memory_event_bus.dart' show InMemoryEventBus; -export 'src/workflow/store/in_memory_workflow_store.dart' - show InMemoryWorkflowStore; +export 'package:stem/memory.dart'; diff --git a/packages/stem_memory/pubspec.yaml b/packages/stem_memory/pubspec.yaml index edea5267..bd4fe304 100644 --- a/packages/stem_memory/pubspec.yaml +++ b/packages/stem_memory/pubspec.yaml @@ -1,6 +1,6 @@ name: stem_memory description: In-memory broker, backend, workflow, and scheduler adapters for Stem. -version: 0.1.0 +version: 0.2.0 repository: https://github.com/kingwill101/stem resolution: workspace environment: @@ -8,11 +8,11 @@ environment: dependencies: collection: ^1.19.1 - stem: ">=0.2.0-dev <0.3.0" + stem: ">=0.3.0 <0.4.0" uuid: ^4.5.2 dev_dependencies: coverage: ^1.15.0 - stem_adapter_tests: ^0.1.1 + stem_adapter_tests: ^0.2.0 test: ^1.29.0 very_good_analysis: ^10.0.0 diff --git a/packages/stem_memory/test/unit/memory_factories_test.dart b/packages/stem_memory/test/unit/memory_factories_test.dart index 25ca02a6..bc9aa197 100644 --- a/packages/stem_memory/test/unit/memory_factories_test.dart +++ b/packages/stem_memory/test/unit/memory_factories_test.dart @@ -1,5 +1,5 @@ +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; -import 'package:stem_memory/stem_memory.dart'; import 'package:test/test.dart'; void main() { @@ -46,7 +46,10 @@ void main() { final store = await factory.create(); expect(store, isA()); - final runId = await store.createRun(workflow: 'wf', params: const {}); + final runId = await store.createRun( + workflow: 'wf', + params: const {}, + ); expect(runId, isNotEmpty); }); @@ -97,6 +100,11 @@ void main() { class _TrackingBroker implements Broker { bool closed = false; + BrokerCapabilities get capabilities => const BrokerCapabilities( + supportsDelayedDelivery: false, + supportsPriorityOrdering: false, + ); + @override Future close() async { closed = true; diff --git a/packages/stem_postgres/CHANGELOG.md b/packages/stem_postgres/CHANGELOG.md index 99b7ba1a..f81ae400 100644 --- a/packages/stem_postgres/CHANGELOG.md +++ b/packages/stem_postgres/CHANGELOG.md @@ -1,5 +1,38 @@ # Changelog +## Unreleased + +- Datasource logger injection now uses Stem's dependency-neutral `StemLogger` + facade; the Ormed contextual logger remains an adapter implementation detail. +- Added conditional terminal-result updates so late completion attempts cannot + overwrite an existing terminal state. + +## 0.2.0 + +- Updated the Postgres adapter for Stem 0.3.0 and the capability-aware broker + contract. +- Added the transactional outbox, distributed rate limiter, migration registry, + and historical upgrade coverage. +- Added durable PostgreSQL lock fencing tokens with an additive migration and + row-locked acquisition semantics. +- Historical upgrade coverage verifies that current lock acquisition respects + active legacy-schema locks before taking over only after expiry. + +## 0.1.3 + +- Removed the prerelease core dependency range from the published-package + manifest. +- Added a PostgreSQL-backed distributed token-bucket `PostgresRateLimiter` + with server-clock refill and transactional row locking. +- Transactional outbox wrapping and relay now accept the narrow `QueueBroker` + contract, so queue-only adapters do not need to implement legacy inspection + and dead-letter methods. + +## 0.1.2 + +- Added a PostgreSQL transactional outbox for atomic application writes and + task publication records, with leased at-least-once dispatch. + ## 0.1.1 - Updated Ormed dependencies to 0.2.0 for the Postgres adapter stack. diff --git a/packages/stem_postgres/README.md b/packages/stem_postgres/README.md index 984e13e2..93558305 100644 --- a/packages/stem_postgres/README.md +++ b/packages/stem_postgres/README.md @@ -55,6 +55,67 @@ Future main() async { } ``` +### Transactional outbox + +Use `PostgresTransactionalOutbox` when a task must be published atomically +with application data in PostgreSQL. The outbox facade is producer-only: +enqueue through it inside `outbox.transaction`, and pass the underlying broker +to the relay. + +```dart +final outbox = await PostgresTransactionalOutbox.connect( + 'postgresql://postgres:postgres@localhost:5432/stem', +); +final broker = await PostgresBroker.connect( + 'postgresql://postgres:postgres@localhost:5432/stem', +); +final producerBroker = outbox.wrap(broker); // accepts any QueueBroker +final stem = Stem(broker: producerBroker, registry: registry); + +await outbox.transaction((transaction) async { + await transaction.context.table('orders').create({ + 'id': orderId, + 'state': 'created', + }); + await stem.enqueue('orders.process', args: {'id': orderId}); +}); + +await outbox.dispatch(broker: broker); + +await broker.close(); +await outbox.close(); +``` + +The relay is at least once. A crash after broker publication and before the +outbox row is marked dispatched can publish the same envelope again. Stem's +Postgres broker deduplicates queue rows by envelope ID, but task handlers and +external side effects must still be idempotent. Run migrations when opening +the outbox, or apply the `stem_task_outbox` migration as part of your normal +deployment process. The transaction boundary covers application writes and +broker publication records; result-backend status writes and unique-task +claims remain separate stores and should not be treated as part of the same +database commit unless they are made transaction-aware by the application. + +### Distributed rate limiting + +`PostgresRateLimiter` shares a token bucket across worker processes. Refill +uses PostgreSQL server time, and each acquire locks and updates one bucket row +inside a transaction. + +```dart +final limiter = await PostgresRateLimiter.connect( + 'postgresql://postgres:postgres@localhost:5432/stem', + namespace: 'billing-worker', +); + +final workerConfig = StemWorkerConfig(rateLimiter: limiter); +``` + +Opening the limiter runs the package migrations, including the +`stem_rate_limit_buckets` table. A denied acquisition includes `retryAfter` so +the worker can schedule the next attempt. Close the limiter with the worker's +other resources. + ### Typed `TaskDefinition` ```dart diff --git a/packages/stem_postgres/lib/src/backend/postgres_backend.dart b/packages/stem_postgres/lib/src/backend/postgres_backend.dart index c833ae95..b9e0086e 100644 --- a/packages/stem_postgres/lib/src/backend/postgres_backend.dart +++ b/packages/stem_postgres/lib/src/backend/postgres_backend.dart @@ -6,7 +6,8 @@ import 'package:stem_postgres/src/connection.dart'; import 'package:stem_postgres/src/database/models/models.dart'; /// PostgreSQL-backed implementation of [ResultBackend]. -class PostgresResultBackend implements ResultBackend { +class PostgresResultBackend + implements ResultBackend, AtomicTerminalResultBackend { PostgresResultBackend._( this._connections, { required this.namespace, @@ -62,6 +63,9 @@ class PostgresResultBackend implements ResultBackend { Timer? _cleanupTimer; bool _closed = false; + @override + bool get supportsAtomicTerminalWrites => true; + /// Connects to a PostgreSQL database and initializes the required tables. /// /// The [connectionString] should be in the format: @@ -186,6 +190,41 @@ class PostgresResultBackend implements ResultBackend { _watchers[taskId]?.add(status); } + @override + Future setTerminalIfAbsent( + TaskStatus status, { + Duration? ttl, + }) async { + final now = stemNow().toUtc(); + final expiresAt = now.add(ttl ?? defaultTtl); + final updated = await _connections.runInTransaction((txn) { + final query = txn + .query() + .whereEquals('id', status.id) + .whereEquals('namespace', namespace) + .where('expiresAt', now, PredicateOperator.greaterThan) + .where( + 'state', + const ['succeeded', 'failed', 'cancelled'], + PredicateOperator.notInValues, + ); + return query.update({ + 'state': status.state.name, + 'payload': status.payload, + 'error': status.error?.toJson(), + 'attempt': status.attempt, + 'meta': status.meta, + 'expiresAt': expiresAt, + 'updatedAt': now, + }); + }); + final applied = updated > 0; + if (applied) { + _watchers[status.id]?.add(status); + } + return applied; + } + @override Future get(String taskId) async { final now = stemNow(); diff --git a/packages/stem_postgres/lib/src/brokers/postgres_broker.dart b/packages/stem_postgres/lib/src/brokers/postgres_broker.dart index 3b759649..cfa2328f 100644 --- a/packages/stem_postgres/lib/src/brokers/postgres_broker.dart +++ b/packages/stem_postgres/lib/src/brokers/postgres_broker.dart @@ -1,15 +1,21 @@ import 'dart:async'; import 'dart:convert'; -import 'package:contextual/contextual.dart'; import 'package:ormed/ormed.dart'; +import 'package:stem/observability.dart' show stemLogger; import 'package:stem/stem.dart'; import 'package:stem_postgres/src/connection.dart'; import 'package:stem_postgres/src/database/models/models.dart'; import 'package:uuid/uuid.dart'; /// PostgreSQL-backed implementation of [Broker]. -class PostgresBroker implements Broker { +class PostgresBroker + implements + Broker, + LeaseBroker, + InspectableBroker, + DeadLetterBroker, + BrokerCapabilitiesProvider { PostgresBroker._( this._connections, { required this.namespace, @@ -20,7 +26,7 @@ class PostgresBroker implements Broker { }) { stemLogger.info( 'PostgresBroker created (namespace=$namespace)', - _logContext(), + fields: _logContext(), ); _startSweeper(); } @@ -118,7 +124,7 @@ class PostgresBroker implements Broker { stemLogger.warning( 'Closing PostgresBroker (namespace=$namespace) ' 'stack=${_closedStack ?? ''}', - _logContext({'stack': _closedStack.toString()}), + fields: _logContext({'stack': _closedStack.toString()}), ); _closed = true; _sweeperTimer?.cancel(); @@ -142,6 +148,17 @@ class PostgresBroker implements Broker { @override bool get supportsPriority => true; + @override + BrokerCapabilities get capabilities => const BrokerCapabilities( + supportsDelayedDelivery: true, + supportsPriorityOrdering: true, + deliveryGuarantee: BrokerDeliveryGuarantee.atLeastOnce, + supportsQueueInspection: true, + supportsLeaseExtension: true, + supportsDeadLettering: true, + supportsDeadLetterReplay: true, + ); + Future _withDb(Future Function() action) { final run = _dbLock.then((_) async { await _connections.ensureReady(); @@ -219,7 +236,7 @@ class PostgresBroker implements Broker { stemLogger.debug( 'Broker consume requested (namespace=$namespace, ' 'queues=${subscription.queues})', - _logContext({'queues': subscription.queues}), + fields: _logContext({'queues': subscription.queues}), ); if (subscription.queues.length > 1) { throw UnsupportedError( @@ -249,7 +266,7 @@ class PostgresBroker implements Broker { final queueLabel = queue ?? ''; stemLogger.debug( 'Consumer stream canceled (queue=$queueLabel, worker=$consumer)', - _logContext({ + fields: _logContext({ 'queue': queueLabel, 'worker': consumer, }), @@ -275,7 +292,9 @@ class PostgresBroker implements Broker { stemLogger.warning( 'Broker already closed; closing consumer stream. ' 'stack=${_closedStack ?? StackTrace.current}', - _logContext({'stack': (_closedStack ?? StackTrace.current).toString()}), + fields: _logContext({ + 'stack': (_closedStack ?? StackTrace.current).toString(), + }), ); scheduleMicrotask(() async { await controller.close(); @@ -293,7 +312,7 @@ class PostgresBroker implements Broker { final jobId = _parseReceipt(delivery.receipt); stemLogger.debug( 'Ack queue job $jobId (${delivery.envelope.queue})', - _logContext({ + fields: _logContext({ 'jobId': jobId, 'queue': delivery.envelope.queue, }), @@ -320,7 +339,7 @@ class PostgresBroker implements Broker { final jobId = _parseReceipt(delivery.receipt); stemLogger.debug( 'Nack queue job $jobId (${delivery.envelope.queue})', - _logContext({ + fields: _logContext({ 'jobId': jobId, 'queue': delivery.envelope.queue, }), @@ -643,7 +662,7 @@ class PostgresBroker implements Broker { if (updated == 0) return null; stemLogger.debug( 'Claimed queue job ${candidate.id} ($queue) by $consumerId', - _logContext({ + fields: _logContext({ 'jobId': candidate.id, 'queue': queue, 'worker': consumerId, @@ -749,13 +768,15 @@ class PostgresBroker implements Broker { }); } - Context _logContext([Map fields = const {}]) { - return Context({ + Map _logContext([ + Map fields = const {}, + ]) { + return { 'component': 'stem_postgres', 'subsystem': 'broker', 'namespace': namespace, ...fields, - }); + }; } String _parseReceipt(String receipt) => receipt; @@ -824,7 +845,7 @@ class _ConsumerRunner { final queueLabel = queue ?? ''; stemLogger.debug( 'Consumer runner started (queue=$queueLabel, worker=$workerId)', - broker._logContext({'queue': queueLabel, 'worker': workerId}), + fields: broker._logContext({'queue': queueLabel, 'worker': workerId}), ); unawaited(_loop()); } @@ -833,7 +854,7 @@ class _ConsumerRunner { final queueLabel = queue ?? ''; stemLogger.debug( 'Consumer runner stopped (queue=$queueLabel, worker=$workerId)', - broker._logContext({'queue': queueLabel, 'worker': workerId}), + fields: broker._logContext({'queue': queueLabel, 'worker': workerId}), ); _stopped = true; } @@ -880,7 +901,7 @@ class _ConsumerRunner { stemLogger.warning( 'Consumer loop error (queue=$queueLabel, worker=$workerId): ' '$error\n$stack', - broker._logContext({ + fields: broker._logContext({ 'queue': queueLabel, 'worker': workerId, 'error': error.toString(), diff --git a/packages/stem_postgres/lib/src/database/datasource.dart b/packages/stem_postgres/lib/src/database/datasource.dart index 0414c342..c845e98d 100644 --- a/packages/stem_postgres/lib/src/database/datasource.dart +++ b/packages/stem_postgres/lib/src/database/datasource.dart @@ -1,14 +1,14 @@ -import 'package:contextual/contextual.dart' as contextual; import 'package:ormed/ormed.dart'; import 'package:ormed_postgres/ormed_postgres.dart'; -import 'package:stem/stem.dart' show stemLogger; +import 'package:stem/observability.dart' show StemLogger, stemLogger; import 'package:stem_postgres/orm_registry.g.dart'; +import 'package:stem_postgres/src/database/stem_orm_logger.dart'; /// Creates a new DataSource instance using the project configuration. DataSource createDataSource({ String? connectionString, bool logging = false, - contextual.Logger? logger, + StemLogger? logger, }) { if (connectionString != null && connectionString.isNotEmpty) { final options = bootstrapOrm() @@ -16,7 +16,7 @@ DataSource createDataSource({ environment: {'DATABASE_URL': connectionString}, logging: logging, ) - .copyWith(logger: logger ?? stemLogger); + .copyWith(logger: createOrmLogger(logger ?? stemLogger)); return DataSource(options); } @@ -36,7 +36,7 @@ DataSource createDataSource({ /// Creates a new DataSource instance using a resolved ORM project config. DataSource createDataSourceFromConfig( OrmProjectConfig config, { - contextual.Logger? logger, + StemLogger? logger, }) { final registry = bootstrapOrm(); final options = Map.from(config.driver.options); @@ -84,5 +84,9 @@ DataSource createDataSourceFromConfig( options['schema']?.toString() ?? 'public', ); - return DataSource(dataSourceOptions.copyWith(logger: logger)); + return DataSource( + dataSourceOptions.copyWith( + logger: createOrmLogger(logger ?? stemLogger), + ), + ); } diff --git a/packages/stem_postgres/lib/src/database/migrations.dart b/packages/stem_postgres/lib/src/database/migrations.dart index 96cd063a..f346afdc 100644 --- a/packages/stem_postgres/lib/src/database/migrations.dart +++ b/packages/stem_postgres/lib/src/database/migrations.dart @@ -7,6 +7,9 @@ import 'package:ormed/migrations.dart'; import 'package:stem_postgres/src/database/migrations/m_20251227071920_stem.dart'; import 'package:stem_postgres/src/database/migrations/m_20251231160000_add_namespace_scoping.dart'; import 'package:stem_postgres/src/database/migrations/m_20260116121000_add_workflow_run_leases.dart'; // +import 'package:stem_postgres/src/database/migrations/m_20260819090000_add_task_outbox.dart'; +import 'package:stem_postgres/src/database/migrations/m_20260819100000_add_rate_limit_buckets.dart'; +import 'package:stem_postgres/src/database/migrations/m_20260820110000_add_lock_fencing_tokens.dart'; final List _entries = [ // @@ -28,6 +31,27 @@ final List _entries = [ ), migration: const AddWorkflowRunLeases(), ), + MigrationEntry( + id: MigrationId( + DateTime(2026, 8, 19, 9), + 'm_20260819090000_add_task_outbox', + ), + migration: const AddTaskOutbox(), + ), + MigrationEntry( + id: MigrationId( + DateTime(2026, 8, 19, 10), + 'm_20260819100000_add_rate_limit_buckets', + ), + migration: const AddRateLimitBuckets(), + ), + MigrationEntry( + id: MigrationId( + DateTime(2026, 8, 20, 11), + 'm_20260820110000_add_lock_fencing_tokens', + ), + migration: const AddLockFencingTokens(), + ), ]; /// Build migration descriptors sorted by timestamp. diff --git a/packages/stem_postgres/lib/src/database/migrations/m_20260819090000_add_task_outbox.dart b/packages/stem_postgres/lib/src/database/migrations/m_20260819090000_add_task_outbox.dart new file mode 100644 index 00000000..2b016bc0 --- /dev/null +++ b/packages/stem_postgres/lib/src/database/migrations/m_20260819090000_add_task_outbox.dart @@ -0,0 +1,40 @@ +import 'package:ormed/migrations.dart'; + +/// Adds the durable PostgreSQL task publication outbox. +class AddTaskOutbox extends Migration { + /// Creates the migration. + const AddTaskOutbox(); + + @override + void up(SchemaBuilder schema) { + schema.create('stem_task_outbox', (table) { + table + ..text('id').primaryKey() + ..text('namespace') + ..json('envelope') + ..json('routing').nullable() + ..text('status').defaultValue('pending') + ..timestampTz('available_at') + ..integer('attempts').defaultValue(0) + ..timestampTz('locked_at').nullable() + ..timestampTz('locked_until').nullable() + ..text('locked_by').nullable() + ..text('last_error').nullable() + ..timestampTz('dispatched_at').nullable() + ..timestampsTz() + ..index( + ['namespace', 'status', 'available_at'], + name: 'stem_task_outbox_claim_idx', + ) + ..index( + ['locked_until'], + name: 'stem_task_outbox_locked_idx', + ); + }); + } + + @override + void down(SchemaBuilder schema) { + schema.drop('stem_task_outbox', ifExists: true); + } +} diff --git a/packages/stem_postgres/lib/src/database/migrations/m_20260819100000_add_rate_limit_buckets.dart b/packages/stem_postgres/lib/src/database/migrations/m_20260819100000_add_rate_limit_buckets.dart new file mode 100644 index 00000000..a721885e --- /dev/null +++ b/packages/stem_postgres/lib/src/database/migrations/m_20260819100000_add_rate_limit_buckets.dart @@ -0,0 +1,34 @@ +import 'package:ormed/migrations.dart'; + +/// Adds durable token-bucket state for the PostgreSQL rate limiter. +class AddRateLimitBuckets extends Migration { + /// Creates the migration. + const AddRateLimitBuckets(); + + @override + void up(SchemaBuilder schema) { + schema.create('stem_rate_limit_buckets', (table) { + table + ..text('namespace') + ..text('rate_key') + ..bigInteger('capacity') + ..bigInteger('interval_ms') + ..bigInteger('available_micros') + ..bigInteger('updated_at_ms') + ..timestampsTz() + ..primary( + ['namespace', 'rate_key'], + name: 'stem_rate_limit_buckets_primary', + ) + ..index( + ['namespace', 'updated_at_ms'], + name: 'stem_rate_limit_buckets_updated_idx', + ); + }); + } + + @override + void down(SchemaBuilder schema) { + schema.drop('stem_rate_limit_buckets', ifExists: true); + } +} diff --git a/packages/stem_postgres/lib/src/database/migrations/m_20260820110000_add_lock_fencing_tokens.dart b/packages/stem_postgres/lib/src/database/migrations/m_20260820110000_add_lock_fencing_tokens.dart new file mode 100644 index 00000000..beb35433 --- /dev/null +++ b/packages/stem_postgres/lib/src/database/migrations/m_20260820110000_add_lock_fencing_tokens.dart @@ -0,0 +1,21 @@ +import 'package:ormed/migrations.dart'; + +/// Adds durable fencing tokens to distributed lock records. +class AddLockFencingTokens extends Migration { + /// Creates the migration. + const AddLockFencingTokens(); + + @override + void up(SchemaBuilder schema) { + schema.table('stem_locks', (table) { + table.bigInteger('fencing_token').defaultValue(0); + }); + } + + @override + void down(SchemaBuilder schema) { + schema.table('stem_locks', (table) { + table.dropColumn('fencing_token'); + }); + } +} diff --git a/packages/stem_postgres/lib/src/database/models/stem_broadcast_ack.orm.dart b/packages/stem_postgres/lib/src/database/models/stem_broadcast_ack.orm.dart index 4ea9ff90..cbac863e 100644 --- a/packages/stem_postgres/lib/src/database/models/stem_broadcast_ack.orm.dart +++ b/packages/stem_postgres/lib/src/database/models/stem_broadcast_ack.orm.dart @@ -246,8 +246,7 @@ class StemBroadcastAckModelFactory { static ModelFactoryBuilder factory({ GeneratorProvider? generatorProvider, - }) => ModelFactoryBuilder( - definition: definition, + }) => ModelFactoryRegistry.factoryFor( generatorProvider: generatorProvider, ); } diff --git a/packages/stem_postgres/lib/src/database/models/stem_broadcast_message.orm.dart b/packages/stem_postgres/lib/src/database/models/stem_broadcast_message.orm.dart index 2ae6bb72..ed0b7d21 100644 --- a/packages/stem_postgres/lib/src/database/models/stem_broadcast_message.orm.dart +++ b/packages/stem_postgres/lib/src/database/models/stem_broadcast_message.orm.dart @@ -267,8 +267,7 @@ class StemBroadcastMessageModelFactory { static ModelFactoryBuilder factory({ GeneratorProvider? generatorProvider, - }) => ModelFactoryBuilder( - definition: definition, + }) => ModelFactoryRegistry.factoryFor( generatorProvider: generatorProvider, ); } diff --git a/packages/stem_postgres/lib/src/database/models/stem_dead_letter.orm.dart b/packages/stem_postgres/lib/src/database/models/stem_dead_letter.orm.dart index c45b3959..ea872ca5 100644 --- a/packages/stem_postgres/lib/src/database/models/stem_dead_letter.orm.dart +++ b/packages/stem_postgres/lib/src/database/models/stem_dead_letter.orm.dart @@ -256,8 +256,7 @@ class StemDeadLetterModelFactory { static ModelFactoryBuilder factory({ GeneratorProvider? generatorProvider, - }) => ModelFactoryBuilder( - definition: definition, + }) => ModelFactoryRegistry.factoryFor( generatorProvider: generatorProvider, ); } diff --git a/packages/stem_postgres/lib/src/database/models/stem_group.orm.dart b/packages/stem_postgres/lib/src/database/models/stem_group.orm.dart index 124ec2b8..18f0ed91 100644 --- a/packages/stem_postgres/lib/src/database/models/stem_group.orm.dart +++ b/packages/stem_postgres/lib/src/database/models/stem_group.orm.dart @@ -240,8 +240,7 @@ class StemGroupModelFactory { static ModelFactoryBuilder factory({ GeneratorProvider? generatorProvider, - }) => ModelFactoryBuilder( - definition: definition, + }) => ModelFactoryRegistry.factoryFor( generatorProvider: generatorProvider, ); } diff --git a/packages/stem_postgres/lib/src/database/models/stem_group_result.orm.dart b/packages/stem_postgres/lib/src/database/models/stem_group_result.orm.dart index c8ed9fe2..7609dbf0 100644 --- a/packages/stem_postgres/lib/src/database/models/stem_group_result.orm.dart +++ b/packages/stem_postgres/lib/src/database/models/stem_group_result.orm.dart @@ -305,8 +305,7 @@ class StemGroupResultModelFactory { static ModelFactoryBuilder factory({ GeneratorProvider? generatorProvider, - }) => ModelFactoryBuilder( - definition: definition, + }) => ModelFactoryRegistry.factoryFor( generatorProvider: generatorProvider, ); } diff --git a/packages/stem_postgres/lib/src/database/models/stem_lock.dart b/packages/stem_postgres/lib/src/database/models/stem_lock.dart index 832ff772..d5433305 100644 --- a/packages/stem_postgres/lib/src/database/models/stem_lock.dart +++ b/packages/stem_postgres/lib/src/database/models/stem_lock.dart @@ -10,6 +10,7 @@ class StemLock extends Model { required this.key, required this.namespace, required this.owner, + required this.fencingToken, required this.expiresAt, required this.createdAt, }); @@ -26,6 +27,10 @@ class StemLock extends Model { @OrmField(columnName: 'owner') final String owner; + /// Monotonically increasing token for this lock acquisition. + @OrmField(columnName: 'fencing_token') + final int fencingToken; + /// Timestamp when the lock expires. @OrmField(columnName: 'expires_at') final DateTime expiresAt; diff --git a/packages/stem_postgres/lib/src/database/models/stem_lock.orm.dart b/packages/stem_postgres/lib/src/database/models/stem_lock.orm.dart index 1a85cb9a..a57b4238 100644 --- a/packages/stem_postgres/lib/src/database/models/stem_lock.orm.dart +++ b/packages/stem_postgres/lib/src/database/models/stem_lock.orm.dart @@ -43,6 +43,18 @@ const FieldDefinition _$StemLockOwnerField = FieldDefinition( autoIncrement: false, ); +const FieldDefinition _$StemLockFencingTokenField = FieldDefinition( + name: 'fencingToken', + columnName: 'fencing_token', + dartType: 'int', + resolvedType: 'int', + isPrimaryKey: false, + isNullable: false, + isUnique: false, + isIndexed: false, + autoIncrement: false, +); + const FieldDefinition _$StemLockExpiresAtField = FieldDefinition( name: 'expiresAt', columnName: 'expires_at', @@ -76,6 +88,10 @@ Map _encodeStemLockUntracked( 'key': registry.encodeField(_$StemLockKeyField, m.key), 'namespace': registry.encodeField(_$StemLockNamespaceField, m.namespace), 'owner': registry.encodeField(_$StemLockOwnerField, m.owner), + 'fencing_token': registry.encodeField( + _$StemLockFencingTokenField, + m.fencingToken, + ), 'expires_at': registry.encodeField(_$StemLockExpiresAtField, m.expiresAt), 'created_at': registry.encodeField(_$StemLockCreatedAtField, m.createdAt), }; @@ -88,6 +104,7 @@ final ModelDefinition<$StemLock> _$StemLockDefinition = ModelDefinition( _$StemLockKeyField, _$StemLockNamespaceField, _$StemLockOwnerField, + _$StemLockFencingTokenField, _$StemLockExpiresAtField, _$StemLockCreatedAtField, ], @@ -211,8 +228,7 @@ class StemLockModelFactory { static ModelFactoryBuilder factory({ GeneratorProvider? generatorProvider, - }) => ModelFactoryBuilder( - definition: definition, + }) => ModelFactoryRegistry.factoryFor( generatorProvider: generatorProvider, ); } @@ -228,6 +244,10 @@ class _$StemLockCodec extends ModelCodec<$StemLock> { model.namespace, ), 'owner': registry.encodeField(_$StemLockOwnerField, model.owner), + 'fencing_token': registry.encodeField( + _$StemLockFencingTokenField, + model.fencingToken, + ), 'expires_at': registry.encodeField( _$StemLockExpiresAtField, model.expiresAt, @@ -253,6 +273,12 @@ class _$StemLockCodec extends ModelCodec<$StemLock> { final String stemLockOwnerValue = registry.decodeField(_$StemLockOwnerField, data['owner']) ?? (throw StateError('Field owner on StemLock cannot be null.')); + final int stemLockFencingTokenValue = + registry.decodeField( + _$StemLockFencingTokenField, + data['fencing_token'], + ) ?? + (throw StateError('Field fencingToken on StemLock cannot be null.')); final DateTime stemLockExpiresAtValue = registry.decodeField( _$StemLockExpiresAtField, @@ -269,6 +295,7 @@ class _$StemLockCodec extends ModelCodec<$StemLock> { key: stemLockKeyValue, namespace: stemLockNamespaceValue, owner: stemLockOwnerValue, + fencingToken: stemLockFencingTokenValue, expiresAt: stemLockExpiresAtValue, createdAt: stemLockCreatedAtValue, ); @@ -276,6 +303,7 @@ class _$StemLockCodec extends ModelCodec<$StemLock> { 'key': stemLockKeyValue, 'namespace': stemLockNamespaceValue, 'owner': stemLockOwnerValue, + 'fencing_token': stemLockFencingTokenValue, 'expires_at': stemLockExpiresAtValue, 'created_at': stemLockCreatedAtValue, }); @@ -291,12 +319,14 @@ class StemLockInsertDto implements InsertDto<$StemLock> { this.key, this.namespace, this.owner, + this.fencingToken, this.expiresAt, this.createdAt, }); final String? key; final String? namespace; final String? owner; + final int? fencingToken; final DateTime? expiresAt; final DateTime? createdAt; @@ -306,6 +336,7 @@ class StemLockInsertDto implements InsertDto<$StemLock> { if (key != null) 'key': key, if (namespace != null) 'namespace': namespace, if (owner != null) 'owner': owner, + if (fencingToken != null) 'fencing_token': fencingToken, if (expiresAt != null) 'expires_at': expiresAt, if (createdAt != null) 'created_at': createdAt, }; @@ -317,6 +348,7 @@ class StemLockInsertDto implements InsertDto<$StemLock> { Object? key = _copyWithSentinel, Object? namespace = _copyWithSentinel, Object? owner = _copyWithSentinel, + Object? fencingToken = _copyWithSentinel, Object? expiresAt = _copyWithSentinel, Object? createdAt = _copyWithSentinel, }) { @@ -328,6 +360,9 @@ class StemLockInsertDto implements InsertDto<$StemLock> { owner: identical(owner, _copyWithSentinel) ? this.owner : owner as String?, + fencingToken: identical(fencingToken, _copyWithSentinel) + ? this.fencingToken + : fencingToken as int?, expiresAt: identical(expiresAt, _copyWithSentinel) ? this.expiresAt : expiresAt as DateTime?, @@ -350,12 +385,14 @@ class StemLockUpdateDto implements UpdateDto<$StemLock> { this.key, this.namespace, this.owner, + this.fencingToken, this.expiresAt, this.createdAt, }); final String? key; final String? namespace; final String? owner; + final int? fencingToken; final DateTime? expiresAt; final DateTime? createdAt; @@ -365,6 +402,7 @@ class StemLockUpdateDto implements UpdateDto<$StemLock> { if (key != null) 'key': key, if (namespace != null) 'namespace': namespace, if (owner != null) 'owner': owner, + if (fencingToken != null) 'fencing_token': fencingToken, if (expiresAt != null) 'expires_at': expiresAt, if (createdAt != null) 'created_at': createdAt, }; @@ -376,6 +414,7 @@ class StemLockUpdateDto implements UpdateDto<$StemLock> { Object? key = _copyWithSentinel, Object? namespace = _copyWithSentinel, Object? owner = _copyWithSentinel, + Object? fencingToken = _copyWithSentinel, Object? expiresAt = _copyWithSentinel, Object? createdAt = _copyWithSentinel, }) { @@ -387,6 +426,9 @@ class StemLockUpdateDto implements UpdateDto<$StemLock> { owner: identical(owner, _copyWithSentinel) ? this.owner : owner as String?, + fencingToken: identical(fencingToken, _copyWithSentinel) + ? this.fencingToken + : fencingToken as int?, expiresAt: identical(expiresAt, _copyWithSentinel) ? this.expiresAt : expiresAt as DateTime?, @@ -409,6 +451,7 @@ class StemLockPartial implements PartialEntity<$StemLock> { this.key, this.namespace, this.owner, + this.fencingToken, this.expiresAt, this.createdAt, }); @@ -422,6 +465,7 @@ class StemLockPartial implements PartialEntity<$StemLock> { key: row['key'] as String?, namespace: row['namespace'] as String?, owner: row['owner'] as String?, + fencingToken: row['fencing_token'] as int?, expiresAt: row['expires_at'] as DateTime?, createdAt: row['created_at'] as DateTime?, ); @@ -430,6 +474,7 @@ class StemLockPartial implements PartialEntity<$StemLock> { final String? key; final String? namespace; final String? owner; + final int? fencingToken; final DateTime? expiresAt; final DateTime? createdAt; @@ -448,6 +493,10 @@ class StemLockPartial implements PartialEntity<$StemLock> { if (ownerValue == null) { throw StateError('Missing required field: owner'); } + final int? fencingTokenValue = fencingToken; + if (fencingTokenValue == null) { + throw StateError('Missing required field: fencingToken'); + } final DateTime? expiresAtValue = expiresAt; if (expiresAtValue == null) { throw StateError('Missing required field: expiresAt'); @@ -460,6 +509,7 @@ class StemLockPartial implements PartialEntity<$StemLock> { key: keyValue, namespace: namespaceValue, owner: ownerValue, + fencingToken: fencingTokenValue, expiresAt: expiresAtValue, createdAt: createdAtValue, ); @@ -471,6 +521,7 @@ class StemLockPartial implements PartialEntity<$StemLock> { if (key != null) 'key': key, if (namespace != null) 'namespace': namespace, if (owner != null) 'owner': owner, + if (fencingToken != null) 'fencing_token': fencingToken, if (expiresAt != null) 'expires_at': expiresAt, if (createdAt != null) 'created_at': createdAt, }; @@ -482,6 +533,7 @@ class StemLockPartial implements PartialEntity<$StemLock> { Object? key = _copyWithSentinel, Object? namespace = _copyWithSentinel, Object? owner = _copyWithSentinel, + Object? fencingToken = _copyWithSentinel, Object? expiresAt = _copyWithSentinel, Object? createdAt = _copyWithSentinel, }) { @@ -493,6 +545,9 @@ class StemLockPartial implements PartialEntity<$StemLock> { owner: identical(owner, _copyWithSentinel) ? this.owner : owner as String?, + fencingToken: identical(fencingToken, _copyWithSentinel) + ? this.fencingToken + : fencingToken as int?, expiresAt: identical(expiresAt, _copyWithSentinel) ? this.expiresAt : expiresAt as DateTime?, @@ -521,12 +576,14 @@ class $StemLock extends StemLock with ModelAttributes implements OrmEntity { required String key, required String namespace, required String owner, + required int fencingToken, required DateTime expiresAt, required DateTime createdAt, }) : super( key: key, namespace: namespace, owner: owner, + fencingToken: fencingToken, expiresAt: expiresAt, createdAt: createdAt, ) { @@ -534,6 +591,7 @@ class $StemLock extends StemLock with ModelAttributes implements OrmEntity { 'key': key, 'namespace': namespace, 'owner': owner, + 'fencing_token': fencingToken, 'expires_at': expiresAt, 'created_at': createdAt, }); @@ -545,6 +603,7 @@ class $StemLock extends StemLock with ModelAttributes implements OrmEntity { key: model.key, namespace: model.namespace, owner: model.owner, + fencingToken: model.fencingToken, expiresAt: model.expiresAt, createdAt: model.createdAt, ); @@ -554,6 +613,7 @@ class $StemLock extends StemLock with ModelAttributes implements OrmEntity { String? key, String? namespace, String? owner, + int? fencingToken, DateTime? expiresAt, DateTime? createdAt, }) { @@ -561,6 +621,7 @@ class $StemLock extends StemLock with ModelAttributes implements OrmEntity { key: key ?? this.key, namespace: namespace ?? this.namespace, owner: owner ?? this.owner, + fencingToken: fencingToken ?? this.fencingToken, expiresAt: expiresAt ?? this.expiresAt, createdAt: createdAt ?? this.createdAt, ); @@ -597,6 +658,14 @@ class $StemLock extends StemLock with ModelAttributes implements OrmEntity { /// Tracked setter for [owner]. set owner(String value) => setAttribute('owner', value); + /// Tracked getter for [fencingToken]. + @override + int get fencingToken => + getAttribute('fencing_token') ?? super.fencingToken; + + /// Tracked setter for [fencingToken]. + set fencingToken(int value) => setAttribute('fencing_token', value); + /// Tracked getter for [expiresAt]. @override DateTime get expiresAt => @@ -630,6 +699,7 @@ extension StemLockOrmExtension on StemLock { Object? key = _copyWithSentinel, Object? namespace = _copyWithSentinel, Object? owner = _copyWithSentinel, + Object? fencingToken = _copyWithSentinel, Object? expiresAt = _copyWithSentinel, Object? createdAt = _copyWithSentinel, }) { @@ -639,6 +709,9 @@ extension StemLockOrmExtension on StemLock { ? this.namespace : namespace as String, owner: identical(owner, _copyWithSentinel) ? this.owner : owner as String, + fencingToken: identical(fencingToken, _copyWithSentinel) + ? this.fencingToken + : fencingToken as int, expiresAt: identical(expiresAt, _copyWithSentinel) ? this.expiresAt : expiresAt as DateTime, @@ -678,6 +751,8 @@ extension StemLockPredicateFields on PredicateBuilder { PredicateField(this, 'namespace'); PredicateField get owner => PredicateField(this, 'owner'); + PredicateField get fencingToken => + PredicateField(this, 'fencingToken'); PredicateField get expiresAt => PredicateField(this, 'expiresAt'); PredicateField get createdAt => diff --git a/packages/stem_postgres/lib/src/database/models/stem_queue_job.orm.dart b/packages/stem_postgres/lib/src/database/models/stem_queue_job.orm.dart index 5260c7f4..042d2bb5 100644 --- a/packages/stem_postgres/lib/src/database/models/stem_queue_job.orm.dart +++ b/packages/stem_postgres/lib/src/database/models/stem_queue_job.orm.dart @@ -342,8 +342,7 @@ class StemQueueJobModelFactory { static ModelFactoryBuilder factory({ GeneratorProvider? generatorProvider, - }) => ModelFactoryBuilder( - definition: definition, + }) => ModelFactoryRegistry.factoryFor( generatorProvider: generatorProvider, ); } diff --git a/packages/stem_postgres/lib/src/database/models/stem_revoke_entry.orm.dart b/packages/stem_postgres/lib/src/database/models/stem_revoke_entry.orm.dart index cc29d440..1543d64c 100644 --- a/packages/stem_postgres/lib/src/database/models/stem_revoke_entry.orm.dart +++ b/packages/stem_postgres/lib/src/database/models/stem_revoke_entry.orm.dart @@ -293,8 +293,7 @@ class StemRevokeEntryModelFactory { static ModelFactoryBuilder factory({ GeneratorProvider? generatorProvider, - }) => ModelFactoryBuilder( - definition: definition, + }) => ModelFactoryRegistry.factoryFor( generatorProvider: generatorProvider, ); } diff --git a/packages/stem_postgres/lib/src/database/models/stem_schedule_entry.orm.dart b/packages/stem_postgres/lib/src/database/models/stem_schedule_entry.orm.dart index 7aaa0470..932f43ca 100644 --- a/packages/stem_postgres/lib/src/database/models/stem_schedule_entry.orm.dart +++ b/packages/stem_postgres/lib/src/database/models/stem_schedule_entry.orm.dart @@ -513,8 +513,7 @@ class StemScheduleEntryModelFactory { static ModelFactoryBuilder factory({ GeneratorProvider? generatorProvider, - }) => ModelFactoryBuilder( - definition: definition, + }) => ModelFactoryRegistry.factoryFor( generatorProvider: generatorProvider, ); } @@ -1441,20 +1440,20 @@ class $StemScheduleEntry extends StemScheduleEntry required DateTime updatedAt, String? args, String? kwargs, - required bool enabled, + bool enabled = true, int? jitter, DateTime? lastRunAt, DateTime? nextRunAt, int? lastJitter, String? lastError, String? timezone, - required int totalRunCount, + int totalRunCount = 0, DateTime? lastSuccessAt, DateTime? lastErrorAt, int? drift, DateTime? expireAt, String? meta, - int? version, + int? version = 0, }) : super( id: id, namespace: namespace, diff --git a/packages/stem_postgres/lib/src/database/models/stem_task_result.orm.dart b/packages/stem_postgres/lib/src/database/models/stem_task_result.orm.dart index c55e6a2d..6dc41acd 100644 --- a/packages/stem_postgres/lib/src/database/models/stem_task_result.orm.dart +++ b/packages/stem_postgres/lib/src/database/models/stem_task_result.orm.dart @@ -301,8 +301,7 @@ class StemTaskResultModelFactory { static ModelFactoryBuilder factory({ GeneratorProvider? generatorProvider, - }) => ModelFactoryBuilder( - definition: definition, + }) => ModelFactoryRegistry.factoryFor( generatorProvider: generatorProvider, ); } diff --git a/packages/stem_postgres/lib/src/database/models/stem_worker_heartbeat.orm.dart b/packages/stem_postgres/lib/src/database/models/stem_worker_heartbeat.orm.dart index e9194430..31d97387 100644 --- a/packages/stem_postgres/lib/src/database/models/stem_worker_heartbeat.orm.dart +++ b/packages/stem_postgres/lib/src/database/models/stem_worker_heartbeat.orm.dart @@ -365,8 +365,7 @@ class StemWorkerHeartbeatModelFactory { static ModelFactoryBuilder factory({ GeneratorProvider? generatorProvider, - }) => ModelFactoryBuilder( - definition: definition, + }) => ModelFactoryRegistry.factoryFor( generatorProvider: generatorProvider, ); } diff --git a/packages/stem_postgres/lib/src/database/models/stem_workflow_run.orm.dart b/packages/stem_postgres/lib/src/database/models/stem_workflow_run.orm.dart index 0c1090d7..ca171867 100644 --- a/packages/stem_postgres/lib/src/database/models/stem_workflow_run.orm.dart +++ b/packages/stem_postgres/lib/src/database/models/stem_workflow_run.orm.dart @@ -407,8 +407,7 @@ class StemWorkflowRunModelFactory { static ModelFactoryBuilder factory({ GeneratorProvider? generatorProvider, - }) => ModelFactoryBuilder( - definition: definition, + }) => ModelFactoryRegistry.factoryFor( generatorProvider: generatorProvider, ); } diff --git a/packages/stem_postgres/lib/src/database/models/stem_workflow_step.orm.dart b/packages/stem_postgres/lib/src/database/models/stem_workflow_step.orm.dart index 8d6c1950..2e2cb561 100644 --- a/packages/stem_postgres/lib/src/database/models/stem_workflow_step.orm.dart +++ b/packages/stem_postgres/lib/src/database/models/stem_workflow_step.orm.dart @@ -211,8 +211,7 @@ class StemWorkflowStepModelFactory { static ModelFactoryBuilder factory({ GeneratorProvider? generatorProvider, - }) => ModelFactoryBuilder( - definition: definition, + }) => ModelFactoryRegistry.factoryFor( generatorProvider: generatorProvider, ); } diff --git a/packages/stem_postgres/lib/src/database/models/stem_workflow_watcher.orm.dart b/packages/stem_postgres/lib/src/database/models/stem_workflow_watcher.orm.dart index 383cb185..6a833b43 100644 --- a/packages/stem_postgres/lib/src/database/models/stem_workflow_watcher.orm.dart +++ b/packages/stem_postgres/lib/src/database/models/stem_workflow_watcher.orm.dart @@ -265,8 +265,7 @@ class StemWorkflowWatcherModelFactory { static ModelFactoryBuilder factory({ GeneratorProvider? generatorProvider, - }) => ModelFactoryBuilder( - definition: definition, + }) => ModelFactoryRegistry.factoryFor( generatorProvider: generatorProvider, ); } diff --git a/packages/stem_postgres/lib/src/database/orm_registry.g.dart b/packages/stem_postgres/lib/src/database/orm_registry.g.dart new file mode 100644 index 00000000..1ec2c956 --- /dev/null +++ b/packages/stem_postgres/lib/src/database/orm_registry.g.dart @@ -0,0 +1,121 @@ +// GENERATED CODE - DO NOT MODIFY BY HAND +// coverage:ignore-file +import 'package:ormed/ormed.dart'; +import 'package:stem_postgres/src/database/models/stem_broadcast_ack.dart'; +import 'package:stem_postgres/src/database/models/stem_broadcast_message.dart'; +import 'package:stem_postgres/src/database/models/stem_dead_letter.dart'; +import 'package:stem_postgres/src/database/models/stem_group.dart'; +import 'package:stem_postgres/src/database/models/stem_group_result.dart'; +import 'package:stem_postgres/src/database/models/stem_lock.dart'; +import 'package:stem_postgres/src/database/models/stem_queue_job.dart'; +import 'package:stem_postgres/src/database/models/stem_revoke_entry.dart'; +import 'package:stem_postgres/src/database/models/stem_schedule_entry.dart'; +import 'package:stem_postgres/src/database/models/stem_task_result.dart'; +import 'package:stem_postgres/src/database/models/stem_worker_heartbeat.dart'; +import 'package:stem_postgres/src/database/models/stem_workflow_run.dart'; +import 'package:stem_postgres/src/database/models/stem_workflow_step.dart'; +import 'package:stem_postgres/src/database/models/stem_workflow_watcher.dart'; + +final List> _$ormModelDefinitions = [ + StemBroadcastAckOrmDefinition.definition, + StemBroadcastMessageOrmDefinition.definition, + StemDeadLetterOrmDefinition.definition, + StemGroupOrmDefinition.definition, + StemGroupResultOrmDefinition.definition, + StemLockOrmDefinition.definition, + StemQueueJobOrmDefinition.definition, + StemRevokeEntryOrmDefinition.definition, + StemScheduleEntryOrmDefinition.definition, + StemTaskResultOrmDefinition.definition, + StemWorkerHeartbeatOrmDefinition.definition, + StemWorkflowRunOrmDefinition.definition, + StemWorkflowStepOrmDefinition.definition, + StemWorkflowWatcherOrmDefinition.definition, +]; + +ModelRegistry buildOrmRegistry() => ModelRegistry() + ..registerAll(_$ormModelDefinitions) + ..registerTypeAlias(_$ormModelDefinitions[0]) + ..registerTypeAlias(_$ormModelDefinitions[1]) + ..registerTypeAlias(_$ormModelDefinitions[2]) + ..registerTypeAlias(_$ormModelDefinitions[3]) + ..registerTypeAlias(_$ormModelDefinitions[4]) + ..registerTypeAlias(_$ormModelDefinitions[5]) + ..registerTypeAlias(_$ormModelDefinitions[6]) + ..registerTypeAlias(_$ormModelDefinitions[7]) + ..registerTypeAlias(_$ormModelDefinitions[8]) + ..registerTypeAlias(_$ormModelDefinitions[9]) + ..registerTypeAlias(_$ormModelDefinitions[10]) + ..registerTypeAlias(_$ormModelDefinitions[11]) + ..registerTypeAlias(_$ormModelDefinitions[12]) + ..registerTypeAlias(_$ormModelDefinitions[13]); + +List> get generatedOrmModelDefinitions => + List.unmodifiable(_$ormModelDefinitions); + +extension GeneratedOrmModels on ModelRegistry { + ModelRegistry registerGeneratedModels() { + registerAll(_$ormModelDefinitions); + registerTypeAlias(_$ormModelDefinitions[0]); + registerTypeAlias(_$ormModelDefinitions[1]); + registerTypeAlias(_$ormModelDefinitions[2]); + registerTypeAlias(_$ormModelDefinitions[3]); + registerTypeAlias(_$ormModelDefinitions[4]); + registerTypeAlias(_$ormModelDefinitions[5]); + registerTypeAlias(_$ormModelDefinitions[6]); + registerTypeAlias(_$ormModelDefinitions[7]); + registerTypeAlias(_$ormModelDefinitions[8]); + registerTypeAlias(_$ormModelDefinitions[9]); + registerTypeAlias(_$ormModelDefinitions[10]); + registerTypeAlias(_$ormModelDefinitions[11]); + registerTypeAlias(_$ormModelDefinitions[12]); + registerTypeAlias(_$ormModelDefinitions[13]); + return this; + } +} + +/// Registers factory definitions for all models that have factory support. +/// Call this before using [Model.factory()] to ensure definitions are available. +void registerOrmFactories() {} + +/// Combined setup: registers both model registry and factories. +/// Returns a ModelRegistry with all generated models registered. +ModelRegistry buildOrmRegistryWithFactories() { + registerOrmFactories(); + return buildOrmRegistry(); +} + +/// Registers generated model event handlers. +void registerModelEventHandlers({EventBus? bus}) { + // No model event handlers were generated. +} + +/// Registers generated model scopes into a [ScopeRegistry]. +void registerModelScopes({ScopeRegistry? scopeRegistry}) { + // No model scopes were generated. +} + +/// Bootstraps generated ORM pieces: registry, factories, event handlers, and scopes. +ModelRegistry bootstrapOrm({ + ModelRegistry? registry, + EventBus? bus, + ScopeRegistry? scopes, + bool registerFactories = true, + bool registerEventHandlers = true, + bool registerScopes = true, +}) { + final reg = registry ?? buildOrmRegistry(); + if (registry != null) { + reg.registerGeneratedModels(); + } + if (registerFactories) { + registerOrmFactories(); + } + if (registerEventHandlers) { + registerModelEventHandlers(bus: bus); + } + if (registerScopes) { + registerModelScopes(scopeRegistry: scopes); + } + return reg; +} diff --git a/packages/stem_postgres/lib/src/database/seed_runtime.dart b/packages/stem_postgres/lib/src/database/seed_runtime.dart index 794bdfcd..1fe4f15e 100644 --- a/packages/stem_postgres/lib/src/database/seed_runtime.dart +++ b/packages/stem_postgres/lib/src/database/seed_runtime.dart @@ -4,7 +4,7 @@ import 'dart:io'; import 'package:artisanal/args.dart'; import 'package:ormed/ormed.dart'; -import 'package:stem/stem.dart' show stemLogger; +import 'package:stem/observability.dart' show stemLogger; import 'package:stem_postgres/src/database/datasource.dart'; diff --git a/packages/stem_postgres/lib/src/database/seeders.dart b/packages/stem_postgres/lib/src/database/seeders.dart index 836ca029..bb3fcaa2 100644 --- a/packages/stem_postgres/lib/src/database/seeders.dart +++ b/packages/stem_postgres/lib/src/database/seeders.dart @@ -1,7 +1,12 @@ +// The public seed helper is intentionally available alongside the CLI main. +// The executable and standalone library analyzers disagree about reachability. +// ignore_for_file: unreachable_from_main, unnecessary_ignore + import 'package:ormed/ormed.dart'; import 'package:stem_postgres/orm_registry.g.dart' as g; import 'package:stem_postgres/src/database/seed_runtime.dart'; import 'package:stem_postgres/src/database/seeders/database_seeder.dart'; + // // @@ -25,7 +30,6 @@ final List seeders = [ /// await runProjectSeeds(connection, names: ['UserSeeder']); /// ``` // Exposed as a library API even though this file includes a main entrypoint. -// ignore: unreachable_from_main Future runProjectSeeds( OrmConnection connection, { List? names, diff --git a/packages/stem_postgres/lib/src/database/stem_orm_logger.dart b/packages/stem_postgres/lib/src/database/stem_orm_logger.dart new file mode 100644 index 00000000..7128cc5b --- /dev/null +++ b/packages/stem_postgres/lib/src/database/stem_orm_logger.dart @@ -0,0 +1,65 @@ +import 'package:contextual/contextual.dart' as contextual; +import 'package:stem/observability.dart' show StemLogger; + +/// Adapts Stem's dependency-neutral logger to the ORM's logger contract. +contextual.Logger createOrmLogger(StemLogger logger) { + final ormLogger = + contextual.Logger( + level: contextual.Level.debug, + defaultChannelEnabled: false, + )..setListener((entry) { + final record = entry.record; + final fields = record.context.all().cast(); + switch (record.level) { + case contextual.Level.debug: + logger.debug( + record.message, + fields: fields, + stackTrace: record.stackTrace, + ); + case contextual.Level.info: + logger.info( + record.message, + fields: fields, + stackTrace: record.stackTrace, + ); + case contextual.Level.notice: + logger.notice( + record.message, + fields: fields, + stackTrace: record.stackTrace, + ); + case contextual.Level.warning: + logger.warning( + record.message, + fields: fields, + stackTrace: record.stackTrace, + ); + case contextual.Level.error: + logger.error( + record.message, + fields: fields, + stackTrace: record.stackTrace, + ); + case contextual.Level.critical: + logger.critical( + record.message, + fields: fields, + stackTrace: record.stackTrace, + ); + case contextual.Level.alert: + logger.alert( + record.message, + fields: fields, + stackTrace: record.stackTrace, + ); + case contextual.Level.emergency: + logger.emergency( + record.message, + fields: fields, + stackTrace: record.stackTrace, + ); + } + }); + return ormLogger; +} diff --git a/packages/stem_postgres/lib/src/outbox/postgres_transactional_outbox.dart b/packages/stem_postgres/lib/src/outbox/postgres_transactional_outbox.dart new file mode 100644 index 00000000..2e8ddded --- /dev/null +++ b/packages/stem_postgres/lib/src/outbox/postgres_transactional_outbox.dart @@ -0,0 +1,442 @@ +import 'dart:async'; +import 'dart:convert'; + +import 'package:ormed/ormed.dart'; +import 'package:stem/stem.dart'; +import 'package:stem_postgres/src/connection.dart'; + +/// Durable, PostgreSQL-backed publication outbox. +/// +/// Application writes and calls to [PostgresOutboxBroker.publish] must happen +/// inside [transaction]. A relay later calls [dispatch] to publish committed +/// rows to the underlying broker. Relay delivery is at least once: a process +/// crash after broker publication and before marking a row dispatched can +/// cause another publication. PostgreSQL broker publication is idempotent by +/// envelope id, but task handlers must still be safe for duplicate delivery. +class PostgresTransactionalOutbox { + PostgresTransactionalOutbox._( + this._connections, { + required this.namespace, + }); + + /// Opens an outbox on an existing initialized [DataSource]. + /// + /// The caller remains responsible for disposing [dataSource]. + static Future fromDataSource( + DataSource dataSource, { + String namespace = 'stem', + bool runMigrations = true, + }) async { + final connections = await PostgresConnections.openWithDataSource( + dataSource, + runMigrations: runMigrations, + ); + return PostgresTransactionalOutbox._( + connections, + namespace: _normalizeNamespace(namespace), + ); + } + + /// Opens an outbox with its own PostgreSQL data source. + static Future connect( + String connectionString, { + String namespace = 'stem', + }) async { + final connections = await PostgresConnections.open( + connectionString: connectionString, + ); + return PostgresTransactionalOutbox._( + connections, + namespace: _normalizeNamespace(namespace), + ); + } + + static final Object _transactionZoneKey = Object(); + + final PostgresConnections _connections; + + /// Namespace used to scope outbox rows. + final String namespace; + + /// Closes the outbox's owned data source. + /// + /// An outbox created with [fromDataSource] does not dispose the caller's + /// data source. + Future close() => _connections.close(); + + /// Makes a producer-only broker facade that records publications in the + /// current [transaction] instead of sending them immediately. + /// + /// All non-publication broker operations delegate to [delegate]. Pass the + /// underlying broker, rather than this facade, to [dispatch]. + PostgresOutboxBroker wrap(QueueBroker delegate) => + PostgresOutboxBroker(this, delegate); + + /// Returns the outbox transaction active in the current asynchronous + /// context, or `null` when no outbox transaction is active. + PostgresOutboxTransaction? get currentTransaction { + final value = Zone.current[_transactionZoneKey]; + return value is PostgresOutboxTransaction ? value : null; + } + + /// Executes [action] and atomically commits its application writes and + /// outbox publications. + Future transaction( + Future Function(PostgresOutboxTransaction transaction) action, + ) { + return _connections.runInTransaction((context) { + final transaction = PostgresOutboxTransaction._( + this, + context, + namespace: namespace, + ); + return runZoned( + () => action(transaction), + zoneValues: {_transactionZoneKey: transaction}, + ); + }); + } + + /// Returns the number of rows not yet marked as dispatched. + Future pendingCount() { + return _connections.runInTransaction((context) async { + return _outboxTable(context).whereEquals('namespace', namespace).whereIn( + 'status', + const ['pending', 'processing', 'failed'], + ).count(); + }); + } + + /// Publishes committed outbox rows through [broker]. + /// + /// Rows are claimed with `FOR UPDATE SKIP LOCKED`, leased, and retried after + /// [retryDelay] when publication fails. A stale processing lease can be + /// reclaimed by a later relay invocation. + Future dispatch({ + required QueueBroker broker, + int limit = 100, + Duration lease = const Duration(minutes: 1), + Duration retryDelay = const Duration(seconds: 5), + String? workerId, + }) async { + if (limit <= 0) return 0; + if (lease <= Duration.zero) { + throw ArgumentError.value(lease, 'lease', 'Lease must be positive.'); + } + final resolvedWorkerId = + workerId ?? 'stem-outbox-${DateTime.now().microsecondsSinceEpoch}'; + final claimed = await _claim( + limit: limit, + lease: lease, + workerId: resolvedWorkerId, + ); + var published = 0; + for (final message in claimed) { + try { + await broker.publish(message.envelope, routing: message.routing); + if (await _markDispatched(message.id, workerId: resolvedWorkerId)) { + published++; + } + } on Object catch (error, stackTrace) { + await _markFailed( + message.id, + workerId: resolvedWorkerId, + error: '$error\n$stackTrace', + retryDelay: retryDelay, + ); + } + } + return published; + } + + Future> _claim({ + required int limit, + required Duration lease, + required String workerId, + }) { + return _connections.runInTransaction((context) async { + final now = DateTime.now().toUtc(); + final lockedUntil = now.add(lease); + final rows = await _outboxTable(context) + .whereEquals('namespace', namespace) + .whereIn('status', const ['pending', 'processing', 'failed']) + .whereRaw('available_at <= ?', [now]) + .whereRaw('(locked_until IS NULL OR locked_until <= ?)', [now]) + .orderBy('created_at') + .limit(limit) + .lock('FOR UPDATE SKIP LOCKED') + .get(); + final claimed = <_ClaimedOutboxMessage>[]; + for (final row in rows) { + final id = row['id']! as String; + final attempts = _asInt(row['attempts']) + 1; + await _outboxTable(context).whereEquals('id', id).update({ + 'status': 'processing', + 'attempts': attempts, + 'locked_at': now, + 'locked_until': lockedUntil, + 'locked_by': workerId, + 'updated_at': now, + }); + claimed.add( + _ClaimedOutboxMessage( + id: id, + envelope: Envelope.fromJson(_asMap(row['envelope'])), + routing: row['routing'] == null + ? null + : RoutingInfo.fromJson(_asMap(row['routing'])), + ), + ); + } + return claimed; + }); + } + + Future _markDispatched(String id, {required String workerId}) { + return _connections.runInTransaction((context) async { + final now = DateTime.now().toUtc(); + final updated = await _outboxTable(context) + .whereEquals('id', id) + .whereEquals('namespace', namespace) + .whereEquals('status', 'processing') + .whereEquals('locked_by', workerId) + .update({ + 'status': 'dispatched', + 'dispatched_at': now, + 'locked_at': null, + 'locked_until': null, + 'locked_by': null, + 'updated_at': now, + }); + return updated > 0; + }); + } + + Future _markFailed( + String id, { + required String workerId, + required String error, + required Duration retryDelay, + }) { + return _connections.runInTransaction((context) async { + final now = DateTime.now().toUtc(); + final message = error.length > 4000 ? error.substring(0, 4000) : error; + final updated = await _outboxTable(context) + .whereEquals('id', id) + .whereEquals('namespace', namespace) + .whereEquals('status', 'processing') + .whereEquals('locked_by', workerId) + .update({ + 'status': 'failed', + 'available_at': now.add(retryDelay), + 'last_error': message, + 'locked_at': null, + 'locked_until': null, + 'locked_by': null, + 'updated_at': now, + }); + return updated > 0; + }); + } +} + +/// Active transaction passed to [PostgresTransactionalOutbox.transaction]. +class PostgresOutboxTransaction { + PostgresOutboxTransaction._( + this.outbox, + this.context, { + required this.namespace, + }); + + /// Outbox that owns this transaction. + final PostgresTransactionalOutbox outbox; + + /// Query context for application writes in the same transaction. + final QueryContext context; + + /// Namespace used for publications in this transaction. + final String namespace; + + /// Adds an already-encoded task envelope to the transaction's outbox. + Future enqueueEnvelope( + Envelope envelope, { + RoutingInfo? routing, + }) async { + final availableAt = envelope.notBefore?.toUtc() ?? DateTime.now().toUtc(); + await _outboxTable(context).create({ + 'id': envelope.id, + 'namespace': namespace, + 'envelope': envelope.toJson(), + 'routing': routing?.toJson(), + 'status': 'pending', + 'available_at': availableAt, + 'attempts': 0, + 'created_at': DateTime.now().toUtc(), + 'updated_at': DateTime.now().toUtc(), + }); + } +} + +/// Producer facade that diverts publication into a Postgres outbox. +class PostgresOutboxBroker extends Broker + implements + LeaseBroker, + InspectableBroker, + DeadLetterBroker, + BrokerCapabilitiesProvider { + /// Creates an outbox facade around [delegate]. + PostgresOutboxBroker(this.outbox, this.delegate); + + /// Outbox receiving publications. + final PostgresTransactionalOutbox outbox; + + /// Broker used for all operations other than publication. + final QueueBroker delegate; + + @override + Future publish(Envelope envelope, {RoutingInfo? routing}) async { + final transaction = outbox.currentTransaction; + if (transaction == null || !identical(transaction.outbox, outbox)) { + throw StateError( + 'PostgresOutboxBroker.publish must run inside ' + 'PostgresTransactionalOutbox.transaction.', + ); + } + await transaction.enqueueEnvelope(envelope, routing: routing); + } + + @override + Stream consume( + RoutingSubscription subscription, { + int prefetch = 1, + String? consumerGroup, + String? consumerName, + }) => delegate.consume( + subscription, + prefetch: prefetch, + consumerGroup: consumerGroup, + consumerName: consumerName, + ); + + @override + Future ack(Delivery delivery) => delegate.ack(delivery); + + @override + Future nack(Delivery delivery, {bool requeue = true}) => + delegate.nack(delivery, requeue: requeue); + + @override + Future deadLetter( + Delivery delivery, { + String? reason, + Map? meta, + }) => delegate.deadLetter(delivery, reason: reason, meta: meta); + + @override + Future purge(String queue) { + final legacy = delegate; + if (legacy is Broker) return legacy.purge(queue); + throw UnsupportedError('Queue purging is not supported.'); + } + + @override + Future extendLease(Delivery delivery, Duration by) => + delegate.extendLease(delivery, by); + + @override + Future pendingCount(String queue) => delegate.pendingCount(queue); + + @override + Future inflightCount(String queue) => delegate.inflightCount(queue); + + @override + bool get supportsDelayed => delegate.capabilities.supportsDelayedDelivery; + + @override + bool get supportsPriority => delegate.capabilities.supportsPriorityOrdering; + + @override + BrokerCapabilities get capabilities => delegate.capabilities; + + @override + Future listDeadLetters( + String queue, { + int limit = 50, + int offset = 0, + }) => delegate.listDeadLetters(queue, limit: limit, offset: offset); + + @override + Future getDeadLetter(String queue, String id) => + delegate.getDeadLetter(queue, id); + + @override + Future replayDeadLetters( + String queue, { + int limit = 50, + DateTime? since, + Duration? delay, + bool dryRun = false, + }) => delegate.replayDeadLetters( + queue, + limit: limit, + since: since, + delay: delay, + dryRun: dryRun, + ); + + @override + Future purgeDeadLetters(String queue, {DateTime? since, int? limit}) => + delegate.purgeDeadLetters(queue, since: since, limit: limit); + + @override + Future close() => delegate.close(); +} + +class _ClaimedOutboxMessage { + const _ClaimedOutboxMessage({ + required this.id, + required this.envelope, + required this.routing, + }); + + final String id; + final Envelope envelope; + final RoutingInfo? routing; +} + +String _normalizeNamespace(String namespace) { + final value = namespace.trim(); + return value.isEmpty ? 'stem' : value; +} + +Map _asMap(Object? value) { + if (value is Map) return value; + if (value is Map) return value.cast(); + if (value is String) { + final decoded = jsonDecode(value); + if (decoded is Map) return decoded.cast(); + } + throw StateError('Expected a JSON object in the task outbox.'); +} + +int _asInt(Object? value) => value is num ? value.toInt() : 0; + +Query _outboxTable(QueryContext context) => context.table( + 'stem_task_outbox', + columns: const [ + AdHocColumn(name: 'id', isNullable: false, isPrimaryKey: true), + AdHocColumn(name: 'namespace', isNullable: false), + AdHocColumn(name: 'envelope', isNullable: false), + AdHocColumn(name: 'routing'), + AdHocColumn(name: 'status', isNullable: false), + AdHocColumn(name: 'available_at', isNullable: false), + AdHocColumn(name: 'attempts', isNullable: false), + AdHocColumn(name: 'locked_at'), + AdHocColumn(name: 'locked_until'), + AdHocColumn(name: 'locked_by'), + AdHocColumn(name: 'last_error'), + AdHocColumn(name: 'dispatched_at'), + AdHocColumn(name: 'created_at', isNullable: false), + AdHocColumn(name: 'updated_at', isNullable: false), + ], +); diff --git a/packages/stem_postgres/lib/src/rate_limiting/postgres_rate_limiter.dart b/packages/stem_postgres/lib/src/rate_limiting/postgres_rate_limiter.dart new file mode 100644 index 00000000..175c2895 --- /dev/null +++ b/packages/stem_postgres/lib/src/rate_limiting/postgres_rate_limiter.dart @@ -0,0 +1,222 @@ +import 'package:ormed/ormed.dart'; +import 'package:stem/stem.dart'; +import 'package:stem_postgres/src/connection.dart'; + +/// PostgreSQL-backed distributed token-bucket rate limiter. +/// +/// Each acquire runs in a database transaction and locks one bucket row with +/// `FOR UPDATE`. This makes refill and consumption atomic across processes and +/// avoids relying on the wall clock of any individual worker. The bucket +/// stores micro-permits as integers so fractional refills do not accumulate +/// floating-point error. +class PostgresRateLimiter implements RateLimiter { + PostgresRateLimiter._(this._connections, {required this.namespace}); + + /// Creates a limiter using an existing initialized [DataSource]. + /// + /// The caller remains responsible for disposing [dataSource]. + static Future fromDataSource( + DataSource dataSource, { + String namespace = 'stem', + bool runMigrations = true, + }) async { + final connections = await PostgresConnections.openWithDataSource( + dataSource, + runMigrations: runMigrations, + ); + return PostgresRateLimiter._( + connections, + namespace: _normalizeNamespace(namespace), + ); + } + + /// Opens a limiter from a PostgreSQL connection string and runs migrations. + static Future connect( + String uri, { + String namespace = 'stem', + }) async { + final connections = await PostgresConnections.open( + connectionString: uri, + ); + return PostgresRateLimiter._( + connections, + namespace: _normalizeNamespace(namespace), + ); + } + + static const int _microsPerPermit = 1000000; + static const int _maxCapacity = 9000000000000; + + final PostgresConnections _connections; + + /// Namespace used to isolate limiter buckets. + final String namespace; + + /// Closes the owned connection when the limiter was created with [connect]. + Future close() => _connections.close(); + + @override + Future acquire( + String key, { + int tokens = 1, + Duration? interval, + Map? meta, + }) async { + if (tokens <= 0 || tokens > _maxCapacity) { + throw ArgumentError.value( + tokens, + 'tokens', + 'Capacity must be between 1 and $_maxCapacity.', + ); + } + final window = interval ?? const Duration(seconds: 1); + final intervalMs = window.inMilliseconds; + if (intervalMs <= 0) { + throw ArgumentError.value( + window, + 'interval', + 'Rate-limit interval must be at least one millisecond.', + ); + } + + final capacityMicros = tokens * _microsPerPermit; + return _connections.runInTransaction((context) async { + final driver = context.driver; + final nowRows = await driver.queryRaw( + 'SELECT FLOOR(EXTRACT(EPOCH FROM clock_timestamp()) * 1000)::bigint ' + 'AS now_ms', + ); + final nowMs = _asInt(nowRows.single['now_ms']); + + // Insert first so concurrent first-use callers serialize on the primary + // key. ON CONFLICT waits for the competing transaction, then the row is + // selected and locked below. + await driver.executeRaw( + ''' +INSERT INTO stem_rate_limit_buckets + (namespace, rate_key, capacity, interval_ms, available_micros, + updated_at_ms, created_at, updated_at) +VALUES (?, ?, ?, ?, ?, ?, clock_timestamp(), clock_timestamp()) +ON CONFLICT (namespace, rate_key) DO NOTHING +''', + [ + namespace, + key, + tokens, + intervalMs, + capacityMicros, + nowMs, + ], + ); + + final rows = await driver.queryRaw( + ''' +SELECT capacity, interval_ms, available_micros, updated_at_ms +FROM stem_rate_limit_buckets +WHERE namespace = ? AND rate_key = ? +FOR UPDATE +''', + [namespace, key], + ); + if (rows.isEmpty) { + throw StateError('Rate-limit bucket was not created: $namespace/$key'); + } + + final row = rows.single; + final storedCapacity = _asInt(row['capacity']); + final storedInterval = _asInt(row['interval_ms']); + final storedAvailable = _asInt(row['available_micros']); + final storedAt = _asInt(row['updated_at_ms']); + final capacityChanged = + storedCapacity != tokens || storedInterval != intervalMs; + + final available = capacityChanged + ? capacityMicros + : _refill( + availableMicros: storedAvailable, + capacityMicros: capacityMicros, + elapsedMs: (nowMs - storedAt).clamp(0, nowMs), + intervalMs: intervalMs, + ); + final allowed = available >= _microsPerPermit; + final remaining = allowed ? available - _microsPerPermit : available; + final retryAfterMs = allowed + ? null + : _retryAfterMs( + missingMicros: _microsPerPermit - available, + capacityMicros: capacityMicros, + intervalMs: intervalMs, + ); + + await driver.executeRaw( + ''' +UPDATE stem_rate_limit_buckets +SET capacity = ?, interval_ms = ?, available_micros = ?, + updated_at_ms = ?, updated_at = clock_timestamp() +WHERE namespace = ? AND rate_key = ? +''', + [ + tokens, + intervalMs, + remaining, + nowMs, + namespace, + key, + ], + ); + + return RateLimitDecision( + allowed: allowed, + retryAfter: retryAfterMs == null + ? null + : Duration(milliseconds: retryAfterMs), + meta: { + 'capacity': tokens, + 'intervalMs': intervalMs, + 'remainingTokens': remaining / _microsPerPermit, + 'backend': 'postgres', + ...?meta, + }, + ); + }); + } + + static int _refill({ + required int availableMicros, + required int capacityMicros, + required int elapsedMs, + required int intervalMs, + }) { + if (elapsedMs <= 0) return availableMicros.clamp(0, capacityMicros); + final added = + (BigInt.from(elapsedMs) * BigInt.from(capacityMicros)) ~/ + BigInt.from(intervalMs); + final total = BigInt.from(availableMicros) + added; + final capped = total.compareTo(BigInt.from(capacityMicros)) > 0 + ? BigInt.from(capacityMicros) + : total; + return capped.toInt(); + } + + static int _retryAfterMs({ + required int missingMicros, + required int capacityMicros, + required int intervalMs, + }) { + final numerator = BigInt.from(missingMicros) * BigInt.from(intervalMs); + final denominator = BigInt.from(capacityMicros); + final milliseconds = (numerator + denominator - BigInt.one) ~/ denominator; + return milliseconds < BigInt.one ? 1 : milliseconds.toInt(); + } +} + +String _normalizeNamespace(String value) { + final trimmed = value.trim(); + return trimmed.isEmpty ? 'stem' : trimmed; +} + +int _asInt(Object? value) { + if (value is int) return value; + if (value is num) return value.toInt(); + return int.parse(value.toString()); +} diff --git a/packages/stem_postgres/lib/src/scheduler/postgres_lock_store.dart b/packages/stem_postgres/lib/src/scheduler/postgres_lock_store.dart index d05af002..0d0f899b 100644 --- a/packages/stem_postgres/lib/src/scheduler/postgres_lock_store.dart +++ b/packages/stem_postgres/lib/src/scheduler/postgres_lock_store.dart @@ -72,47 +72,73 @@ class PostgresLockStore implements LockStore { final ownerValue = _owner(owner); final now = stemNow().toUtc(); final expiresAt = now.add(ttl); - final ctx = _connections.context; - final repository = ctx.repository<$StemLock>(); - - final inserted = await repository.insertOrIgnore( - $StemLock( - key: key, - namespace: namespace, - owner: ownerValue, - expiresAt: expiresAt, - createdAt: now, - ), - ); - if (inserted > 0) { - return _PostgresLock(store: this, key: key, owner: ownerValue); - } - - // Lock exists, try to clean up expired and retry. - final expired = await ctx - .query<$StemLock>() - .whereEquals('key', key) - .whereEquals('namespace', namespace) - .where('expiresAt', now, PredicateOperator.lessThan) - .get(); - - for (final lock in expired) { - await repository.delete(lock); - } - - final retryInserted = await repository.insertOrIgnore( - $StemLock( - key: key, - namespace: namespace, - owner: ownerValue, - expiresAt: expiresAt, - createdAt: now, - ), - ); - if (retryInserted > 0) { - return _PostgresLock(store: this, key: key, owner: ownerValue); - } - return null; + return _connections.runInTransaction((ctx) async { + // Insert first so concurrent first-use callers serialize on the unique + // key. ON CONFLICT waits for the competing transaction, then the row is + // selected and locked below. RETURNING tells us whether this call + // created the row, avoiding owner-value heuristics when callers reuse an + // owner id. + final inserted = await ctx.driver.queryRaw( + ''' +INSERT INTO stem_locks + (key, namespace, owner, expires_at, created_at, fencing_token) +VALUES (?, ?, ?, ?, ?, ?) +ON CONFLICT (key) DO NOTHING +RETURNING fencing_token +''', + [key, namespace, ownerValue, expiresAt, now, 1], + ); + if (inserted.isNotEmpty) { + return _PostgresLock( + store: this, + key: key, + owner: ownerValue, + fencingToken: _asInt(inserted.single['fencing_token']), + ); + } + + final rows = await ctx.driver.queryRaw( + ''' +SELECT owner, expires_at, fencing_token +FROM stem_locks +WHERE key = ? AND namespace = ? +FOR UPDATE +''', + [key, namespace], + ); + if (rows.isNotEmpty) { + final row = rows.single; + final currentExpiry = row['expires_at']; + if (currentExpiry is DateTime && currentExpiry.isAfter(now)) { + return null; + } + + final fencingToken = _asInt(row['fencing_token']) + 1; + await ctx.driver.executeRaw( + ''' +UPDATE stem_locks +SET owner = ?, expires_at = ?, created_at = ?, fencing_token = ? +WHERE key = ? AND namespace = ? +''', + [ + ownerValue, + expiresAt, + now, + fencingToken, + key, + namespace, + ], + ); + return _PostgresLock( + store: this, + key: key, + owner: ownerValue, + fencingToken: fencingToken, + ); + } + + throw StateError('Lock row was not created: $namespace/$key'); + }); } Future _renew(String key, String owner, Duration ttl) async { @@ -154,8 +180,9 @@ class PostgresLockStore implements LockStore { if (locks.isEmpty) return false; + final now = stemNow().toUtc(); for (final lock in locks) { - await ctx.repository<$StemLock>().delete(lock); + await ctx.repository<$StemLock>().update(lock.copyWith(expiresAt: now)); } return true; } @@ -181,8 +208,18 @@ class PostgresLockStore implements LockStore { Future release(String key, String owner) => _release(key, owner); } -class _PostgresLock implements Lock { - _PostgresLock({required this.store, required this.key, required this.owner}); +int _asInt(Object? value) { + if (value is int) return value; + return int.tryParse(value?.toString() ?? '') ?? 0; +} + +class _PostgresLock implements FencedLock { + _PostgresLock({ + required this.store, + required this.key, + required this.owner, + required this.fencingToken, + }); final PostgresLockStore store; @override @@ -190,6 +227,9 @@ class _PostgresLock implements Lock { @override final String owner; + @override + final int fencingToken; + @override Future renew(Duration ttl) => store._renew(key, owner, ttl); diff --git a/packages/stem_postgres/lib/src/workflow/postgres_workflow_store.dart b/packages/stem_postgres/lib/src/workflow/postgres_workflow_store.dart index c6f867af..90ac5ff9 100644 --- a/packages/stem_postgres/lib/src/workflow/postgres_workflow_store.dart +++ b/packages/stem_postgres/lib/src/workflow/postgres_workflow_store.dart @@ -99,9 +99,9 @@ class PostgresWorkflowStore implements WorkflowStore { @override Future createRun({ - String? runId, required String workflow, required Map params, + String? runId, String? parentRunId, Duration? ttl, WorkflowCancellationPolicy? cancellationPolicy, diff --git a/packages/stem_postgres/lib/src/workflow/postgres_workflow_store_new.dart b/packages/stem_postgres/lib/src/workflow/postgres_workflow_store_new.dart index ec883b7a..ae1a0968 100644 --- a/packages/stem_postgres/lib/src/workflow/postgres_workflow_store_new.dart +++ b/packages/stem_postgres/lib/src/workflow/postgres_workflow_store_new.dart @@ -74,9 +74,9 @@ class PostgresWorkflowStore implements WorkflowStore { @override Future createRun({ - String? runId, required String workflow, required Map params, + String? runId, String? parentRunId, Duration? ttl, WorkflowCancellationPolicy? cancellationPolicy, diff --git a/packages/stem_postgres/lib/stem_postgres.dart b/packages/stem_postgres/lib/stem_postgres.dart index 873049ae..46bf2e9f 100644 --- a/packages/stem_postgres/lib/stem_postgres.dart +++ b/packages/stem_postgres/lib/stem_postgres.dart @@ -1,6 +1,12 @@ export 'src/backend/postgres_backend.dart' show PostgresResultBackend; export 'src/brokers/postgres_broker.dart' show PostgresBroker; export 'src/control/postgres_revoke_store.dart' show PostgresRevokeStore; +export 'src/outbox/postgres_transactional_outbox.dart' + show + PostgresOutboxBroker, + PostgresOutboxTransaction, + PostgresTransactionalOutbox; +export 'src/rate_limiting/postgres_rate_limiter.dart' show PostgresRateLimiter; export 'src/scheduler/postgres_lock_store.dart' show PostgresLockStore; export 'src/scheduler/postgres_schedule_store.dart' show PostgresScheduleStore; export 'src/stack/postgres_adapter.dart' diff --git a/packages/stem_postgres/pubspec.yaml b/packages/stem_postgres/pubspec.yaml index 7bdcdeb6..9eb9aaec 100644 --- a/packages/stem_postgres/pubspec.yaml +++ b/packages/stem_postgres/pubspec.yaml @@ -1,6 +1,6 @@ name: stem_postgres description: Postgres broker, result backend, and scheduler utilities for Stem. -version: 0.1.1 +version: 0.2.0 repository: https://github.com/kingwill101/stem resolution: workspace environment: @@ -14,13 +14,13 @@ dependencies: ormed_postgres: ^0.2.0 path: ^1.9.1 postgres: ^3.5.9 - stem: ">=0.2.0-dev <0.3.0" + stem: ">=0.3.0 <0.4.0" uuid: ^4.5.2 dev_dependencies: build_runner: ^2.10.5 coverage: ^1.15.0 lints: ^6.0.0 - stem_adapter_tests: ^0.1.1 + stem_adapter_tests: ^0.2.0 test: ^1.29.0 very_good_analysis: ^10.0.0 diff --git a/packages/stem_postgres/test/datasource_logger_test.dart b/packages/stem_postgres/test/datasource_logger_test.dart index a2039833..62a71777 100644 --- a/packages/stem_postgres/test/datasource_logger_test.dart +++ b/packages/stem_postgres/test/datasource_logger_test.dart @@ -1,19 +1,18 @@ import 'package:contextual/contextual.dart' as contextual; -import 'package:stem/stem.dart'; +import 'package:stem/observability.dart' show stemLogger; import 'package:stem_postgres/src/database/datasource.dart'; import 'package:test/test.dart'; void main() { - test('createDataSource forwards logger and logging flags', () { - final logger = contextual.Logger(); + test('createDataSource accepts a Stem logger and enables ORM logging', () { final dataSource = createDataSource( connectionString: 'postgresql://user:pass@localhost:5432/stem', logging: true, - logger: logger, + logger: stemLogger, ); expect(dataSource.options.logging, isTrue); - expect(dataSource.options.logger, same(logger)); + expect(dataSource.options.logger, isA()); }); test('createDataSource defaults to stemLogger', () { @@ -21,6 +20,6 @@ void main() { connectionString: 'postgresql://user:pass@localhost:5432/stem', ); - expect(dataSource.options.logger, same(stemLogger)); + expect(dataSource.options.logger, isA()); }); } diff --git a/packages/stem_postgres/test/integration/backends/postgres_backend_integration_test.dart b/packages/stem_postgres/test/integration/backends/postgres_backend_integration_test.dart index 2adce8f9..ffb574cc 100644 --- a/packages/stem_postgres/test/integration/backends/postgres_backend_integration_test.dart +++ b/packages/stem_postgres/test/integration/backends/postgres_backend_integration_test.dart @@ -132,6 +132,48 @@ Future main() async { } }); + test('terminal writes arbitrate concurrent completion attempts', () async { + final namespace = + 'backend-terminal-${DateTime.now().microsecondsSinceEpoch}'; + final backend = await PostgresResultBackend.fromDataSource( + dataSource, + namespace: namespace, + defaultTtl: const Duration(seconds: 2), + groupDefaultTtl: const Duration(seconds: 2), + heartbeatTtl: const Duration(seconds: 2), + runMigrations: false, + ); + try { + await backend.set('terminal-task', TaskState.running); + final applied = await backend.setTerminalIfAbsent( + TaskStatus( + id: 'terminal-task', + state: TaskState.succeeded, + payload: 'first', + attempt: 1, + ), + ); + final rejected = await backend.setTerminalIfAbsent( + TaskStatus( + id: 'terminal-task', + state: TaskState.failed, + error: const TaskError(type: 'late', message: 'late failure'), + attempt: 1, + ), + ); + + expect(applied, isTrue); + expect(rejected, isFalse); + expect((await backend.get('terminal-task'))?.payload, 'first'); + expect( + (await backend.get('terminal-task'))?.state, + TaskState.succeeded, + ); + } finally { + await backend.close(); + } + }); + test('fromDataSource initializes lazy data sources', () async { final schema = dataSource.options.defaultSchema; if (schema == null || schema.isEmpty) { diff --git a/packages/stem_postgres/test/integration/backends/postgres_backend_tls_test.dart b/packages/stem_postgres/test/integration/backends/postgres_backend_tls_test.dart index 487c69f0..ec810f60 100644 --- a/packages/stem_postgres/test/integration/backends/postgres_backend_tls_test.dart +++ b/packages/stem_postgres/test/integration/backends/postgres_backend_tls_test.dart @@ -14,7 +14,15 @@ Future main() async { Platform.environment['STEM_TEST_POSTGRES_URL']; final caOverride = Platform.environment['STEM_TEST_POSTGRES_TLS_CA_CERT'] ?.trim(); - final defaultCa = File('docker/testing/certs/postgres-root.crt'); + final defaultCaCandidates = [ + File('docker/testing/postgres/certs/root.crt'), + File('packages/stem_cli/docker/testing/postgres/certs/root.crt'), + File('../stem_cli/docker/testing/postgres/certs/root.crt'), + ]; + final defaultCa = defaultCaCandidates.firstWhere( + (file) => file.existsSync(), + orElse: () => defaultCaCandidates.first, + ); final caPath = caOverride?.isNotEmpty ?? false ? caOverride : (defaultCa.existsSync() ? defaultCa.path : null); diff --git a/packages/stem_postgres/test/integration/locks/postgres_lock_store_test.dart b/packages/stem_postgres/test/integration/locks/postgres_lock_store_test.dart index 51692aba..f0aec492 100644 --- a/packages/stem_postgres/test/integration/locks/postgres_lock_store_test.dart +++ b/packages/stem_postgres/test/integration/locks/postgres_lock_store_test.dart @@ -1,6 +1,7 @@ import 'dart:io'; import 'package:ormed/ormed.dart'; +import 'package:stem/stem.dart'; import 'package:stem_adapter_tests/stem_adapter_tests.dart'; import 'package:stem_postgres/stem_postgres.dart'; import 'package:test/test.dart'; @@ -41,5 +42,51 @@ Future main() async { expiryBackoff: Duration(seconds: 1), ), ); + + test('lock acquisitions expose increasing fencing tokens', () async { + final store = await PostgresLockStore.fromDataSource( + dataSource, + namespace: 'stem_lock_fencing', + runMigrations: false, + ); + final first = await store.acquire('fenced', owner: 'first'); + expect(first, isA()); + final firstToken = (first! as FencedLock).fencingToken; + await first.release(); + + final second = await store.acquire('fenced', owner: 'second'); + expect(second, isA()); + expect((second! as FencedLock).fencingToken, greaterThan(firstToken)); + await second.release(); + await store.close(); + }); + + test( + 'concurrent first acquisitions return one winner without a conflict', + () async { + final namespace = + 'stem_lock_race_${DateTime.now().microsecondsSinceEpoch}'; + final firstStore = await PostgresLockStore.connect( + connectionString, + namespace: namespace, + ); + final secondStore = await PostgresLockStore.connect( + connectionString, + namespace: namespace, + ); + addTearDown(firstStore.close); + addTearDown(secondStore.close); + + final results = await Future.wait([ + firstStore.acquire('new-key', owner: 'first'), + secondStore.acquire('new-key', owner: 'second'), + ]); + + expect(results.whereType(), hasLength(1)); + for (final result in results.whereType()) { + await result.release(); + } + }, + ); }, config: harness.config); } diff --git a/packages/stem_postgres/test/integration/outbox/postgres_transactional_outbox_test.dart b/packages/stem_postgres/test/integration/outbox/postgres_transactional_outbox_test.dart new file mode 100644 index 00000000..ba82ede7 --- /dev/null +++ b/packages/stem_postgres/test/integration/outbox/postgres_transactional_outbox_test.dart @@ -0,0 +1,177 @@ +import 'dart:io'; + +import 'package:ormed/ormed.dart'; +import 'package:stem/stem.dart'; +import 'package:stem_postgres/stem_postgres.dart'; +import 'package:test/test.dart'; + +import '../../support/postgres_test_harness.dart'; + +Future main() async { + final connectionString = Platform.environment['STEM_TEST_POSTGRES_URL']; + if (connectionString == null || connectionString.isEmpty) { + test( + 'Postgres transactional outbox requires STEM_TEST_POSTGRES_URL', + () {}, + skip: 'Set STEM_TEST_POSTGRES_URL to run transactional outbox tests.', + ); + return; + } + + final harness = await createStemPostgresTestHarness( + connectionString: connectionString, + ); + tearDownAll(harness.dispose); + + ormedGroup('postgres transactional outbox', (dataSource) { + late PostgresTransactionalOutbox outbox; + late PostgresBroker broker; + late Broker outboxBroker; + late String namespace; + late String queue; + + setUp(() async { + namespace = 'outbox-${DateTime.now().microsecondsSinceEpoch}'; + queue = 'queue-${DateTime.now().microsecondsSinceEpoch}'; + outbox = await PostgresTransactionalOutbox.fromDataSource( + dataSource, + namespace: namespace, + runMigrations: false, + ); + broker = await PostgresBroker.fromDataSource( + dataSource, + namespace: namespace, + pollInterval: const Duration(milliseconds: 20), + sweeperInterval: const Duration(milliseconds: 100), + runMigrations: false, + ); + outboxBroker = outbox.wrap(broker); + }); + + tearDown(() => broker.close()); + + test( + 'rolls back publication records with the application transaction', + () async { + final envelope = Envelope( + id: 'rollback-${DateTime.now().microsecondsSinceEpoch}', + name: 'outbox.rollback', + args: const {'value': 1}, + queue: queue, + ); + + await expectLater( + outbox.transaction((_) async { + await outboxBroker.publish(envelope); + throw StateError('application transaction failed'); + }), + throwsStateError, + ); + + expect(await outbox.pendingCount(), 0); + expect(await broker.pendingCount(queue), 0); + }, + ); + + test( + 'dispatches committed rows once and preserves envelope identity', + () async { + final envelope = Envelope( + id: 'commit-${DateTime.now().microsecondsSinceEpoch}', + name: 'outbox.commit', + args: const {'value': 1}, + queue: queue, + ); + + await outbox.transaction((_) => outboxBroker.publish(envelope)); + + expect(await outbox.pendingCount(), 1); + expect(await outbox.dispatch(broker: broker), 1); + expect(await outbox.dispatch(broker: broker), 0); + expect(await outbox.pendingCount(), 0); + expect(await broker.pendingCount(queue), 1); + + final delivery = await broker + .consume(RoutingSubscription.singleQueue(queue)) + .first; + expect(delivery.envelope.id, envelope.id); + await broker.ack(delivery); + }, + ); + + test( + 'two relay workers claim committed rows without overlap', + () async { + final envelopes = [ + for (var index = 0; index < 8; index++) + Envelope( + id: 'concurrent-$index-${DateTime.now().microsecondsSinceEpoch}', + name: 'outbox.concurrent', + args: {'value': index}, + queue: queue, + ), + ]; + + await outbox.transaction((_) async { + for (final envelope in envelopes) { + await outboxBroker.publish(envelope); + } + }); + + final dispatched = await Future.wait([ + outbox.dispatch( + broker: broker, + limit: envelopes.length, + workerId: 'relay-a', + ), + outbox.dispatch( + broker: broker, + limit: envelopes.length, + workerId: 'relay-b', + ), + ]); + + expect( + dispatched.reduce((left, right) => left + right), + envelopes.length, + ); + expect(await outbox.pendingCount(), 0); + expect(await broker.pendingCount(queue), envelopes.length); + }, + ); + + test('rejects publication outside an outbox transaction', () async { + final envelope = Envelope( + name: 'outbox.invalid', + args: const {}, + queue: queue, + ); + + await expectLater( + outboxBroker.publish(envelope), + throwsStateError, + ); + }); + + test('records relay failures for a later leased retry', () async { + final envelope = Envelope( + id: 'failure-${DateTime.now().microsecondsSinceEpoch}', + name: 'outbox.failure', + args: const {}, + queue: queue, + ); + + await outbox.transaction((_) => outboxBroker.publish(envelope)); + + expect( + await outbox.dispatch( + broker: outboxBroker, + retryDelay: const Duration(hours: 1), + ), + 0, + ); + expect(await outbox.pendingCount(), 1); + expect(await broker.pendingCount(queue), 0); + }); + }, config: harness.config); +} diff --git a/packages/stem_postgres/test/integration/rate_limiting/postgres_rate_limiter_integration_test.dart b/packages/stem_postgres/test/integration/rate_limiting/postgres_rate_limiter_integration_test.dart new file mode 100644 index 00000000..f8fa6db2 --- /dev/null +++ b/packages/stem_postgres/test/integration/rate_limiting/postgres_rate_limiter_integration_test.dart @@ -0,0 +1,129 @@ +import 'dart:io'; + +import 'package:stem_postgres/stem_postgres.dart'; +import 'package:test/test.dart'; + +Future main() async { + final connectionString = Platform.environment['STEM_TEST_POSTGRES_URL']; + if (connectionString == null || connectionString.isEmpty) { + test( + 'Postgres rate limiter requires STEM_TEST_POSTGRES_URL', + () {}, + skip: 'Set STEM_TEST_POSTGRES_URL to run Postgres rate limiter tests.', + ); + return; + } + + group('postgres rate limiter', () { + late PostgresRateLimiter limiter; + late String namespace; + + setUp(() async { + namespace = 'rate-${DateTime.now().microsecondsSinceEpoch}'; + limiter = await PostgresRateLimiter.connect( + connectionString, + namespace: namespace, + ); + }); + + tearDown(() => limiter.close()); + + test('atomically allows capacity and returns a retry delay', () async { + final decisions = [ + for (var i = 0; i < 4; i++) + await limiter.acquire( + 'shared', + tokens: 3, + interval: const Duration(seconds: 1), + ), + ]; + + expect(decisions.take(3).every((decision) => decision.allowed), isTrue); + expect(decisions[3].allowed, isFalse); + expect(decisions[3].retryAfter, isNotNull); + expect(decisions[3].retryAfter, greaterThan(Duration.zero)); + expect(decisions[3].meta['backend'], 'postgres'); + }); + + test('refills a bucket using the database clock', () async { + expect( + (await limiter.acquire( + 'refill', + interval: const Duration(milliseconds: 100), + )).allowed, + isTrue, + ); + expect( + (await limiter.acquire( + 'refill', + interval: const Duration(milliseconds: 100), + )).allowed, + isFalse, + ); + + await Future.delayed(const Duration(milliseconds: 125)); + + expect( + (await limiter.acquire( + 'refill', + interval: const Duration(milliseconds: 100), + )).allowed, + isTrue, + ); + }); + + test('serializes concurrent acquisitions across connections', () async { + final secondConnection = await PostgresRateLimiter.connect( + connectionString, + namespace: namespace, + ); + addTearDown(secondConnection.close); + + final decisions = await Future.wait([ + for (var i = 0; i < 20; i++) + (i.isEven ? limiter : secondConnection).acquire( + 'concurrent', + tokens: 5, + // Keep refill outside the test window so the assertion proves + // row-lock serialization rather than timing-dependent refill. + interval: const Duration(hours: 1), + ), + ]); + + expect( + decisions.where((decision) => decision.allowed), + hasLength(5), + ); + expect( + decisions.where((decision) => !decision.allowed), + hasLength(15), + ); + }); + + test('separates namespaces and validates configuration', () async { + final other = await PostgresRateLimiter.connect( + connectionString, + namespace: '$namespace-other', + ); + addTearDown(other.close); + + expect( + (await limiter.acquire('key')).allowed, + isTrue, + ); + expect( + (await other.acquire('key')).allowed, + isTrue, + ); + + await expectLater( + limiter.acquire('invalid', tokens: 0), + throwsArgumentError, + ); + await expectLater( + limiter.acquire('invalid', interval: Duration.zero), + throwsArgumentError, + ); + }); + }); +} diff --git a/packages/stem_postgres/test/migration_upgrade_test.dart b/packages/stem_postgres/test/migration_upgrade_test.dart new file mode 100644 index 00000000..bd39458e --- /dev/null +++ b/packages/stem_postgres/test/migration_upgrade_test.dart @@ -0,0 +1,202 @@ +import 'dart:io'; + +import 'package:ormed/migrations.dart'; +import 'package:ormed/ormed.dart'; +import 'package:ormed_postgres/ormed_postgres.dart'; +import 'package:stem/stem.dart'; +import 'package:stem_postgres/src/database/datasource.dart'; +import 'package:stem_postgres/src/database/migrations.dart'; +import 'package:stem_postgres/stem_postgres.dart'; +import 'package:test/test.dart'; + +void main() { + final connectionString = Platform.environment['STEM_TEST_POSTGRES_URL']; + if (connectionString == null || connectionString.isEmpty) { + test( + 'Postgres migration upgrades require STEM_TEST_POSTGRES_URL', + () {}, + skip: 'Set STEM_TEST_POSTGRES_URL to run migration upgrade tests.', + ); + return; + } + + test( + 'upgrades every historical schema prefix to the current registry', + () async { + final migrations = buildMigrations(); + final admin = PostgresDriverAdapter.fromUrl(connectionString); + + try { + for ( + var prefixLength = 1; + prefixLength < migrations.length; + prefixLength += 1 + ) { + final schema = + 'stem_migration_' + '${DateTime.now().microsecondsSinceEpoch}_$prefixLength'; + await admin.createSchema(schema); + try { + final schemaUrl = _withSearchPath(connectionString, schema); + final oldAdapter = PostgresDriverAdapter.fromUrl(schemaUrl); + await oldAdapter.setCurrentSchema(schema); + await _createMigrationLedger(oldAdapter, schema); + final oldLedger = SqlMigrationLedger( + oldAdapter, + tableName: 'orm_migrations', + ); + final oldRunner = MigrationRunner( + schemaDriver: oldAdapter, + ledger: oldLedger, + migrations: migrations.take(prefixLength).toList(), + defaultSchema: schema, + emitEvents: false, + ); + final oldReport = await oldRunner.applyAll(); + expect( + oldReport.actions, + hasLength(prefixLength), + reason: 'old prefix length $prefixLength', + ); + await oldAdapter.close(); + + final currentAdapter = PostgresDriverAdapter.fromUrl(schemaUrl); + await currentAdapter.setCurrentSchema(schema); + final currentLedger = SqlMigrationLedger( + currentAdapter, + tableName: 'orm_migrations', + ); + final currentRunner = MigrationRunner( + schemaDriver: currentAdapter, + ledger: currentLedger, + migrations: migrations, + defaultSchema: schema, + emitEvents: false, + ); + final upgrade = await currentRunner.applyAll(); + expect( + upgrade.actions, + hasLength(migrations.length - prefixLength), + reason: 'prefix length $prefixLength', + ); + final statuses = await currentRunner.status(); + expect( + statuses.every((status) => status.applied), + isTrue, + reason: 'prefix length $prefixLength', + ); + await currentAdapter.close(); + } finally { + await admin.dropSchemaIfExists(schema); + } + } + } finally { + await admin.close(); + } + }, + ); + + test( + 'current additive schema accepts legacy-shaped lock writes', + () async { + final migrations = buildMigrations(); + final admin = PostgresDriverAdapter.fromUrl(connectionString); + final schema = + 'stem_mixed_version_${DateTime.now().microsecondsSinceEpoch}'; + await admin.createSchema(schema); + try { + final schemaUrl = _withSearchPath(connectionString, schema); + final adapter = PostgresDriverAdapter.fromUrl(schemaUrl); + await adapter.setCurrentSchema(schema); + await _createMigrationLedger(adapter, schema); + final runner = MigrationRunner( + schemaDriver: adapter, + ledger: SqlMigrationLedger(adapter, tableName: 'orm_migrations'), + migrations: migrations, + defaultSchema: schema, + emitEvents: false, + ); + await runner.applyAll(); + + final now = DateTime.now().toUtc(); + const namespace = 'legacy-worker'; + const key = 'legacy-lock'; + await adapter.executeRaw( + ''' +INSERT INTO stem_locks (key, namespace, owner, expires_at, created_at) +VALUES (?, ?, ?, ?, ?) +''', + [ + key, + namespace, + 'old-worker', + now.add(const Duration(seconds: 30)), + now, + ], + ); + + final dataSource = createDataSource(connectionString: schemaUrl); + await dataSource.init(); + final dataSourceDriver = dataSource.connection.driver as SchemaDriver; + await dataSourceDriver.setCurrentSchema(schema); + final store = await PostgresLockStore.fromDataSource( + dataSource, + namespace: namespace, + runMigrations: false, + ); + try { + expect( + await store.acquire(key, owner: 'new-worker'), + isNull, + reason: 'a current worker must respect an old active lock', + ); + + await adapter.executeRaw( + ''' +UPDATE stem_locks +SET expires_at = ? +WHERE key = ? AND namespace = ? +''', + [now.subtract(const Duration(seconds: 1)), key, namespace], + ); + final takeover = await store.acquire(key, owner: 'new-worker'); + expect(takeover, isA()); + expect((takeover! as FencedLock).fencingToken, equals(1)); + await takeover.release(); + } finally { + await store.close(); + await dataSource.dispose(); + } + await adapter.close(); + } finally { + await admin.dropSchemaIfExists(schema); + await admin.close(); + } + }, + ); +} + +String _withSearchPath(String url, String schema) { + final uri = Uri.parse(url); + final params = Map.from(uri.queryParameters); + params['options'] = '-c search_path=$schema,public'; + return uri.replace(queryParameters: params).toString(); +} + +Future _createMigrationLedger( + PostgresDriverAdapter adapter, + String schema, +) { + // The test database may already contain a public orm_migrations table from + // another package test. Seed the ledger in the isolated historical schema + // so the upgrade starts from the same ledger shape a released installation + // would have, rather than falling back through PostgreSQL's search_path. + return adapter.executeRaw(''' +CREATE TABLE "$schema"."orm_migrations" ( + "id" TEXT PRIMARY KEY, + "checksum" TEXT NOT NULL, + "applied_at" TIMESTAMPTZ NOT NULL, + "batch" INTEGER NOT NULL +) +'''); +} diff --git a/packages/stem_postgres/test/migrations_registry_test.dart b/packages/stem_postgres/test/migrations_registry_test.dart new file mode 100644 index 00000000..63d7c6cf --- /dev/null +++ b/packages/stem_postgres/test/migrations_registry_test.dart @@ -0,0 +1,28 @@ +import 'package:stem_postgres/src/database/migrations.dart'; +import 'package:test/test.dart'; + +void main() { + test( + 'migration registry is ordered, unique, and retains release history', + () { + final migrations = buildMigrations(); + final ids = migrations + .map((migration) => migration.id.toString()) + .toList(); + + expect(ids, isNotEmpty); + expect(ids.toSet(), hasLength(ids.length)); + expect(ids, orderedEquals([...ids]..sort())); + expect(ids.first, contains('m_20251227071920_stem')); + expect(ids, contains(contains('m_20260819090000_add_task_outbox'))); + expect( + ids, + contains(contains('m_20260819100000_add_rate_limit_buckets')), + ); + expect( + ids, + contains(contains('m_20260820110000_add_lock_fencing_tokens')), + ); + }, + ); +} diff --git a/packages/stem_redis/CHANGELOG.md b/packages/stem_redis/CHANGELOG.md index 6024c30f..35e59294 100644 --- a/packages/stem_redis/CHANGELOG.md +++ b/packages/stem_redis/CHANGELOG.md @@ -1,5 +1,33 @@ # Changelog +## Unreleased + +- Added an atomic Redis Lua terminal-result write that preserves the first + terminal task state across concurrent workers. + +## 0.2.0 + +- Updated the Redis adapter for Stem 0.3.0 and the capability-aware broker + contract. +- Retained the production distributed Redis rate limiter and added its focused + unit and integration coverage. +- Redis lock acquisition now allocates an atomic, monotonic fencing token while + retaining owner-checked renewal and release behavior. + +## 0.1.3 + +- Added the production `RedisRateLimiter`, an atomic Redis Lua token bucket + with Redis-server time, retry-after decisions, TLS support, and namespaced + keys. +- Removed the prerelease core dependency range from the published-package + manifest. + +## 0.1.2 + +- Added the production `RedisRateLimiter`, an atomic Redis Lua token bucket + with Redis-server time, retry-after decisions, TLS support, and namespaced + keys. + ## 0.1.1 - Updated the Redis workflow store to honor caller-provided run ids, matching diff --git a/packages/stem_redis/README.md b/packages/stem_redis/README.md index bd0e9671..9940bb07 100644 --- a/packages/stem_redis/README.md +++ b/packages/stem_redis/README.md @@ -89,6 +89,29 @@ Future main() async { } ``` +### Distributed rate limiting + +`RedisRateLimiter` shares a token bucket across worker processes. The +`RateLimit` capacity is the number of permits available per interval; each +acquire consumes one permit. + +```dart +final limiter = await RedisRateLimiter.connect( + 'rediss://localhost:6380/0', + namespace: 'billing-worker', + tls: const TlsConfig( + caCertificateFile: '/etc/ssl/certs/redis-ca.pem', + ), +); + +final workerConfig = StemWorkerConfig(rateLimiter: limiter); +``` + +Redis server time is used for refill calculations, and the Lua script performs +refill and acquisition atomically. A denied acquisition includes `retryAfter` +for the worker's retry scheduling. Close the limiter with the worker's other +resources. + ## Tests Integration suites require the dockerised Redis/Postgres stack provided by the diff --git a/packages/stem_redis/lib/src/backend/redis_backend.dart b/packages/stem_redis/lib/src/backend/redis_backend.dart index c8bc925a..29a3d569 100644 --- a/packages/stem_redis/lib/src/backend/redis_backend.dart +++ b/packages/stem_redis/lib/src/backend/redis_backend.dart @@ -7,7 +7,7 @@ import 'package:redis/redis.dart'; import 'package:stem/stem.dart'; /// Redis-backed implementation of [ResultBackend]. -class RedisResultBackend implements ResultBackend { +class RedisResultBackend implements ResultBackend, AtomicTerminalResultBackend { RedisResultBackend._( this._connection, this._command, { @@ -35,6 +35,9 @@ class RedisResultBackend implements ResultBackend { final Map> _watchers = {}; bool _closed = false; + @override + bool get supportsAtomicTerminalWrites => true; + /// Connects to Redis and returns a result backend instance. static Future connect( String uri, { @@ -163,6 +166,49 @@ class RedisResultBackend implements ResultBackend { _watchers[taskId]?.add(status); } + @override + Future setTerminalIfAbsent( + TaskStatus status, { + Duration? ttl, + }) async { + final key = _taskKey(status.id); + final now = stemNow().toUtc(); + final value = { + ...status.toJson(), + 'createdAt': now.toIso8601String(), + 'updatedAt': now.toIso8601String(), + }; + final expire = (ttl ?? defaultTtl).inMilliseconds; + final result = await _send([ + 'EVAL', + ''' +local current = redis.call('GET', KEYS[1]) +local value = cjson.decode(ARGV[1]) +if current then + local decoded = cjson.decode(current) + local state = decoded['state'] + if state == 'succeeded' or state == 'failed' or state == 'cancelled' then + return 0 + end + if decoded['createdAt'] then + value['createdAt'] = decoded['createdAt'] + end +end +redis.call('SET', KEYS[1], cjson.encode(value), 'PX', ARGV[2]) +return 1 +''', + '1', + key, + jsonEncode(value), + expire.toString(), + ]); + final applied = result == 1 || result == '1'; + if (applied) { + _watchers[status.id]?.add(status); + } + return applied; + } + @override Future get(String taskId) async { final raw = await _send(['GET', _taskKey(taskId)]); diff --git a/packages/stem_redis/lib/src/brokers/redis_broker.dart b/packages/stem_redis/lib/src/brokers/redis_broker.dart index 586030a7..f2d37ceb 100644 --- a/packages/stem_redis/lib/src/brokers/redis_broker.dart +++ b/packages/stem_redis/lib/src/brokers/redis_broker.dart @@ -7,7 +7,13 @@ import 'package:stem/stem.dart'; import 'package:uuid/uuid.dart'; /// Redis streams-backed implementation of [Broker]. -class RedisStreamsBroker implements Broker { +class RedisStreamsBroker + implements + Broker, + LeaseBroker, + InspectableBroker, + DeadLetterBroker, + BrokerCapabilitiesProvider { /// Creates a broker instance using injected [connection] and [command]. /// /// This is intended for unit tests that need to stub Redis behaviour without @@ -172,10 +178,7 @@ class RedisStreamsBroker implements Broker { } Object? failure; StackTrace? failureStack; - await runZonedGuarded(() => _connection.close(), ( - Object error, - StackTrace stack, - ) { + await runZonedGuarded(() => _connection.close(), (error, stack) { if (_shouldSuppressClosedError(error)) { return; } @@ -305,6 +308,17 @@ class RedisStreamsBroker implements Broker { @override bool get supportsPriority => true; + @override + BrokerCapabilities get capabilities => const BrokerCapabilities( + supportsDelayedDelivery: true, + supportsPriorityOrdering: true, + deliveryGuarantee: BrokerDeliveryGuarantee.atLeastOnce, + supportsQueueInspection: true, + supportsLeaseExtension: true, + supportsDeadLettering: true, + supportsDeadLetterReplay: true, + ); + Future _ensureGroupForStream(String queue, String streamKey) async { final key = '$streamKey|${_groupKey(queue)}'; if (_groupsCreated.contains(key)) return; @@ -898,6 +912,16 @@ class RedisStreamsBroker implements Broker { String _claimTimerKey(String stream, String group, String consumer) => '$stream|$group|$consumer'; + // The timer controls how often we look for stalled deliveries. Redis + // should only claim a delivery after its visibility lease has elapsed. + // Keeping these values separate prevents a short polling cadence from + // turning into an unexpectedly short lease. + Duration get _claimIdleThreshold { + return defaultVisibilityTimeout > claimInterval + ? defaultVisibilityTimeout + : claimInterval; + } + String _scheduleClaim( String stream, String group, @@ -917,7 +941,7 @@ class RedisStreamsBroker implements Broker { stream, group, consumer, - claimInterval.inMilliseconds.toString(), + _claimIdleThreshold.inMilliseconds.toString(), '0-0', 'COUNT', delayedDrainBatch.toString(), @@ -1112,17 +1136,28 @@ class RedisStreamsBroker implements Broker { @override Future extendLease(Delivery delivery, Duration by) async { + if (delivery.route.isBroadcast || by <= Duration.zero) { + return; + } final info = _parseReceipt(delivery.receipt); - await _send(['XACK', info.stream, info.group, info.id]); - final nextVisibleAt = stemNow().add(by); - final delayedEnvelope = delivery.envelope.copyWith( - notBefore: nextVisibleAt, - ); + // Redis Streams model visibility as the pending entry's idle time. Reset + // the delivery timestamp in place instead of ACKing and publishing a + // delayed copy, which would allow the original pending entry to be + // claimed while the task is still executing. + // + // XAUTOCLAIM uses [_claimIdleThreshold] as its minimum idle threshold. A + // future delivery timestamp is therefore needed when [by] exceeds that + // threshold so the entry remains invisible for the full requested lease. + final deliveryTime = stemNow().subtract(_claimIdleThreshold - by); await _send([ - 'ZADD', - _delayedKey(delayedEnvelope.queue), - nextVisibleAt.millisecondsSinceEpoch.toString(), - jsonEncode(delayedEnvelope.toJson()), + 'XCLAIM', + info.stream, + info.group, + info.consumer, + '0', + info.id, + 'TIME', + deliveryTime.millisecondsSinceEpoch.toString(), ]); } diff --git a/packages/stem_redis/lib/src/rate_limiting/redis_rate_limiter.dart b/packages/stem_redis/lib/src/rate_limiting/redis_rate_limiter.dart new file mode 100644 index 00000000..5d5da024 --- /dev/null +++ b/packages/stem_redis/lib/src/rate_limiting/redis_rate_limiter.dart @@ -0,0 +1,242 @@ +import 'dart:async'; +import 'dart:io'; + +import 'package:redis/redis.dart'; +import 'package:stem/stem.dart'; + +/// Factory used to inject a Redis command connection into tests. +typedef RedisRateLimiterCommandFactory = + Future<({RedisConnection connection, Command command})> Function( + Uri uri, + TlsConfig? tls, + ); + +/// Redis-backed distributed token-bucket rate limiter. +/// +/// [RateLimit.tokens] is the bucket capacity and each successful [acquire] +/// consumes one permit. Refill, consume, and retry calculation happen in one +/// Redis Lua evaluation, so multiple worker processes share one atomic limit. +/// Redis server time is used instead of the Dart process clock. +class RedisRateLimiter implements RateLimiter { + RedisRateLimiter._( + this._connection, + this._command, { + required this.namespace, + }); + + /// Creates a limiter with injected Redis handles for tests and advanced + /// connection management. + factory RedisRateLimiter.test({ + required RedisConnection connection, + required Command command, + String namespace = 'stem', + }) { + return RedisRateLimiter._( + connection, + command, + namespace: _normalizeNamespace(namespace), + ); + } + + /// Opens a Redis-backed limiter from [uri]. + /// + /// `redis://` and `rediss://` URLs are supported. Passwords and database + /// numbers embedded in the URL are applied before the limiter is returned. + /// [commandFactory] is intended for tests; production callers should omit + /// it. + static Future connect( + String uri, { + String namespace = 'stem', + TlsConfig? tls, + RedisRateLimiterCommandFactory? commandFactory, + }) async { + final parsed = Uri.parse(uri); + final handle = await (commandFactory ?? _defaultCommandFactory)( + parsed, + tls, + ); + final resolvedNamespace = parsed.queryParameters['ns'] ?? namespace; + return RedisRateLimiter._( + handle.connection, + handle.command, + namespace: _normalizeNamespace(resolvedNamespace), + ); + } + + /// Atomic refill-and-acquire script. + /// + /// The hash stores fractional permits and the last Redis-server timestamp. + /// A key expires after two windows so inactive limit keys do not accumulate. + static const String script = ''' +local key = KEYS[1] +local capacity = tonumber(ARGV[1]) +local interval_ms = tonumber(ARGV[2]) + +local server_time = redis.call('TIME') +local now_ms = tonumber(server_time[1]) * 1000 + math.floor(tonumber(server_time[2]) / 1000) +local stored_tokens = redis.call('HGET', key, 'tokens') +local stored_at = redis.call('HGET', key, 'at') +local available = capacity + +if stored_tokens and stored_at then + local elapsed = math.max(0, now_ms - tonumber(stored_at)) + available = math.min(capacity, tonumber(stored_tokens) + (elapsed * capacity / interval_ms)) +end + +local allowed = 0 +local retry_ms = 0 +if available >= 1 then + available = available - 1 + allowed = 1 +else + retry_ms = math.ceil((1 - available) * interval_ms / capacity) +end + +redis.call('HSET', key, 'tokens', available, 'at', now_ms) +redis.call('PEXPIRE', key, math.max(interval_ms * 2, 1000)) +return {allowed, retry_ms, available} +'''; + + final RedisConnection _connection; + final Command _command; + bool _closed = false; + + /// Namespace used to isolate limiter keys. + final String namespace; + + /// Returns the Redis key used for [key]. + String keyFor(String key) => '$namespace:rate:$key'; + + @override + Future acquire( + String key, { + int tokens = 1, + Duration? interval, + Map? meta, + }) async { + if (tokens <= 0) { + throw ArgumentError.value(tokens, 'tokens', 'Capacity must be positive.'); + } + final window = interval ?? const Duration(seconds: 1); + if (window <= Duration.zero) { + throw ArgumentError.value( + window, + 'interval', + 'Rate-limit interval must be positive.', + ); + } + final intervalMs = window.inMilliseconds; + if (intervalMs <= 0) { + throw ArgumentError.value( + window, + 'interval', + 'Rate-limit interval must be at least one millisecond.', + ); + } + + final response = await _command.send_object([ + 'EVAL', + script, + 1, + keyFor(key), + tokens, + intervalMs, + ]); + if (response is! List || response.length < 3) { + throw StateError( + 'Unexpected response from Redis rate limiter: $response', + ); + } + + final allowed = _asInt(response[0]) == 1; + final retryMs = _asInt(response[1]); + final remaining = _asDouble(response[2]); + return RateLimitDecision( + allowed: allowed, + retryAfter: allowed || retryMs <= 0 + ? null + : Duration(milliseconds: retryMs), + meta: { + 'capacity': tokens, + 'intervalMs': intervalMs, + 'remainingTokens': remaining, + ...?meta, + }, + ); + } + + /// Closes the Redis connection owned by this limiter. + Future close() async { + if (_closed) return; + _closed = true; + await _connection.close(); + } + + static Future<({RedisConnection connection, Command command})> + _defaultCommandFactory(Uri parsed, TlsConfig? tls) async { + final host = parsed.host.isNotEmpty ? parsed.host : 'localhost'; + final port = parsed.hasPort ? parsed.port : 6379; + final connection = RedisConnection(); + final scheme = parsed.scheme.isEmpty ? 'redis' : parsed.scheme; + Command command; + if (scheme == 'rediss') { + final securityContext = tls?.toSecurityContext(); + try { + final socket = await SecureSocket.connect( + host, + port, + context: securityContext, + onBadCertificate: tls?.allowInsecure ?? false ? (_) => true : null, + timeout: const Duration(seconds: 5), + ); + command = await connection.connectWithSocket(socket); + } on HandshakeException catch (error, stack) { + logTlsHandshakeFailure( + component: 'redis rate limiter', + host: host, + port: port, + config: tls, + error: error, + stack: stack, + ); + await connection.close(); + rethrow; + } + } else { + command = await connection.connect(host, port); + } + + try { + if (parsed.userInfo.isNotEmpty) { + final parts = parsed.userInfo.split(':'); + final password = parts.length == 2 ? parts[1] : parts[0]; + await command.send_object(['AUTH', password]); + } + if (parsed.pathSegments.isNotEmpty) { + final db = int.tryParse(parsed.pathSegments.first); + if (db != null) { + await command.send_object(['SELECT', db]); + } + } + } on Object { + await connection.close(); + rethrow; + } + return (connection: connection, command: command); + } +} + +String _normalizeNamespace(String value) { + final trimmed = value.trim(); + return trimmed.isEmpty ? 'stem' : trimmed; +} + +int _asInt(Object? value) { + if (value is num) return value.toInt(); + return int.parse(value.toString()); +} + +double _asDouble(Object? value) { + if (value is num) return value.toDouble(); + return double.parse(value.toString()); +} diff --git a/packages/stem_redis/lib/src/scheduler/redis_lock_store.dart b/packages/stem_redis/lib/src/scheduler/redis_lock_store.dart index d77c5fff..8bfc3904 100644 --- a/packages/stem_redis/lib/src/scheduler/redis_lock_store.dart +++ b/packages/stem_redis/lib/src/scheduler/redis_lock_store.dart @@ -77,6 +77,9 @@ class RedisLockStore implements LockStore { } String _key(String key) => '$namespace:lock:$key'; + + String _tokenKey(String key) => '$namespace:lock-token:$key'; + String _owner(String? owner) => owner ?? const Uuid().v7(); Future _send(List command) => _command.send_object(command); @@ -88,24 +91,48 @@ class RedisLockStore implements LockStore { String? owner, }) async { final redisKey = _key(key); + final tokenKey = _tokenKey(key); final value = _owner(owner); + // Keep the acquisition and token allocation atomic. The counter is a + // separate key so deleting an expired lock never reuses its token. + const script = ''' +if redis.call("EXISTS", KEYS[1]) == 0 then + local token = redis.call("INCR", KEYS[2]) + redis.call("HSET", KEYS[1], "owner", ARGV[1], "token", token) + redis.call("PEXPIRE", KEYS[1], ARGV[2]) + return token +else + return 0 +end +'''; final result = await _send([ - 'SET', + 'EVAL', + script, + '2', redisKey, + tokenKey, value, - 'NX', - 'PX', ttl.inMilliseconds.toString(), ]); - if (result != 'OK') { + final fencingToken = _asInt(result); + if (fencingToken <= 0) { return null; } - return _RedisLock(store: this, key: key, redisKey: redisKey, owner: value); + return _RedisLock( + store: this, + key: key, + redisKey: redisKey, + owner: value, + fencingToken: fencingToken, + ); } Future _renew(String redisKey, String owner, Duration ttl) async { const script = ''' -if redis.call("GET", KEYS[1]) == ARGV[1] then +local kind = redis.call("TYPE", KEYS[1]).ok +if kind == "hash" and redis.call("HGET", KEYS[1], "owner") == ARGV[1] then + return redis.call("PEXPIRE", KEYS[1], ARGV[2]) +elseif kind == "string" and redis.call("GET", KEYS[1]) == ARGV[1] then return redis.call("PEXPIRE", KEYS[1], ARGV[2]) else return 0 @@ -124,7 +151,10 @@ end Future _release(String redisKey, String owner) async { const script = ''' -if redis.call("GET", KEYS[1]) == ARGV[1] then +local kind = redis.call("TYPE", KEYS[1]).ok +local matches = (kind == "hash" and redis.call("HGET", KEYS[1], "owner") == ARGV[1]) + or (kind == "string" and redis.call("GET", KEYS[1]) == ARGV[1]) +if matches then return redis.call("DEL", KEYS[1]) else return 0 @@ -136,7 +166,11 @@ end @override Future ownerOf(String key) async { - final result = await _send(['GET', _key(key)]); + final redisKey = _key(key); + final type = await _send(['TYPE', redisKey]); + final result = type == 'hash' + ? await _send(['HGET', redisKey, 'owner']) + : await _send(['GET', redisKey]); return result is String ? result : result?.toString(); } @@ -146,14 +180,20 @@ end @override Future release(String key, String owner) => _release(_key(key), owner); + + static int _asInt(Object? value) { + if (value is int) return value; + return int.tryParse(value?.toString() ?? '') ?? 0; + } } -class _RedisLock implements Lock { +class _RedisLock implements FencedLock { _RedisLock({ required this.store, required this.key, required this.redisKey, required this.owner, + required this.fencingToken, }); final RedisLockStore store; @@ -163,6 +203,9 @@ class _RedisLock implements Lock { @override final String owner; + @override + final int fencingToken; + @override Future renew(Duration ttl) => store._renew(redisKey, owner, ttl); diff --git a/packages/stem_redis/lib/src/workflow/redis_workflow_store.dart b/packages/stem_redis/lib/src/workflow/redis_workflow_store.dart index 064862e5..d5232d2a 100644 --- a/packages/stem_redis/lib/src/workflow/redis_workflow_store.dart +++ b/packages/stem_redis/lib/src/workflow/redis_workflow_store.dart @@ -348,9 +348,9 @@ return 1 @override Future createRun({ - String? runId, required String workflow, required Map params, + String? runId, String? parentRunId, Duration? ttl, WorkflowCancellationPolicy? cancellationPolicy, @@ -374,9 +374,10 @@ return 1 nowIso, '', '', - cancellationPolicy != null && !cancellationPolicy.isEmpty - ? jsonEncode(cancellationPolicy.toJson()) - : '', + if (cancellationPolicy != null && !cancellationPolicy.isEmpty) + jsonEncode(cancellationPolicy.toJson()) + else + '', ]); if (result != 1 && result != '1') { throw StateError('Workflow run "$id" already exists.'); diff --git a/packages/stem_redis/lib/stem_redis.dart b/packages/stem_redis/lib/stem_redis.dart index 97b99465..580cc7b4 100644 --- a/packages/stem_redis/lib/stem_redis.dart +++ b/packages/stem_redis/lib/stem_redis.dart @@ -3,6 +3,8 @@ export 'src/brokers/redis_broker.dart' show RedisStreamsBroker; export 'src/control/redis_revoke_store.dart' show RedisRevokeStore; export 'src/observability/redis_heartbeat_transport.dart' show RedisHeartbeatCommandFactory, RedisHeartbeatTransport; +export 'src/rate_limiting/redis_rate_limiter.dart' + show RedisRateLimiter, RedisRateLimiterCommandFactory; export 'src/scheduler/redis_lock_store.dart' show RedisLockStore; export 'src/scheduler/redis_schedule_store.dart' show RedisScheduleStore; export 'src/stack/redis_adapter.dart' diff --git a/packages/stem_redis/pubspec.yaml b/packages/stem_redis/pubspec.yaml index 3cdf8170..8f6f9e39 100644 --- a/packages/stem_redis/pubspec.yaml +++ b/packages/stem_redis/pubspec.yaml @@ -1,6 +1,6 @@ name: stem_redis description: Redis broker, result backend, and scheduler utilities for Stem. -version: 0.1.1 +version: 0.2.0 repository: https://github.com/kingwill101/stem resolution: workspace environment: @@ -10,12 +10,12 @@ dependencies: async: ^2.13.0 collection: ^1.19.1 redis: ^4.0.0 - stem: ">=0.2.0-dev <0.3.0" + stem: ">=0.3.0 <0.4.0" uuid: ^4.5.2 dev_dependencies: coverage: ^1.15.0 lints: ^6.0.0 - stem_adapter_tests: ^0.1.1 + stem_adapter_tests: ^0.2.0 test: ^1.29.0 very_good_analysis: ^10.0.0 diff --git a/packages/stem_redis/test/chaos/worker_resilience_test.dart b/packages/stem_redis/test/chaos/worker_resilience_test.dart index 10d17f24..e88cbf62 100644 --- a/packages/stem_redis/test/chaos/worker_resilience_test.dart +++ b/packages/stem_redis/test/chaos/worker_resilience_test.dart @@ -1,6 +1,7 @@ import 'dart:async'; import 'dart:io'; +import 'package:stem/memory.dart'; import 'package:stem/stem.dart'; import 'package:stem_redis/stem_redis.dart'; import 'package:test/test.dart'; diff --git a/packages/stem_redis/test/integration/brokers/redis_broker_integration_test.dart b/packages/stem_redis/test/integration/brokers/redis_broker_integration_test.dart index 21ce6caa..f4a24fa9 100644 --- a/packages/stem_redis/test/integration/brokers/redis_broker_integration_test.dart +++ b/packages/stem_redis/test/integration/brokers/redis_broker_integration_test.dart @@ -378,7 +378,7 @@ String _uniqueNamespace() { Future _safeCloseRedisBroker(RedisStreamsBroker broker) async { try { - await runZonedGuarded(() => broker.close(), (Object _, StackTrace _) {}); + await runZonedGuarded(() => broker.close(), (_, _) {}); } on Object { // Ignore broker shutdown errors in cleanup. } diff --git a/packages/stem_redis/test/integration/observability/redis_heartbeat_transport_integration_test.dart b/packages/stem_redis/test/integration/observability/redis_heartbeat_transport_integration_test.dart index e7abd7c2..e9b525b9 100644 --- a/packages/stem_redis/test/integration/observability/redis_heartbeat_transport_integration_test.dart +++ b/packages/stem_redis/test/integration/observability/redis_heartbeat_transport_integration_test.dart @@ -6,17 +6,23 @@ import 'package:test/test.dart'; void main() { const namespace = 'integration-heartbeat'; - const redisUrl = 'redis://127.0.0.1:56379'; - const redissUrl = 'rediss://127.0.0.1:56379'; + final redisUrl = + Platform.environment['STEM_TEST_REDIS_URL'] ?? 'redis://127.0.0.1:56379'; + final parsedRedisUrl = Uri.parse(redisUrl); + final redissUrl = parsedRedisUrl.replace(scheme: 'rediss').toString(); final tlsUrl = Platform.environment['STEM_TEST_REDIS_TLS_URL']; final tlsCa = Platform.environment['STEM_TEST_REDIS_TLS_CA_CERT']; Future ensureRedisAvailable() async { - final available = await _canConnect('127.0.0.1', 56379); + final available = await _canConnect( + parsedRedisUrl.host, + parsedRedisUrl.port == 0 ? 6379 : parsedRedisUrl.port, + ); if (!available) { fail( - 'Redis test service is not reachable on 127.0.0.1:56379. ' - 'Start docker/testing/docker-compose.yml before running integration tests.', + 'Redis test service is not reachable at $redisUrl. ' + 'Set STEM_TEST_REDIS_URL or start docker/testing/docker-compose.yml ' + 'before running integration tests.', ); } } diff --git a/packages/stem_redis/test/integration/rate_limiting/redis_rate_limiter_integration_test.dart b/packages/stem_redis/test/integration/rate_limiting/redis_rate_limiter_integration_test.dart new file mode 100644 index 00000000..bfbf554e --- /dev/null +++ b/packages/stem_redis/test/integration/rate_limiting/redis_rate_limiter_integration_test.dart @@ -0,0 +1,76 @@ +import 'dart:io'; + +import 'package:stem/stem.dart'; +import 'package:stem_redis/stem_redis.dart'; +import 'package:test/test.dart'; + +void main() { + final redisUrl = Platform.environment['STEM_TEST_REDIS_URL']; + if (redisUrl == null || redisUrl.isEmpty) { + test( + 'Redis rate limiter integration requires STEM_TEST_REDIS_URL', + () {}, + skip: 'Set STEM_TEST_REDIS_URL to run Redis rate limiter tests.', + ); + return; + } + + late RedisRateLimiter limiter; + + setUp(() async { + limiter = await RedisRateLimiter.connect( + redisUrl, + namespace: 'rate-test-${DateTime.now().microsecondsSinceEpoch}', + ); + }); + + tearDown(() => limiter.close()); + + test('shares one atomic bucket across concurrent acquires', () async { + final decisions = await Future.wait( + List.generate( + 20, + (_) => limiter.acquire( + 'shared', + tokens: 5, + interval: const Duration(seconds: 1), + ), + ), + ); + + expect(decisions.where((decision) => decision.allowed), hasLength(5)); + expect( + decisions.where((decision) => !decision.allowed), + everyElement( + predicate( + (decision) => decision.retryAfter != null, + ), + ), + ); + }); + + test('refills permits and reports retry delay', () async { + final first = await limiter.acquire( + 'refill', + interval: const Duration(milliseconds: 100), + ); + final denied = await limiter.acquire( + 'refill', + interval: const Duration(milliseconds: 100), + ); + + expect(first.allowed, isTrue); + expect(denied.allowed, isFalse); + final retryAfter = denied.retryAfter; + expect(retryAfter, isNotNull); + await Future.delayed(retryAfter! + const Duration(milliseconds: 25)); + + expect( + (await limiter.acquire( + 'refill', + interval: const Duration(milliseconds: 100), + )).allowed, + isTrue, + ); + }); +} diff --git a/packages/stem_redis/test/integration/redis/redis_real_test.dart b/packages/stem_redis/test/integration/redis/redis_real_test.dart index 5f00b1f8..76630f59 100644 --- a/packages/stem_redis/test/integration/redis/redis_real_test.dart +++ b/packages/stem_redis/test/integration/redis/redis_real_test.dart @@ -128,6 +128,32 @@ void main() async { expect(expired, isNull); }); + test('terminal writes arbitrate concurrent completion attempts', () async { + const taskId = 'integration-terminal-race'; + await backend.set(taskId, TaskState.running); + final applied = await backend.setTerminalIfAbsent( + TaskStatus( + id: taskId, + state: TaskState.succeeded, + payload: 'first', + attempt: 1, + ), + ); + final rejected = await backend.setTerminalIfAbsent( + TaskStatus( + id: taskId, + state: TaskState.failed, + error: const TaskError(type: 'late', message: 'late failure'), + attempt: 1, + ), + ); + + expect(applied, isTrue); + expect(rejected, isFalse); + expect((await backend.get(taskId))?.payload, 'first'); + expect((await backend.get(taskId))?.state, TaskState.succeeded); + }); + test('RedisResultBackend namespaces isolate data', () async { const taskId = 'integration-namespace-task'; final other = await RedisResultBackend.connect( @@ -206,7 +232,10 @@ void main() async { final afterRelease = await lockStore.acquire('integration-lock'); expect(afterRelease, isNotNull); - await afterRelease!.release(); + final firstFenced = lock as FencedLock; + final secondFenced = afterRelease! as FencedLock; + expect(secondFenced.fencingToken, greaterThan(firstFenced.fencingToken)); + await secondFenced.release(); }); test('UniqueTaskCoordinator deduplicates with RedisLockStore', () async { diff --git a/packages/stem_redis/test/integration/redis/redis_tls_integration_test.dart b/packages/stem_redis/test/integration/redis/redis_tls_integration_test.dart index a9ab9e2f..f3d76caa 100644 --- a/packages/stem_redis/test/integration/redis/redis_tls_integration_test.dart +++ b/packages/stem_redis/test/integration/redis/redis_tls_integration_test.dart @@ -28,6 +28,8 @@ void _runRedisTlsSuite({ }) { final trimmedUrl = url?.trim(); final trimmedCa = caCertPath?.trim(); + var resolvedClientCert = clientCertPath; + var resolvedClientKey = clientKeyPath; if (trimmedUrl == null || trimmedUrl.isEmpty) { test( 'redis $label integration skipped', @@ -87,14 +89,12 @@ void _runRedisTlsSuite({ ); return; } - clientCertPath = trimmedClientCert; - clientKeyPath = trimmedClientKey; + resolvedClientCert = trimmedClientCert; + resolvedClientKey = trimmedClientKey; } final resolvedUrl = trimmedUrl; final resolvedCa = trimmedCa; - final resolvedClientCert = clientCertPath; - final resolvedClientKey = clientKeyPath; final tls = TlsConfig( caCertificateFile: resolvedCa, clientCertificateFile: resolvedClientCert, diff --git a/packages/stem_redis/test/unit/brokers/redis_broker_test.dart b/packages/stem_redis/test/unit/brokers/redis_broker_test.dart index d263a3a0..dc97190c 100644 --- a/packages/stem_redis/test/unit/brokers/redis_broker_test.dart +++ b/packages/stem_redis/test/unit/brokers/redis_broker_test.dart @@ -65,6 +65,40 @@ void main() { expect(delTargets.contains('unit:dead:emails'), isTrue); }); + test( + 'extends a pending entry in place without publishing a duplicate', + () async { + final connection = FakeRedisConnection(); + final command = FakeRedisCommand(connection); + final broker = RedisStreamsBroker.test( + connection: connection, + command: command, + namespace: 'unit', + claimInterval: const Duration(seconds: 1), + ); + final delivery = Delivery( + envelope: Envelope(name: 'lease.test', args: const {}), + receipt: 'unit:stream:default|unit:group:default|consumer|1-0', + leaseExpiresAt: DateTime.now().add(const Duration(seconds: 1)), + ); + + await broker.extendLease(delivery, const Duration(seconds: 5)); + + expect(command.sent, hasLength(1)); + expect(command.sent.single, contains('XCLAIM')); + expect(command.sent.single, contains('TIME')); + expect( + command.sent.where((command) => command.first == 'XACK'), + isEmpty, + ); + expect( + command.sent.where((command) => command.first == 'ZADD'), + isEmpty, + ); + await broker.close(); + }, + ); + test('subscription cancellation tears down claim timers', () async { final connection = FakeRedisConnection(); final command = FakeRedisCommand(connection); diff --git a/packages/stem_redis/test/unit/rate_limiting/redis_rate_limiter_test.dart b/packages/stem_redis/test/unit/rate_limiting/redis_rate_limiter_test.dart new file mode 100644 index 00000000..c869921c --- /dev/null +++ b/packages/stem_redis/test/unit/rate_limiting/redis_rate_limiter_test.dart @@ -0,0 +1,80 @@ +import 'package:stem_redis/stem_redis.dart'; +import 'package:test/test.dart'; + +import '../../support/fakes/fake_redis.dart'; + +void main() { + group('RedisRateLimiter', () { + test( + 'evaluates the atomic token-bucket script and parses grants', + () async { + final connection = FakeRedisConnection(); + final command = FakeRedisCommand(connection) + ..queueResponse((_) => [1, 0, 1.5]); + final limiter = RedisRateLimiter.test( + connection: connection, + command: command, + namespace: 'unit', + ); + + final decision = await limiter.acquire( + 'tenant-a', + tokens: 3, + interval: const Duration(seconds: 2), + meta: const {'source': 'test'}, + ); + + expect(decision.allowed, isTrue); + expect(decision.retryAfter, isNull); + expect(decision.meta['capacity'], 3); + expect(decision.meta['remainingTokens'], 1.5); + expect(decision.meta['source'], 'test'); + expect(command.sent, hasLength(1)); + expect(command.sent.single.first, 'EVAL'); + expect(command.sent.single[3], 'unit:rate:tenant-a'); + expect(command.sent.single[4], 3); + expect(command.sent.single[5], 2000); + expect(command.sent.single[1], contains("redis.call('TIME')")); + + await limiter.close(); + expect(connection.closed, isTrue); + }, + ); + + test('parses denied decisions with retry-after', () async { + final connection = FakeRedisConnection(); + final command = FakeRedisCommand(connection) + ..queueResponse((_) => [0, 125, 0.25]); + final limiter = RedisRateLimiter.test( + connection: connection, + command: command, + ); + + final decision = await limiter.acquire('busy'); + + expect(decision.allowed, isFalse); + expect(decision.retryAfter, const Duration(milliseconds: 125)); + expect(decision.meta['remainingTokens'], 0.25); + }); + + test('rejects invalid capacity, interval, and Redis responses', () async { + final connection = FakeRedisConnection(); + final command = FakeRedisCommand(connection) + ..queueResponse((_) => const [1]); + final limiter = RedisRateLimiter.test( + connection: connection, + command: command, + ); + + expect( + () => limiter.acquire('invalid', tokens: 0), + throwsArgumentError, + ); + expect( + () => limiter.acquire('invalid', interval: Duration.zero), + throwsArgumentError, + ); + await expectLater(limiter.acquire('invalid'), throwsStateError); + }); + }); +} diff --git a/packages/stem_sqlite/CHANGELOG.md b/packages/stem_sqlite/CHANGELOG.md index 908e2ddc..739a185a 100644 --- a/packages/stem_sqlite/CHANGELOG.md +++ b/packages/stem_sqlite/CHANGELOG.md @@ -1,5 +1,34 @@ # Changelog +## Unreleased + +- Datasource logger injection now uses Stem's dependency-neutral `StemLogger` + facade; the Ormed contextual logger remains an adapter implementation detail. +- Serialized SQLite transactions and broker mutations across shared file + connections to prevent savepoint corruption and `database is locked` errors + under concurrent worker/result-backend traffic. +- Added a file-backed throughput benchmark with a regression baseline. +- Added conditional terminal-result updates so late completion attempts cannot + overwrite an existing terminal state. +- Added a restart-recovery integration test that reopens a durable SQLite + workflow store and resumes a persisted checkpoint with replacement broker, + runtime, and worker instances. +- Added mixed-version queue coverage: a row written against the pre-namespace + schema is upgraded and consumed by the current SQLite broker. + +## 0.2.0 + +- Updated the adapter for Stem 0.3.0 and the narrowed broker capability model. +- Added migration-registry and historical-schema upgrade coverage. +- Removed the deprecated `StemSqliteDatabase` placeholder API; use + `SqliteConnections` and `QueryContext` instead. + +## 0.1.4 + +- Moved shared logger imports to the explicit + `package:stem/observability.dart` entrypoint as the stable core barrel was + narrowed. + ## 0.1.3 - Updated `ormed_sqlite` to 0.3.0 and raised the package SDK floor to Dart diff --git a/packages/stem_sqlite/analysis_options.yaml b/packages/stem_sqlite/analysis_options.yaml index 35a8662e..2946997f 100644 --- a/packages/stem_sqlite/analysis_options.yaml +++ b/packages/stem_sqlite/analysis_options.yaml @@ -15,6 +15,10 @@ include: package:very_good_analysis/analysis_options.yaml analyzer: + # Ormed invokes the CLI `main`, while applications may call this registry + # programmatically through `runProjectSeeds`. + errors: + unreachable_from_main: ignore exclude: - "**/*.g.dart" - "**/*.orm.dart" diff --git a/packages/stem_sqlite/benchmark/sqlite_throughput.dart b/packages/stem_sqlite/benchmark/sqlite_throughput.dart new file mode 100644 index 00000000..b9b1204e --- /dev/null +++ b/packages/stem_sqlite/benchmark/sqlite_throughput.dart @@ -0,0 +1,132 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:stem/stem.dart'; +import 'package:stem_sqlite/stem_sqlite.dart'; + +Future main(List args) async { + final tasks = _intOption(args, '--tasks') ?? 1000; + final concurrency = _intOption(args, '--concurrency') ?? 4; + final checkBaseline = args.contains('--check-baseline'); + if (tasks <= 0 || concurrency <= 0) { + throw ArgumentError('Tasks and concurrency must be positive.'); + } + + final directory = await Directory.systemTemp.createTemp( + 'stem-sqlite-throughput-', + ); + final file = File('${directory.path}/stem.db'); + final broker = await SqliteBroker.open( + file, + namespace: 'benchmark', + pollInterval: const Duration(milliseconds: 5), + sweeperInterval: const Duration(hours: 1), + ); + final backend = await SqliteResultBackend.open( + file, + namespace: 'benchmark', + cleanupInterval: const Duration(hours: 1), + ); + final completed = Completer(); + var completedTasks = 0; + final registry = InMemoryTaskRegistry() + ..register( + _BenchmarkTask( + onComplete: () { + completedTasks += 1; + if (completedTasks == tasks && !completed.isCompleted) { + completed.complete(); + } + }, + ), + ); + final stem = Stem(broker: broker, registry: registry, backend: backend); + final worker = Worker( + broker: broker, + registry: registry, + backend: backend, + concurrency: concurrency, + prefetchMultiplier: 1, + heartbeatTransport: const NoopHeartbeatTransport(), + consumerName: 'sqlite-benchmark-worker', + lifecycle: const WorkerLifecycleConfig(installSignalHandlers: false), + ); + + try { + await worker.start(); + final total = Stopwatch()..start(); + final enqueue = Stopwatch()..start(); + for (var index = 0; index < tasks; index++) { + await stem.enqueue('benchmark.sqlite.noop', args: {'index': index}); + } + enqueue.stop(); + + await completed.future.timeout(const Duration(minutes: 2)); + total.stop(); + final result = { + 'adapter': 'sqlite', + 'tasks': tasks, + 'concurrency': concurrency, + 'enqueue_ms': enqueue.elapsedMicroseconds / 1000, + 'end_to_end_ms': total.elapsedMicroseconds / 1000, + 'enqueue_tasks_per_second': _rate(tasks, enqueue.elapsed), + 'end_to_end_tasks_per_second': _rate(tasks, total.elapsed), + }; + stdout.writeln(const JsonEncoder.withIndent(' ').convert(result)); + if (checkBaseline) _checkBaseline(result); + await stdout.flush(); + } finally { + await worker.shutdown(); + await backend.close(); + await broker.close(); + await directory.delete(recursive: true); + } +} + +double _rate(int count, Duration duration) { + final seconds = duration.inMicroseconds / Duration.microsecondsPerSecond; + return seconds == 0 ? double.infinity : count / seconds; +} + +int? _intOption(List args, String name) { + final index = args.indexOf(name); + if (index == -1 || index + 1 >= args.length) return null; + return int.tryParse(args[index + 1]); +} + +void _checkBaseline(Map result) { + final baselineFile = File('benchmark/sqlite_throughput_baseline.json'); + if (!baselineFile.existsSync()) { + throw StateError('Missing benchmark baseline: ${baselineFile.path}'); + } + final baseline = jsonDecode(baselineFile.readAsStringSync()); + if (baseline is! Map) { + throw StateError('Benchmark baseline must be a JSON object.'); + } + final minimum = baseline['minimum_end_to_end_tasks_per_second']; + final measured = result['end_to_end_tasks_per_second']; + if (minimum is! num || measured is! num || measured < minimum) { + throw StateError( + 'SQLite throughput regressed: measured $measured tasks/s, ' + 'minimum $minimum tasks/s.', + ); + } +} + +final class _BenchmarkTask extends TaskHandler { + _BenchmarkTask({required this.onComplete}); + + final void Function() onComplete; + + @override + String get name => 'benchmark.sqlite.noop'; + + @override + TaskOptions get options => const TaskOptions(); + + @override + Future call(TaskContext context, Map args) async { + onComplete(); + } +} diff --git a/packages/stem_sqlite/benchmark/sqlite_throughput_baseline.json b/packages/stem_sqlite/benchmark/sqlite_throughput_baseline.json new file mode 100644 index 00000000..87179ed4 --- /dev/null +++ b/packages/stem_sqlite/benchmark/sqlite_throughput_baseline.json @@ -0,0 +1,3 @@ +{ + "minimum_end_to_end_tasks_per_second": 50 +} diff --git a/packages/stem_sqlite/lib/src/backend/sqlite_result_backend.dart b/packages/stem_sqlite/lib/src/backend/sqlite_result_backend.dart index 767a596a..e4f04452 100644 --- a/packages/stem_sqlite/lib/src/backend/sqlite_result_backend.dart +++ b/packages/stem_sqlite/lib/src/backend/sqlite_result_backend.dart @@ -9,7 +9,8 @@ import 'package:stem_sqlite/src/connection.dart'; import 'package:stem_sqlite/src/models/models.dart'; /// SQLite-backed implementation of [ResultBackend]. -class SqliteResultBackend implements ResultBackend { +class SqliteResultBackend + implements ResultBackend, AtomicTerminalResultBackend { SqliteResultBackend._( this._connections, { required this.namespace, @@ -144,6 +145,9 @@ class SqliteResultBackend implements ResultBackend { Timer? _cleanupTimer; bool _closed = false; + @override + bool get supportsAtomicTerminalWrites => true; + /// Closes the backend and releases any database resources. @override Future close() async { @@ -199,6 +203,41 @@ class SqliteResultBackend implements ResultBackend { _watchers[taskId]?.add(status); } + @override + Future setTerminalIfAbsent( + TaskStatus status, { + Duration? ttl, + }) async { + final now = stemNow(); + final expiresAt = now.add(ttl ?? defaultTtl); + final updated = await _connections.runInTransaction((txn) { + final query = txn + .query() + .whereEquals('id', status.id) + .whereEquals('namespace', namespace) + .where('expiresAt', now, PredicateOperator.greaterThan) + .where( + 'state', + const ['succeeded', 'failed', 'cancelled'], + PredicateOperator.notInValues, + ); + return query.update({ + 'state': status.state.name, + 'payload': _wrapScalarJson(status.payload), + 'error': status.error?.toJson(), + 'attempt': status.attempt, + 'meta': status.meta, + 'expiresAt': expiresAt, + 'updatedAt': now, + }); + }); + final applied = updated > 0; + if (applied) { + _watchers[status.id]?.add(status); + } + return applied; + } + @override Future get(String taskId) async { final now = stemNow(); @@ -434,9 +473,11 @@ class SqliteResultBackend implements ResultBackend { @override Future expire(String taskId, Duration ttl) async { final expiresAt = stemNow().add(ttl); - await _context.repository().update( - StemTaskResultUpdateDto(expiresAt: expiresAt), - where: StemTaskResultPartial(id: taskId, namespace: namespace), + await _connections.runInTransaction( + (txn) => txn.repository().update( + StemTaskResultUpdateDto(expiresAt: expiresAt), + where: StemTaskResultPartial(id: taskId, namespace: namespace), + ), ); } diff --git a/packages/stem_sqlite/lib/src/broker/sqlite_broker.dart b/packages/stem_sqlite/lib/src/broker/sqlite_broker.dart index 5319337d..fa408e11 100644 --- a/packages/stem_sqlite/lib/src/broker/sqlite_broker.dart +++ b/packages/stem_sqlite/lib/src/broker/sqlite_broker.dart @@ -5,13 +5,20 @@ import 'dart:io'; import 'package:meta/meta.dart'; import 'package:ormed/ormed.dart'; +import 'package:stem/observability.dart' show stemLogger; import 'package:stem/stem.dart'; import 'package:stem_sqlite/src/connection.dart'; import 'package:stem_sqlite/src/models/models.dart'; import 'package:uuid/uuid.dart'; /// SQLite-backed implementation of [Broker]. -class SqliteBroker implements Broker { +class SqliteBroker + implements + Broker, + LeaseBroker, + InspectableBroker, + DeadLetterBroker, + BrokerCapabilitiesProvider { SqliteBroker._( this._connections, { required this.namespace, @@ -102,6 +109,18 @@ class SqliteBroker implements Broker { @override bool get supportsPriority => true; + @override + BrokerCapabilities get capabilities => const BrokerCapabilities( + supportsDelayedDelivery: true, + supportsPriorityOrdering: true, + deliveryGuarantee: BrokerDeliveryGuarantee.atLeastOnce, + supportsBroadcastFanout: true, + supportsQueueInspection: true, + supportsLeaseExtension: true, + supportsDeadLettering: true, + supportsDeadLetterReplay: true, + ); + /// Closes the broker and releases any database resources. @override Future close() async { @@ -211,11 +230,13 @@ class SqliteBroker implements Broker { return; } final jobId = _parseReceipt(delivery.receipt); - await _context - .query() - .whereEquals('id', jobId) - .whereEquals('namespace', namespace) - .delete(); + await _connections.runInTransaction( + (txn) => txn + .query() + .whereEquals('id', jobId) + .whereEquals('namespace', namespace) + .delete(), + ); } @override @@ -229,18 +250,20 @@ class SqliteBroker implements Broker { } final jobId = _parseReceipt(delivery.receipt); final now = stemNow(); - await _context - .query() - .whereEquals('id', jobId) - .whereEquals('namespace', namespace) - .update({ - 'lockedAt': null, - 'lockedUntil': null, - 'lockedBy': null, - 'attempt': delivery.envelope.attempt + 1, - 'notBefore': null, - 'updatedAt': now, - }); + await _connections.runInTransaction( + (txn) => txn + .query() + .whereEquals('id', jobId) + .whereEquals('namespace', namespace) + .update({ + 'lockedAt': null, + 'lockedUntil': null, + 'lockedBy': null, + 'attempt': delivery.envelope.attempt + 1, + 'notBefore': null, + 'updatedAt': now, + }), + ); } @override @@ -255,29 +278,31 @@ class SqliteBroker implements Broker { final jobId = _parseReceipt(delivery.receipt); final now = stemNow(); - final row = await _context - .query() - .whereEquals('id', jobId) - .whereEquals('namespace', namespace) - .firstOrNull(); - await _context - .query() - .whereEquals('id', jobId) - .whereEquals('namespace', namespace) - .delete(); - if (row != null) { - await _context.repository().insert( - StemDeadLetterInsertDto( - id: row.id, - namespace: namespace, - queue: row.queue, - envelope: row.envelope, - reason: reason, - meta: meta, - deadAt: now, - ), - ); - } + await _connections.runInTransaction((txn) async { + final row = await txn + .query() + .whereEquals('id', jobId) + .whereEquals('namespace', namespace) + .firstOrNull(); + await txn + .query() + .whereEquals('id', jobId) + .whereEquals('namespace', namespace) + .delete(); + if (row != null) { + await txn.repository().insert( + StemDeadLetterInsertDto( + id: row.id, + namespace: namespace, + queue: row.queue, + envelope: row.envelope, + reason: reason, + meta: meta, + deadAt: now, + ), + ); + } + }); } @override @@ -287,19 +312,23 @@ class SqliteBroker implements Broker { } final jobId = _parseReceipt(delivery.receipt); final now = stemNow(); - await _context.repository().update( - StemQueueJobUpdateDto(lockedUntil: now.add(by)), - where: StemQueueJobPartial(id: jobId, namespace: namespace), + await _connections.runInTransaction( + (txn) => txn.repository().update( + StemQueueJobUpdateDto(lockedUntil: now.add(by)), + where: StemQueueJobPartial(id: jobId, namespace: namespace), + ), ); } @override Future purge(String queue) async { - await _context - .query() - .whereEquals('queue', queue) - .whereEquals('namespace', namespace) - .delete(); + await _connections.runInTransaction( + (txn) => txn + .query() + .whereEquals('queue', queue) + .whereEquals('namespace', namespace) + .delete(), + ); } @override @@ -434,26 +463,30 @@ class SqliteBroker implements Broker { .limit(limit) .pluck('id'); if (ids.isEmpty) return 0; - await _context - .query() - .whereIn('id', ids) - .whereEquals('namespace', namespace) - .delete(); + await _connections.runInTransaction( + (txn) => txn + .query() + .whereIn('id', ids) + .whereEquals('namespace', namespace) + .delete(), + ); return ids.length; } - var query = _context - .query() - .whereEquals('queue', queue) - .whereEquals('namespace', namespace); - if (since != null) { - query = query.where( - 'deadAt', - since, - PredicateOperator.greaterThanOrEqual, - ); - } - return query.delete(); + return _connections.runInTransaction((txn) { + var transactionalQuery = txn + .query() + .whereEquals('queue', queue) + .whereEquals('namespace', namespace); + if (since != null) { + transactionalQuery = transactionalQuery.where( + 'deadAt', + since, + PredicateOperator.greaterThanOrEqual, + ); + } + return transactionalQuery.delete(); + }); } Future<_QueuedJob?> _claimNextJob(String queue, String consumerId) async { @@ -518,14 +551,12 @@ class SqliteBroker implements Broker { _runSweeperCycle().catchError((Object error, StackTrace stackTrace) { stemLogger.warning( 'SQLite broker sweeper cycle failed: $error', - stemLogContext( - component: 'stem_sqlite', - subsystem: 'broker_sweeper', - fields: { - 'namespace': namespace, - 'stack': stackTrace.toString(), - }, - ), + fields: { + 'component': 'stem_sqlite', + 'subsystem': 'broker_sweeper', + 'namespace': namespace, + 'stack': stackTrace.toString(), + }, ); }), ); diff --git a/packages/stem_sqlite/lib/src/connection.dart b/packages/stem_sqlite/lib/src/connection.dart index 82e66b42..7aaf21c1 100644 --- a/packages/stem_sqlite/lib/src/connection.dart +++ b/packages/stem_sqlite/lib/src/connection.dart @@ -1,3 +1,4 @@ +import 'dart:async'; import 'dart:io'; import 'package:ormed/ormed.dart'; @@ -14,15 +15,32 @@ class SqliteConnections { /// /// The caller remains responsible for disposing [dataSource]. factory SqliteConnections.fromDataSource(DataSource dataSource) => - SqliteConnections._(dataSource, ownsDataSource: false); + SqliteConnections._( + dataSource, + ownsDataSource: false, + coordinationKey: _databaseCoordinationKey(dataSource), + ); /// Creates a connection wrapper for an initialized data source. - SqliteConnections._(this.dataSource, {required bool ownsDataSource}) - : _ownsDataSource = ownsDataSource; + SqliteConnections._( + this.dataSource, { + required bool ownsDataSource, + String? coordinationKey, + }) : _ownsDataSource = ownsDataSource, + _coordinationKey = coordinationKey; /// Underlying data source instance. final DataSource dataSource; final bool _ownsDataSource; + final String? _coordinationKey; + + // The native SQLite driver keeps transaction depth on the connection and + // uses savepoints for nested transactions. Since transaction callbacks are + // asynchronous, two callers can otherwise interleave on the same ORM + // connection and make one caller release or roll back the other's + // savepoint. Serialize the transaction boundary at the wrapper, where all + // Stem SQLite stores already converge. + Future _transactionTail = Future.value(); /// Convenience accessor for the raw ORM connection. OrmConnection get connection => dataSource.connection; @@ -40,7 +58,11 @@ class SqliteConnections { await _runMigrations(file); } final dataSource = await _openDataSource(file, readOnly: readOnly); - return SqliteConnections._(dataSource, ownsDataSource: true); + return SqliteConnections._( + dataSource, + ownsDataSource: true, + coordinationKey: file.absolute.path, + ); }); } @@ -51,27 +73,79 @@ class SqliteConnections { DataSource dataSource, ) async { await _runMigrationsForDataSource(dataSource); - return SqliteConnections._(dataSource, ownsDataSource: false); + return SqliteConnections._( + dataSource, + ownsDataSource: false, + coordinationKey: _databaseCoordinationKey(dataSource), + ); } /// Runs [action] inside a database transaction. Future runInTransaction( Future Function(QueryContext context) action, ) async { - if (!dataSource.isInitialized) { - await dataSource.init(); + final previous = _transactionTail; + final release = Completer(); + _transactionTail = release.future; + + await previous; + try { + Future transaction() async { + if (!dataSource.isInitialized) { + await dataSource.init(); + } + return dataSource.connection.transaction( + () => action(dataSource.context), + ); + } + + final key = _coordinationKey; + return await (key == null + ? transaction() + : _serializeFileTransactionForKey(key, transaction)); + } finally { + release.complete(); } - return dataSource.connection.transaction(() => action(dataSource.context)); } /// Closes the data source. Future close() async { + await _transactionTail; if (_ownsDataSource) { await dataSource.dispose(); } } } +final Map> _fileTransactionTails = {}; + +Future _serializeFileTransactionForKey( + String key, + Future Function() action, +) async { + final previous = _fileTransactionTails[key] ?? Future.value(); + final release = Completer(); + _fileTransactionTails[key] = release.future; + await previous; + try { + return await action(); + } finally { + release.complete(); + if (identical(_fileTransactionTails[key], release.future)) { + unawaited(_fileTransactionTails.remove(key)); + } + } +} + +String? _databaseCoordinationKey(DataSource dataSource) { + final path = dataSource.isInitialized + ? (dataSource.connection.options['path'] ?? + dataSource.connection.options['database']) + : dataSource.options.database; + if (path == null || path == ':memory:') return null; + return File(path.toString()).absolute.path; +} + Future _openDataSource(File file, {required bool readOnly}) async { if (!file.parent.existsSync()) { file.parent.createSync(recursive: true); diff --git a/packages/stem_sqlite/lib/src/database.dart b/packages/stem_sqlite/lib/src/database.dart deleted file mode 100644 index 91edd38c..00000000 --- a/packages/stem_sqlite/lib/src/database.dart +++ /dev/null @@ -1,20 +0,0 @@ -import 'dart:io'; - -/// Legacy database wrapper kept for backward compatibility. -@Deprecated('Use SqliteConnections/QueryContext from stem_sqlite instead.') -class StemSqliteDatabase { - /// Creates the legacy wrapper. - @Deprecated('Use SqliteConnections/QueryContext from stem_sqlite instead.') - StemSqliteDatabase._(); - - /// Throws an [UnsupportedError] in favor of `SqliteConnections`. - static Future openFile( - File file, { - bool readOnly = false, - }) async { - throw UnsupportedError( - 'StemSqliteDatabase is deprecated. ' - 'Use SqliteConnections.open(file, readOnly: $readOnly) instead.', - ); - } -} diff --git a/packages/stem_sqlite/lib/src/database/datasource.dart b/packages/stem_sqlite/lib/src/database/datasource.dart index 30a70ff7..5272f71b 100644 --- a/packages/stem_sqlite/lib/src/database/datasource.dart +++ b/packages/stem_sqlite/lib/src/database/datasource.dart @@ -1,13 +1,13 @@ -import 'package:contextual/contextual.dart' as contextual; import 'package:ormed/ormed.dart'; import 'package:ormed_sqlite/ormed_sqlite.dart'; -import 'package:stem/stem.dart' show stemLogger; +import 'package:stem/observability.dart' show StemLogger, stemLogger; import 'package:stem_sqlite/orm_registry.g.dart'; +import 'package:stem_sqlite/src/database/stem_orm_logger.dart'; /// Creates a new DataSource instance using the project configuration. DataSource createDataSource({ bool logging = false, - contextual.Logger? logger, + StemLogger? logger, }) { var config = loadOrmConfig(); if (logging) { @@ -23,7 +23,7 @@ DataSource createDataSource({ /// Creates a new DataSource instance using a resolved ORM project config. DataSource createDataSourceFromConfig( OrmProjectConfig config, { - contextual.Logger? logger, + StemLogger? logger, }) { final registry = bootstrapOrm(); final options = Map.from(config.driver.options); @@ -45,5 +45,9 @@ DataSource createDataSourceFromConfig( tablePrefix: options['table_prefix']?.toString() ?? '', defaultSchema: options['default_schema']?.toString(), ); - return DataSource(dataSourceOptions.copyWith(logger: logger)); + return DataSource( + dataSourceOptions.copyWith( + logger: createOrmLogger(logger ?? stemLogger), + ), + ); } diff --git a/packages/stem_sqlite/lib/src/database/seed_runtime.dart b/packages/stem_sqlite/lib/src/database/seed_runtime.dart index 90abb196..708f2d0d 100644 --- a/packages/stem_sqlite/lib/src/database/seed_runtime.dart +++ b/packages/stem_sqlite/lib/src/database/seed_runtime.dart @@ -4,7 +4,7 @@ import 'dart:io'; import 'package:artisanal/args.dart'; import 'package:ormed/ormed.dart'; -import 'package:stem/stem.dart' show stemLogger; +import 'package:stem/observability.dart' show stemLogger; import 'package:stem_sqlite/src/database/datasource.dart'; diff --git a/packages/stem_sqlite/lib/src/database/seeders.dart b/packages/stem_sqlite/lib/src/database/seeders.dart index ced331b3..4493b2d0 100644 --- a/packages/stem_sqlite/lib/src/database/seeders.dart +++ b/packages/stem_sqlite/lib/src/database/seeders.dart @@ -1,10 +1,8 @@ -// Ignoring unreachable_from_main because tooling imports use runProjectSeeds. -// ignore_for_file: unreachable_from_main - import 'package:ormed/ormed.dart'; import 'package:stem_sqlite/orm_registry.g.dart'; import 'package:stem_sqlite/src/database/seed_runtime.dart'; import 'package:stem_sqlite/src/database/seeders/database_seeder.dart'; + // // diff --git a/packages/stem_sqlite/lib/src/database/seeders/database_seeder.dart b/packages/stem_sqlite/lib/src/database/seeders/database_seeder.dart index e4ee0d0b..aee14e92 100644 --- a/packages/stem_sqlite/lib/src/database/seeders/database_seeder.dart +++ b/packages/stem_sqlite/lib/src/database/seeders/database_seeder.dart @@ -7,7 +7,8 @@ class AppDatabaseSeeder extends DatabaseSeeder { @override Future run() async { - // TODO(stem): add seed logic here + // No default seeds are installed by stem; add application-specific data + // from the consuming service instead. // Examples: // await seed([ // {'name': 'Admin User', 'email': 'admin@example.com'}, diff --git a/packages/stem_sqlite/lib/src/database/stem_orm_logger.dart b/packages/stem_sqlite/lib/src/database/stem_orm_logger.dart new file mode 100644 index 00000000..7128cc5b --- /dev/null +++ b/packages/stem_sqlite/lib/src/database/stem_orm_logger.dart @@ -0,0 +1,65 @@ +import 'package:contextual/contextual.dart' as contextual; +import 'package:stem/observability.dart' show StemLogger; + +/// Adapts Stem's dependency-neutral logger to the ORM's logger contract. +contextual.Logger createOrmLogger(StemLogger logger) { + final ormLogger = + contextual.Logger( + level: contextual.Level.debug, + defaultChannelEnabled: false, + )..setListener((entry) { + final record = entry.record; + final fields = record.context.all().cast(); + switch (record.level) { + case contextual.Level.debug: + logger.debug( + record.message, + fields: fields, + stackTrace: record.stackTrace, + ); + case contextual.Level.info: + logger.info( + record.message, + fields: fields, + stackTrace: record.stackTrace, + ); + case contextual.Level.notice: + logger.notice( + record.message, + fields: fields, + stackTrace: record.stackTrace, + ); + case contextual.Level.warning: + logger.warning( + record.message, + fields: fields, + stackTrace: record.stackTrace, + ); + case contextual.Level.error: + logger.error( + record.message, + fields: fields, + stackTrace: record.stackTrace, + ); + case contextual.Level.critical: + logger.critical( + record.message, + fields: fields, + stackTrace: record.stackTrace, + ); + case contextual.Level.alert: + logger.alert( + record.message, + fields: fields, + stackTrace: record.stackTrace, + ); + case contextual.Level.emergency: + logger.emergency( + record.message, + fields: fields, + stackTrace: record.stackTrace, + ); + } + }); + return ormLogger; +} diff --git a/packages/stem_sqlite/lib/src/workflow/sqlite_workflow_store.dart b/packages/stem_sqlite/lib/src/workflow/sqlite_workflow_store.dart index 6449b91d..686074f6 100644 --- a/packages/stem_sqlite/lib/src/workflow/sqlite_workflow_store.dart +++ b/packages/stem_sqlite/lib/src/workflow/sqlite_workflow_store.dart @@ -82,9 +82,9 @@ class SqliteWorkflowStore implements WorkflowStore { @override Future createRun({ - String? runId, required String workflow, required Map params, + String? runId, String? parentRunId, Duration? ttl, WorkflowCancellationPolicy? cancellationPolicy, diff --git a/packages/stem_sqlite/lib/stem_sqlite.dart b/packages/stem_sqlite/lib/stem_sqlite.dart index fe60dd33..43f71af8 100644 --- a/packages/stem_sqlite/lib/stem_sqlite.dart +++ b/packages/stem_sqlite/lib/stem_sqlite.dart @@ -3,9 +3,6 @@ export 'src/backend/sqlite_result_backend.dart' show SqliteResultBackend; export 'src/broker/sqlite_broker.dart' show SqliteBroker; export 'src/connection.dart' show SqliteConnections; export 'src/control/sqlite_revoke_store.dart' show SqliteRevokeStore; -// Exported for compatibility with the deprecated StemSqliteDatabase API. -// ignore: deprecated_member_use_from_same_package -export 'src/database.dart' show StemSqliteDatabase; export 'src/database/datasource.dart' show createDataSource; export 'src/models/models.dart'; export 'src/stack/sqlite_adapter.dart' show StemSqliteAdapter; diff --git a/packages/stem_sqlite/pubspec.yaml b/packages/stem_sqlite/pubspec.yaml index ea448109..57c6389f 100644 --- a/packages/stem_sqlite/pubspec.yaml +++ b/packages/stem_sqlite/pubspec.yaml @@ -1,6 +1,6 @@ name: stem_sqlite description: SQLite broker and result backend for Stem. -version: 0.1.3 +version: 0.2.0 repository: https://github.com/kingwill101/stem resolution: workspace environment: @@ -14,13 +14,13 @@ dependencies: ormed: ^0.2.0 ormed_sqlite: ^0.3.0 path: ^1.9.1 - stem: ">=0.2.1 <0.3.0" + stem: ">=0.3.0 <0.4.0" uuid: ^4.5.2 dev_dependencies: build_runner: ^2.10.5 coverage: ^1.15.0 lints: ^6.0.0 - stem_adapter_tests: ">=0.1.2 <0.2.0" + stem_adapter_tests: ">=0.2.0 <0.3.0" test: ^1.29.0 very_good_analysis: ^10.0.0 diff --git a/packages/stem_sqlite/test/backend/sqlite_result_backend_test.dart b/packages/stem_sqlite/test/backend/sqlite_result_backend_test.dart index c384643c..6ab70eea 100644 --- a/packages/stem_sqlite/test/backend/sqlite_result_backend_test.dart +++ b/packages/stem_sqlite/test/backend/sqlite_result_backend_test.dart @@ -113,4 +113,41 @@ void main() { await backendB.close(); } }); + + test('terminal writes arbitrate concurrent completion attempts', () async { + final backend = await SqliteResultBackend.open( + dbFile, + namespace: 'terminal-race', + defaultTtl: const Duration(seconds: 2), + groupDefaultTtl: const Duration(seconds: 2), + heartbeatTtl: const Duration(seconds: 2), + cleanupInterval: const Duration(milliseconds: 200), + ); + try { + await backend.set('terminal-task', TaskState.running); + final applied = await backend.setTerminalIfAbsent( + TaskStatus( + id: 'terminal-task', + state: TaskState.succeeded, + payload: 'first', + attempt: 1, + ), + ); + final rejected = await backend.setTerminalIfAbsent( + TaskStatus( + id: 'terminal-task', + state: TaskState.failed, + error: const TaskError(type: 'late', message: 'late failure'), + attempt: 1, + ), + ); + + expect(applied, isTrue); + expect(rejected, isFalse); + expect((await backend.get('terminal-task'))?.payload, 'first'); + expect((await backend.get('terminal-task'))?.state, TaskState.succeeded); + } finally { + await backend.close(); + } + }); } diff --git a/packages/stem_sqlite/test/connection_test.dart b/packages/stem_sqlite/test/connection_test.dart new file mode 100644 index 00000000..290038c0 --- /dev/null +++ b/packages/stem_sqlite/test/connection_test.dart @@ -0,0 +1,33 @@ +import 'dart:io'; + +import 'package:stem_sqlite/src/connection.dart'; +import 'package:test/test.dart'; + +void main() { + test( + 'serializes concurrent transactions across SQLite connections', + () async { + final directory = await Directory.systemTemp.createTemp( + 'stem-sqlite-connection-test-', + ); + final file = File('${directory.path}/stem.db'); + final connections = await SqliteConnections.open(file); + final secondConnections = await SqliteConnections.open(file); + try { + await Future.wait( + List.generate( + 32, + (index) => (index.isEven ? connections : secondConnections) + .runInTransaction((_) async { + await Future.delayed(const Duration(milliseconds: 1)); + }), + ), + ); + } finally { + await secondConnections.close(); + await connections.close(); + await directory.delete(recursive: true); + } + }, + ); +} diff --git a/packages/stem_sqlite/test/datasource_logger_test.dart b/packages/stem_sqlite/test/datasource_logger_test.dart index f1d7cb06..1ebdbd64 100644 --- a/packages/stem_sqlite/test/datasource_logger_test.dart +++ b/packages/stem_sqlite/test/datasource_logger_test.dart @@ -1,20 +1,19 @@ import 'package:contextual/contextual.dart' as contextual; -import 'package:stem/stem.dart'; +import 'package:stem/observability.dart' show stemLogger; import 'package:stem_sqlite/src/database/datasource.dart'; import 'package:test/test.dart'; void main() { - test('createDataSource forwards logger and logging flags', () { - final logger = contextual.Logger(); - final dataSource = createDataSource(logging: true, logger: logger); + test('createDataSource accepts a Stem logger and enables ORM logging', () { + final dataSource = createDataSource(logging: true, logger: stemLogger); expect(dataSource.options.logging, isTrue); - expect(dataSource.options.logger, same(logger)); + expect(dataSource.options.logger, isA()); }); test('createDataSource defaults to stemLogger', () { final dataSource = createDataSource(); - expect(dataSource.options.logger, same(stemLogger)); + expect(dataSource.options.logger, isA()); }); } diff --git a/packages/stem_sqlite/test/migration_upgrade_test.dart b/packages/stem_sqlite/test/migration_upgrade_test.dart new file mode 100644 index 00000000..111592cc --- /dev/null +++ b/packages/stem_sqlite/test/migration_upgrade_test.dart @@ -0,0 +1,187 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:ormed/migrations.dart'; +import 'package:ormed_sqlite/ormed_sqlite.dart'; +import 'package:stem/stem.dart'; +import 'package:stem_sqlite/src/database/migrations.dart'; +import 'package:stem_sqlite/stem_sqlite.dart'; +import 'package:test/test.dart'; + +void main() { + test( + 'upgrades every historical schema prefix to the current registry', + () async { + final migrations = buildMigrations(); + + for ( + var prefixLength = 1; + prefixLength < migrations.length; + prefixLength += 1 + ) { + final directory = await Directory.systemTemp.createTemp( + 'stem-sqlite-migration-upgrade-', + ); + final file = File('${directory.path}/stem.db'); + try { + final oldAdapter = SqliteDriverAdapter.file(file.path); + final oldLedger = SqlMigrationLedger( + oldAdapter, + tableName: 'orm_migrations', + ); + final oldRunner = MigrationRunner( + schemaDriver: oldAdapter, + ledger: oldLedger, + migrations: migrations.take(prefixLength).toList(), + emitEvents: false, + ); + final oldReport = await oldRunner.applyAll(); + expect(oldReport.actions, hasLength(prefixLength)); + await oldAdapter.close(); + + final currentAdapter = SqliteDriverAdapter.file(file.path); + final currentLedger = SqlMigrationLedger( + currentAdapter, + tableName: 'orm_migrations', + ); + final currentRunner = MigrationRunner( + schemaDriver: currentAdapter, + ledger: currentLedger, + migrations: migrations, + emitEvents: false, + ); + final upgrade = await currentRunner.applyAll(); + expect( + upgrade.actions, + hasLength(migrations.length - prefixLength), + reason: 'prefix length $prefixLength', + ); + + final statuses = await currentRunner.status(); + expect( + statuses.every((status) => status.applied), + isTrue, + reason: 'prefix length $prefixLength', + ); + await currentAdapter.close(); + } finally { + await directory.delete(recursive: true); + } + } + }, + ); + + test('current additive schema accepts legacy-shaped queue writes', () async { + final directory = await Directory.systemTemp.createTemp( + 'stem-sqlite-mixed-version-', + ); + final file = File('${directory.path}/stem.db'); + try { + final adapter = SqliteDriverAdapter.file(file.path); + final runner = MigrationRunner( + schemaDriver: adapter, + ledger: SqlMigrationLedger(adapter, tableName: 'orm_migrations'), + migrations: buildMigrations(), + emitEvents: false, + ); + await runner.applyAll(); + + final now = DateTime.now().toUtc(); + await adapter.executeRaw( + ''' +INSERT INTO stem_queue_jobs + (id, queue, envelope, attempt, max_retries, priority, not_before, + locked_at, locked_until, locked_by, created_at, updated_at) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +''', + [ + 'legacy-queue-job', + 'default', + jsonEncode({'name': 'legacy.task', 'args': {}}), + 0, + 0, + 0, + null, + null, + null, + null, + now, + now, + ], + ); + final rows = await adapter.queryRaw( + 'SELECT namespace FROM stem_queue_jobs WHERE id = ?', + ['legacy-queue-job'], + ); + expect(rows.single['namespace'], equals('stem')); + await adapter.close(); + } finally { + await directory.delete(recursive: true); + } + }); + + test( + 'current broker consumes a queue row written before namespace migration', + () async { + final directory = await Directory.systemTemp.createTemp( + 'stem-sqlite-mixed-worker-', + ); + final file = File('${directory.path}/stem.db'); + try { + final migrations = buildMigrations(); + final oldAdapter = SqliteDriverAdapter.file(file.path); + final oldRunner = MigrationRunner( + schemaDriver: oldAdapter, + ledger: SqlMigrationLedger(oldAdapter, tableName: 'orm_migrations'), + migrations: migrations.take(1).toList(), + emitEvents: false, + ); + await oldRunner.applyAll(); + final now = DateTime.now().toUtc(); + await oldAdapter.executeRaw( + ''' +INSERT INTO stem_queue_jobs + (id, queue, envelope, attempt, max_retries, priority, not_before, + locked_at, locked_until, locked_by, created_at, updated_at) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +''', + [ + 'legacy-before-namespace', + 'default', + jsonEncode({'name': 'legacy.task', 'args': {}}), + 0, + 0, + 0, + null, + null, + null, + null, + now, + now, + ], + ); + await oldAdapter.close(); + + final broker = await SqliteBroker.open( + file, + pollInterval: const Duration(milliseconds: 5), + ); + try { + final delivery = await broker + .consume( + RoutingSubscription.singleQueue('default'), + consumerName: 'current-worker', + ) + .first + .timeout(const Duration(seconds: 2)); + expect(delivery.envelope.name, equals('legacy.task')); + await broker.ack(delivery); + } finally { + await broker.close(); + } + } finally { + await directory.delete(recursive: true); + } + }, + ); +} diff --git a/packages/stem_sqlite/test/migrations_registry_test.dart b/packages/stem_sqlite/test/migrations_registry_test.dart new file mode 100644 index 00000000..454f1a33 --- /dev/null +++ b/packages/stem_sqlite/test/migrations_registry_test.dart @@ -0,0 +1,20 @@ +import 'package:stem_sqlite/src/database/migrations.dart'; +import 'package:test/test.dart'; + +void main() { + test( + 'migration registry is ordered, unique, and retains release history', + () { + final migrations = buildMigrations(); + final ids = migrations + .map((migration) => migration.id.toString()) + .toList(); + + expect(ids, isNotEmpty); + expect(ids.toSet(), hasLength(ids.length)); + expect(ids, orderedEquals([...ids]..sort())); + expect(ids.first, contains('m_20251222070816_create_stem_tables')); + expect(ids, contains(contains('m_20260224103000_add_revoke_store'))); + }, + ); +} diff --git a/packages/stem_sqlite/test/workflow/workflow_restart_recovery_test.dart b/packages/stem_sqlite/test/workflow/workflow_restart_recovery_test.dart new file mode 100644 index 00000000..164eacf0 --- /dev/null +++ b/packages/stem_sqlite/test/workflow/workflow_restart_recovery_test.dart @@ -0,0 +1,215 @@ +import 'dart:async'; +import 'dart:io'; + +import 'package:stem/memory.dart'; +import 'package:stem/stem.dart'; +import 'package:stem_sqlite/stem_sqlite.dart'; +import 'package:test/test.dart'; + +void main() { + late Directory tempDir; + late File dbFile; + + setUp(() { + tempDir = Directory.systemTemp.createTempSync( + 'stem_sqlite_workflow_restart_test', + ); + dbFile = File('${tempDir.path}/workflow.db'); + }); + + tearDown(() async { + if (dbFile.existsSync()) { + await dbFile.delete(); + } + await tempDir.delete(recursive: true); + }); + + test( + 'resumes a persisted checkpoint after broker and runtime restart', + () async { + final clock = FakeWorkflowClock(DateTime.utc(2026)); + final brokerBeforeRestart = InMemoryBroker( + delayedInterval: const Duration(milliseconds: 5), + claimInterval: const Duration(milliseconds: 5), + ); + final backendBeforeRestart = InMemoryResultBackend(); + final registryBeforeRestart = InMemoryTaskRegistry(); + final storeBeforeRestart = await SqliteWorkflowStore.open( + dbFile, + clock: clock, + ); + final runtimeBeforeRestart = WorkflowRuntime( + stem: Stem( + broker: brokerBeforeRestart, + registry: registryBeforeRestart, + backend: backendBeforeRestart, + ), + store: storeBeforeRestart, + eventBus: InMemoryEventBus(storeBeforeRestart), + clock: clock, + pollInterval: const Duration(milliseconds: 5), + leaseExtension: const Duration(seconds: 5), + runLeaseDuration: const Duration(seconds: 5), + runtimeId: 'workflow-runtime-before-restart', + ); + registryBeforeRestart.register( + runtimeBeforeRestart.workflowRunnerHandler(), + ); + var checkpointExecutions = 0; + final workflow = Flow( + name: 'restart.recovery.workflow', + build: (flow) { + flow + ..step('checkpoint', (context) async { + checkpointExecutions += 1; + return 'persisted'; + }) + ..step('wait', (context) async { + if (context.takeResumeData() == true) { + return 'resumed'; + } + context.sleep(const Duration(milliseconds: 20)); + return null; + }) + ..step( + 'finish', + (context) async => '${context.previousResult}-done', + ); + }, + ).definition; + runtimeBeforeRestart.registerWorkflow(workflow); + final workerBeforeRestart = Worker( + broker: brokerBeforeRestart, + backend: backendBeforeRestart, + tasks: [runtimeBeforeRestart.workflowRunnerHandler()], + queue: 'workflow', + subscription: RoutingSubscription.singleQueue('workflow'), + consumerName: 'workflow-worker-before-restart', + concurrency: 1, + prefetchMultiplier: 1, + retryStrategy: ExponentialJitterRetryStrategy( + base: const Duration(milliseconds: 5), + max: const Duration(milliseconds: 20), + seed: 1, + ), + lifecycle: const WorkerLifecycleConfig( + installSignalHandlers: false, + ), + ); + + final runId = await _runBeforeRestart( + clock: clock, + runtime: runtimeBeforeRestart, + store: storeBeforeRestart, + worker: workerBeforeRestart, + workflowName: workflow.name, + ); + expect(checkpointExecutions, equals(1)); + + await workerBeforeRestart.shutdown(); + await runtimeBeforeRestart.dispose(); + await storeBeforeRestart.close(); + brokerBeforeRestart.dispose(); + + final brokerAfterRestart = InMemoryBroker( + delayedInterval: const Duration(milliseconds: 5), + claimInterval: const Duration(milliseconds: 5), + ); + final backendAfterRestart = InMemoryResultBackend(); + final registryAfterRestart = InMemoryTaskRegistry(); + final storeAfterRestart = await SqliteWorkflowStore.open( + dbFile, + clock: clock, + ); + final runtimeAfterRestart = WorkflowRuntime( + stem: Stem( + broker: brokerAfterRestart, + registry: registryAfterRestart, + backend: backendAfterRestart, + ), + store: storeAfterRestart, + eventBus: InMemoryEventBus(storeAfterRestart), + clock: clock, + pollInterval: const Duration(milliseconds: 5), + leaseExtension: const Duration(seconds: 5), + runLeaseDuration: const Duration(seconds: 5), + runtimeId: 'workflow-runtime-after-restart', + )..registerWorkflow(workflow); + registryAfterRestart.register( + runtimeAfterRestart.workflowRunnerHandler(), + ); + final workerAfterRestart = Worker( + broker: brokerAfterRestart, + backend: backendAfterRestart, + tasks: [runtimeAfterRestart.workflowRunnerHandler()], + queue: 'workflow', + subscription: RoutingSubscription.singleQueue('workflow'), + consumerName: 'workflow-worker-after-restart', + concurrency: 1, + prefetchMultiplier: 1, + lifecycle: const WorkerLifecycleConfig( + installSignalHandlers: false, + ), + ); + + try { + await runtimeAfterRestart.start(); + await workerAfterRestart.start(); + clock.advance(const Duration(milliseconds: 30)); + + await _waitForRun( + storeAfterRestart, + runId, + (state) => state?.status == WorkflowStatus.completed, + ); + + final completed = await storeAfterRestart.get(runId); + expect(completed?.result, equals('resumed-done')); + expect(checkpointExecutions, equals(1)); + } finally { + await workerAfterRestart.shutdown(); + await runtimeAfterRestart.dispose(); + await storeAfterRestart.close(); + brokerAfterRestart.dispose(); + } + }, + ); +} + +Future _runBeforeRestart({ + required FakeWorkflowClock clock, + required WorkflowRuntime runtime, + required WorkflowStore store, + required Worker worker, + required String workflowName, +}) async { + await worker.start(); + final runId = await runtime.startWorkflow(workflowName); + await _waitForRun( + store, + runId, + (state) => state?.status == WorkflowStatus.suspended, + ); + expect(clock.now(), equals(DateTime.utc(2026))); + return runId; +} + +Future _waitForRun( + WorkflowStore store, + String runId, + bool Function(RunState? state) predicate, { + Duration timeout = const Duration(seconds: 3), +}) async { + final deadline = DateTime.now().add(timeout); + while (true) { + if (predicate(await store.get(runId))) { + return; + } + if (DateTime.now().isAfter(deadline)) { + throw TimeoutException( + 'Workflow $runId did not reach the expected state', + ); + } + await Future.delayed(const Duration(milliseconds: 10)); + } +} diff --git a/pubspec.yaml b/pubspec.yaml index a0c26bfb..4ef156a6 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -4,7 +4,6 @@ environment: sdk: ">=3.10.0 <4.0.0" workspace: - packages/stem - - packages/dashboard - packages/stem_builder - packages/stem_sqlite - packages/stem_flutter @@ -16,9 +15,5 @@ workspace: - packages/stem_cli dev_dependencies: coverage: ^1.15.0 + yaml: ^3.1.3 very_good_analysis: ^10.0.0 -dependency_overrides: - meta: ^1.18.0 - liquify: ^1.5.0 - timezone: 0.11.0 - pointycastle: ^4.0.0 diff --git a/tool/check_examples.dart b/tool/check_examples.dart new file mode 100644 index 00000000..6dc79e97 --- /dev/null +++ b/tool/check_examples.dart @@ -0,0 +1,163 @@ +import 'dart:io'; + +import 'package:path/path.dart' as p; +import 'package:yaml/yaml.dart'; + +/// Resolves, generates, analyzes, and verifies every example in the root +/// workspace. Examples are executable documentation and must stay in sync +/// with the package APIs they demonstrate. +Future main(List args) async { + final skipDiff = args.contains('--skip-diff'); + final unknown = args.where((arg) => arg != '--skip-diff').toList(); + if (unknown.isNotEmpty) { + stderr.writeln('Unknown option(s): ${unknown.join(', ')}'); + exitCode = 64; + return; + } + + final workspace = _workspacePackages(); + final examples = <_ExampleProject>[]; + for (final packagePath in workspace) { + final exampleRoot = Directory(p.join(packagePath, 'example')); + if (!exampleRoot.existsSync()) continue; + + await for (final entity in exampleRoot.list(recursive: true)) { + if (entity is! File || p.basename(entity.path) != 'pubspec.yaml') { + continue; + } + final relativeParts = p.split( + p.relative(entity.path, from: exampleRoot.path), + ); + if (relativeParts.any(_isGeneratedExampleDirectory)) { + continue; + } + final yaml = loadYaml(entity.readAsStringSync()); + if (yaml is! YamlMap) { + throw StateError('${entity.path} is not a YAML map.'); + } + final dependencies = { + ..._dependencyNames(yaml['dependencies']), + ..._dependencyNames(yaml['dev_dependencies']), + }; + examples.add( + _ExampleProject( + path: p.dirname(entity.path), + tool: dependencies.contains('flutter') ? 'flutter' : 'dart', + hasBuildRunner: _dependencyNames( + yaml['dev_dependencies'], + ).contains('build_runner'), + ), + ); + } + } + + examples.sort((a, b) => a.path.compareTo(b.path)); + if (examples.isEmpty) { + throw StateError('No workspace examples were discovered.'); + } + + print('[INFO] Checking ${examples.length} workspace examples.'); + for (final example in examples) { + final executable = example.tool; + await _run(executable, const ['pub', 'get'], example.path); + if (example.hasBuildRunner) { + await _run('dart', const [ + 'run', + 'build_runner', + 'build', + '--delete-conflicting-outputs', + ], example.path); + } + await _run(executable, const ['analyze', '--fatal-infos'], example.path); + + if (!skipDiff) { + final relativePath = p.relative(example.path); + final diff = await Process.run('git', [ + 'diff', + '--exit-code', + '--', + relativePath, + ]); + final status = await Process.run('git', [ + 'status', + '--porcelain', + '--', + relativePath, + ]); + if (diff.exitCode != 0 || (status.stdout as String).trim().isNotEmpty) { + throw StateError( + 'Generated or analyzed files changed in $relativePath. ' + 'Commit the example output before merging.', + ); + } + } + } + print('[SUCCESS] Workspace examples are valid.'); +} + +List _workspacePackages() { + final file = File('pubspec.yaml'); + if (!file.existsSync()) { + throw StateError('Run this tool from the repository root.'); + } + final yaml = loadYaml(file.readAsStringSync()); + if (yaml is! YamlMap || yaml['workspace'] is! YamlList) { + throw StateError('Root pubspec.yaml has no workspace package list.'); + } + return (yaml['workspace'] as YamlList) + .map((value) => value.toString()) + .toList(); +} + +Set _dependencyNames(Object? value) { + if (value is! YamlMap) return {}; + return value.keys.map((key) => key.toString()).toSet(); +} + +bool _isGeneratedExampleDirectory(String part) => + part == '.dart_tool' || + part == 'build' || + part == 'ephemeral' || + part == '.plugin_symlinks' || + part == 'android' || + part == 'ios' || + part == 'linux' || + part == 'macos' || + part == 'web' || + part == 'windows'; + +Future _run( + String executable, + List arguments, + String directory, +) async { + print('[RUN] $directory: $executable ${arguments.join(' ')}'); + final result = await Process.run( + executable, + arguments, + workingDirectory: directory, + runInShell: true, + ); + stdout.write(result.stdout); + stderr.write(result.stderr); + if (result.exitCode != 0) { + throw ProcessException( + executable, + arguments, + 'Command failed in $directory', + result.exitCode, + ); + } +} + +final class _ExampleProject { + const _ExampleProject({ + required this.path, + required this.tool, + required this.hasBuildRunner, + }); + + final String path; + final String tool; + final bool hasBuildRunner; +} diff --git a/tool/dagger_test.sh b/tool/dagger_test.sh new file mode 100755 index 00000000..692313e1 --- /dev/null +++ b/tool/dagger_test.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +DAGGER_BIN="${DAGGER_BIN:-dagger}" +DAGGER_PROGRESS="${DAGGER_PROGRESS:-plain}" +DAGGER_CALL="${1:-all}" + +if ! command -v "$DAGGER_BIN" >/dev/null 2>&1 && [[ ! -x "$DAGGER_BIN" ]]; then + echo "Dagger CLI is required. Install it or set DAGGER_BIN to its path." >&2 + exit 1 +fi + +staged_source="$(mktemp -d "${TMPDIR:-/tmp}/stem-dagger-source.XXXXXX")" +cleanup() { + rm -rf "$staged_source" +} +trap cleanup EXIT + +cd "$REPO_ROOT" + +# Dagger's source input is content-addressed before the pipeline starts. Send +# only tracked files plus non-ignored working files so local build artifacts +# cannot turn a test run into a multi-gigabyte source upload. +git ls-files --cached --others --exclude-standard -z \ + | while IFS= read -r -d '' path; do + [[ -e "$path" ]] && printf '%s\0' "$path" + done \ + | tar --null --files-from=- --create \ + | tar --directory="$staged_source" --extract + +"$DAGGER_BIN" \ + --mod .dagger \ + call "$DAGGER_CALL" \ + --source="$staged_source" \ + --progress="$DAGGER_PROGRESS" diff --git a/tool/publish.dart b/tool/publish.dart index a2bf1d05..bf72b06b 100755 --- a/tool/publish.dart +++ b/tool/publish.dart @@ -3,282 +3,473 @@ import 'dart:convert'; import 'dart:io'; -/// Packages to publish in dependency order. -final packages = [ - 'packages/stem', - 'packages/stem_adapter_tests', - 'packages/stem_sqlite', - 'packages/stem_redis', - 'packages/stem_postgres', - 'packages/stem_cli', -]; +import 'package:yaml/yaml.dart'; +/// Release automation for the publishable packages in the Dart workspace. +/// +/// The tool intentionally derives package order and membership from the root +/// workspace and package manifests. A release must not silently omit a newly +/// added package or publish a dependent package before its dependencies. Future main(List args) async { - final isDryRun = !args.contains('--force'); - final includeUnchanged = args.contains('--include-unchanged'); - final skipPublished = args.contains('--skip-published'); - final baselineRef = await _resolveBaselineRef(); + final options = _ReleaseOptions.parse(args); + final workspace = await _loadWorkspace(); print('--- Stem Release Automation ---'); - if (isDryRun) { - print('[MODE] Dry Run (use --force to actually publish)'); - } else { - print('[MODE] ACTUAL PUBLISH'); + print(options.isDryRun ? '[MODE] Dry run' : '[MODE] ACTUAL PUBLISH'); + + if (options.allowDirty && !options.planOnly) { + throw StateError( + '--allow-dirty is restricted to release planning; run the full release ' + 'gate from a clean worktree so pub.dev validates the exact commit.', + ); } - if (baselineRef == null) { - print('[INFO] No release tag found. Processing all packages.'); - } else { - print('[INFO] Baseline ref: $baselineRef'); + + if (!options.allowDirty && await _hasDirtyTree()) { + throw StateError( + 'The Git worktree is dirty. Commit release metadata before publishing, ' + 'or pass --allow-dirty for an explicitly non-reproducible dry run.', + ); } - if (includeUnchanged) { - print('[INFO] Including unchanged packages.'); + + final ordered = _topologicallySort(workspace.publishablePackages); + final selected = await _selectPackages( + ordered, + includeUnchanged: options.includeUnchanged, + ); + + if (selected.isEmpty) { + print('No changed publishable packages found.'); + return; } - if (skipPublished) { - print('[INFO] Skipping already published versions.'); + + print( + '[INFO] Release order: ${selected.map((pkg) => pkg.name).join(' -> ')}', + ); + + if (options.planOnly) { + return; } - try { - for (final pkgPath in packages) { - await _formatPackage(pkgPath); - } + for (final package in selected) { + await _validatePackage(package, checkGenerated: options.checkGenerated); + } - for (final pkgPath in packages) { - await publishPackage( - pkgPath, - isDryRun, - baselineRef: baselineRef, - includeUnchanged: includeUnchanged, - skipPublished: skipPublished, + for (final package in selected) { + final lookup = await _fetchPubPackageInfo(package.name); + if (!lookup.reachable) { + throw StateError( + 'Could not verify pub.dev state for ${package.name}; refusing to ' + 'treat an unreachable registry as unpublished.', + ); + } + final latest = _latestVersion(lookup.versions); + if (latest != null && _compareVersions(latest, package.version!) > 0) { + throw StateError( + '${package.name} is behind pub.dev: local ${package.version}, ' + 'published $latest. Recover the release metadata before publishing.', + ); + } + if (lookup.versions.contains(package.version)) { + if (options.skipPublished) { + print( + '[INFO] Skipping ${package.name} ${package.version}: already published.', + ); + continue; + } + throw StateError( + '${package.name} ${package.version} is already published. Bump the ' + 'version and add its changelog entry before releasing.', ); } - print('\n[SUCCESS] All packages processed successfully!'); - } catch (e) { - print('\n[FAILURE] Publishing interrupted: $e'); - exit(1); + + await _publishPackage(package, isDryRun: options.isDryRun); } + + print('[SUCCESS] Release checks completed.'); } -Future _formatPackage(String pkgPath, {bool exitIfChanged = true}) async { - final fullPath = _resolvePackagePath(pkgPath); - - print('\n--> Running dart format at $fullPath...'); - final process = await Process.start( - 'dart', - [ - 'format', - '.', - ...[if (exitIfChanged) '--set-exit-if-changed'], - ], - workingDirectory: fullPath, - mode: ProcessStartMode.inheritStdio, - ); - final exitCode = await process.exitCode; - if (exitCode != 0) { - throw Exception('dart format failed (exit code $exitCode)'); +class _ReleaseOptions { + const _ReleaseOptions({ + required this.isDryRun, + required this.allowDirty, + required this.includeUnchanged, + required this.skipPublished, + required this.checkGenerated, + required this.planOnly, + }); + + final bool isDryRun; + final bool allowDirty; + final bool includeUnchanged; + final bool skipPublished; + final bool checkGenerated; + final bool planOnly; + + factory _ReleaseOptions.parse(List args) { + final known = { + '--allow-dirty', + '--check-generated', + '--force', + '--include-unchanged', + '--plan', + '--skip-generated', + '--skip-published', + }; + final unknown = args.where((arg) => !known.contains(arg)); + if (unknown.isNotEmpty) { + throw ArgumentError('Unknown release option(s): ${unknown.join(', ')}'); + } + return _ReleaseOptions( + isDryRun: !args.contains('--force'), + allowDirty: args.contains('--allow-dirty'), + includeUnchanged: args.contains('--include-unchanged'), + skipPublished: args.contains('--skip-published'), + checkGenerated: !args.contains('--skip-generated'), + planOnly: args.contains('--plan'), + ); } - print('✓ dart format complete.'); } -Future publishPackage( - String pkgPath, - bool isDryRun, { - required String? baselineRef, +class _Workspace { + const _Workspace(this.publishablePackages); + + final List<_Package> publishablePackages; +} + +class _Package { + const _Package({ + required this.path, + required this.name, + required this.version, + required this.dependencies, + required this.isFlutter, + required this.hasBuildRunner, + }); + + final String path; + final String name; + final String? version; + final Set dependencies; + final bool isFlutter; + final bool hasBuildRunner; +} + +Future<_Workspace> _loadWorkspace() async { + final rootFile = File('pubspec.yaml'); + if (!rootFile.existsSync()) { + throw StateError('Run the release tool from the repository root.'); + } + + final root = loadYaml(rootFile.readAsStringSync()); + if (root is! YamlMap || root['workspace'] is! YamlList) { + throw StateError('Root pubspec.yaml has no workspace package list.'); + } + + final packages = <_Package>[]; + for (final rawPath in root['workspace'] as YamlList) { + final path = rawPath.toString(); + final pubspecFile = File('$path/pubspec.yaml'); + if (!pubspecFile.existsSync()) { + throw StateError('Workspace entry $path has no pubspec.yaml.'); + } + + final manifest = loadYaml(pubspecFile.readAsStringSync()); + if (manifest is! YamlMap) { + throw StateError('$path/pubspec.yaml is not a YAML map.'); + } + final publishTo = manifest['publish_to']?.toString(); + if (publishTo == 'none') { + continue; + } + + final name = manifest['name']?.toString(); + if (name == null || name.isEmpty) { + throw StateError('$path/pubspec.yaml has no package name.'); + } + final dependencies = { + ..._dependencyNames(manifest['dependencies']), + ..._dependencyNames(manifest['dependency_overrides']), + }; + final devDependencies = _dependencyNames(manifest['dev_dependencies']); + packages.add( + _Package( + path: path, + name: name, + version: manifest['version']?.toString(), + dependencies: dependencies, + isFlutter: dependencies.contains('flutter'), + hasBuildRunner: devDependencies.contains('build_runner'), + ), + ); + } + + return _Workspace(packages); +} + +Set _dependencyNames(Object? value) { + if (value is! YamlMap) return {}; + return value.keys.map((key) => key.toString()).toSet(); +} + +List<_Package> _topologicallySort(List<_Package> packages) { + final byName = {for (final package in packages) package.name: package}; + final state = {}; + final ordered = <_Package>[]; + + void visit(_Package package, List chain) { + final currentState = state[package.name] ?? 0; + if (currentState == 2) return; + if (currentState == 1) { + final cycle = [...chain, package.name].join(' -> '); + throw StateError('Workspace package dependency cycle: $cycle'); + } + state[package.name] = 1; + for (final dependencyName in package.dependencies) { + final dependency = byName[dependencyName]; + if (dependency != null) { + visit(dependency, [...chain, package.name]); + } + } + state[package.name] = 2; + ordered.add(package); + } + + for (final package in packages) { + visit(package, const []); + } + return ordered; +} + +Future> _selectPackages( + List<_Package> ordered, { required bool includeUnchanged, - required bool skipPublished, }) async { - final fullPath = _resolvePackagePath(pkgPath); - final pubspecFile = File('${fullPath}pubspec.yaml'); + if (includeUnchanged) return ordered; - if (!pubspecFile.existsSync()) { - throw Exception('pubspec.yaml not found in $pkgPath'); + final changed = {}; + for (final package in ordered) { + final baselineRef = await _resolveBaselineRef(package.name); + if (baselineRef == null) { + changed.add(package.name); + continue; + } + if (await _packageChangedSince(package.path, baselineRef)) { + changed.add(package.name); + } } - final pubspec = _readPubspec(pubspecFile); - final name = pubspec.name ?? _basename(pkgPath); - final version = pubspec.version; - if (!includeUnchanged) { - final changed = await _packageChangedSince(pkgPath, baselineRef); - if (!changed) { - print( - '\n--> Skipping $name ($pkgPath): no changes since ${baselineRef ?? 'initial commit'}.', - ); - return; + // A changed package must bring changed dependents through the release gate; + // otherwise a dependency update can be published without its compatible + // downstream package being validated or released. + var expanded = true; + while (expanded) { + expanded = false; + for (final package in ordered) { + if (changed.contains(package.name)) continue; + if (package.dependencies.any(changed.contains)) { + changed.add(package.name); + expanded = true; + } } } - if (skipPublished && version != null) { - final published = await _isVersionPublished(name, version); - if (published) { - print('\n--> Skipping $name ($pkgPath): $version already published.'); - return; - } + return ordered.where((package) => changed.contains(package.name)).toList(); +} + +Future _validatePackage( + _Package package, { + required bool checkGenerated, +}) async { + await _run('dart', [ + 'format', + 'lib', + 'test', + '--set-exit-if-changed', + ], package); + final executable = package.isFlutter ? 'flutter' : 'dart'; + await _run(executable, ['analyze', '--fatal-infos'], package); + final testArguments = ['test', '--fail-fast']; + if (package.path == 'packages/stem') { + testArguments.insert(1, '--exclude-tags'); + testArguments.insert(2, 'soak'); } - print('\n--> Processing $name ($pkgPath)...'); - - // 1. Dry run first (always) - print('Running dry-run check...'); - final dryRunResult = await Process.run('dart', [ - 'pub', - 'publish', - '--dry-run', - ], workingDirectory: fullPath); - - if (dryRunResult.exitCode != 0) { - print('Dry-run failed for $name:'); - print(dryRunResult.stdout); - print(dryRunResult.stderr); - throw Exception('Dry-run failed for $name'); + await _run(executable, testArguments, package); + + if (checkGenerated && package.hasBuildRunner) { + await _run('dart', [ + 'run', + 'build_runner', + 'build', + '--delete-conflicting-outputs', + ], package); + await _runGitDiffCheck(package.path); } - print('✓ Dry-run passed.'); - - if (!isDryRun) { - print('Publishing $name to pub.dev...'); - // We use inheritStdio to allow the user to see progress and handle any - // unexpected prompts, though --force should skip them. - final process = await Process.start( - 'dart', - ['pub', 'publish', '--force'], - workingDirectory: fullPath, - mode: ProcessStartMode.inheritStdio, + + final changelog = File('${package.path}/CHANGELOG.md'); + if (!changelog.existsSync()) { + throw StateError('${package.name} has no CHANGELOG.md.'); + } + final version = package.version; + if (version == null || version.isEmpty) { + throw StateError('${package.name} has no version.'); + } + final versionHeading = RegExp( + r'^##\s+' + RegExp.escape(version) + r'(?:\s|$)', + multiLine: true, + ); + if (!versionHeading.hasMatch(changelog.readAsStringSync())) { + throw StateError( + '${package.name} CHANGELOG.md has no heading for version $version.', ); + } - final exitCode = await process.exitCode; - if (exitCode != 0) { - throw Exception('Failed to publish $name (exit code $exitCode)'); - } - print('✓ Published $name.'); + await _run('dart', ['pub', 'publish', '--dry-run'], package); +} + +Future _publishPackage(_Package package, {required bool isDryRun}) async { + if (isDryRun) { + print('[DRY-RUN] ${package.name} ${package.version} is publishable.'); + return; } + await _run('dart', ['pub', 'publish', '--force'], package); } -String _resolvePackagePath(String pkgPath) { - final normalized = pkgPath.endsWith('/') ? pkgPath : '$pkgPath/'; - return Directory.current.uri.resolve(normalized).toFilePath(); +Future _run( + String executable, + List arguments, + _Package package, +) async { + print('[RUN] ${package.name}: $executable ${arguments.join(' ')}'); + final result = await Process.run( + executable, + arguments, + workingDirectory: package.path, + runInShell: true, + ); + stdout.write(result.stdout); + stderr.write(result.stderr); + if (result.exitCode != 0) { + throw ProcessException( + executable, + arguments, + 'Command failed for ${package.name}', + result.exitCode, + ); + } } -String _basename(String path) { - final parts = path.split(RegExp(r'[\\/]+')).where((part) => part.isNotEmpty); - return parts.isEmpty ? path : parts.last; +Future _runGitDiffCheck(String path) async { + final result = await Process.run('git', ['diff', '--quiet', '--', path]); + final status = await Process.run('git', [ + 'status', + '--porcelain', + '--', + path, + ]); + if (result.exitCode != 0 || (status.stdout as String).trim().isNotEmpty) { + throw StateError( + 'Generated files for $path are out of date. Run the generator and ' + 'commit its output before releasing.', + ); + } } -Future _resolveBaselineRef() async { - final result = await Process.run('git', ['describe', '--tags', '--abbrev=0']); +Future _hasDirtyTree() async { + final result = await Process.run('git', ['status', '--porcelain']); if (result.exitCode != 0) { - return null; + throw StateError('Unable to inspect Git worktree state.'); } + return (result.stdout as String).trim().isNotEmpty; +} + +Future _resolveBaselineRef(String packageName) async { + final result = await Process.run('git', [ + 'describe', + '--tags', + '--match', + '$packageName-v*', + '--abbrev=0', + ]); + if (result.exitCode != 0) return null; final tag = (result.stdout as String).trim(); return tag.isEmpty ? null : tag; } -Future _packageChangedSince(String pkgPath, String? baselineRef) async { - if (baselineRef == null) return true; +Future _packageChangedSince(String path, String baselineRef) async { final result = await Process.run('git', [ 'diff', '--name-only', '$baselineRef...HEAD', '--', - pkgPath, + path, ]); if (result.exitCode != 0) { - final stderr = (result.stderr as String).trim(); - throw Exception( - 'Failed to detect changes for $pkgPath: ' - '${stderr.isEmpty ? 'git diff failed' : stderr}', - ); + throw StateError('Unable to inspect changes for $path.'); } - final output = (result.stdout as String).trim(); - return output.isNotEmpty; + return (result.stdout as String).trim().isNotEmpty; } -_PubspecInfo _readPubspec(File pubspecFile) { - final namePattern = RegExp(r'^name:\s*(.+)$'); - final versionPattern = RegExp(r'^version:\s*(.+)$'); - String? name; - String? version; - for (final rawLine in pubspecFile.readAsLinesSync()) { - final line = rawLine.trim(); - if (line.isEmpty || line.startsWith('#')) { - continue; - } - final nameMatch = namePattern.firstMatch(line); - if (nameMatch != null) { - name = _stripQuotes(nameMatch.group(1)); - continue; - } - final versionMatch = versionPattern.firstMatch(line); - if (versionMatch != null) { - version = _stripQuotes(versionMatch.group(1)); - } - } - return _PubspecInfo(name: name, version: version); +class _PubPackageInfo { + const _PubPackageInfo({required this.reachable, required this.versions}); + + final bool reachable; + final Set versions; } -String? _stripQuotes(String? value) { - if (value == null) return null; - final trimmed = value.trim(); - if (trimmed.length >= 2) { - final start = trimmed[0]; - final end = trimmed[trimmed.length - 1]; - if ((start == '"' && end == '"') || (start == '\'' && end == '\'')) { - return trimmed.substring(1, trimmed.length - 1); - } +String? _latestVersion(Iterable versions) { + if (versions.isEmpty) return null; + return versions.reduce( + (current, candidate) => + _compareVersions(candidate, current) > 0 ? candidate : current, + ); +} + +int _compareVersions(String left, String right) { + final leftParts = _versionParts(left); + final rightParts = _versionParts(right); + for (var index = 0; index < 3; index++) { + final comparison = leftParts[index].compareTo(rightParts[index]); + if (comparison != 0) return comparison; } - return trimmed; + return 0; } -Future _isVersionPublished(String packageName, String version) async { - final info = await _fetchPubPackageInfo(packageName); - if (info == null) return false; - return info.versions.contains(version); +List _versionParts(String value) { + final match = RegExp(r'^(\d+)\.(\d+)\.(\d+)').firstMatch(value); + if (match == null) return const [0, 0, 0]; + return [ + int.parse(match.group(1)!), + int.parse(match.group(2)!), + int.parse(match.group(3)!), + ]; } -Future<_PubPackageInfo?> _fetchPubPackageInfo(String packageName) async { +Future<_PubPackageInfo> _fetchPubPackageInfo(String packageName) async { final client = HttpClient(); try { final uri = Uri.https('pub.dev', '/api/packages/$packageName'); final request = await client.getUrl(uri); final response = await request.close(); if (response.statusCode != HttpStatus.ok) { - return null; + return const _PubPackageInfo(reachable: false, versions: {}); } final body = await response.transform(utf8.decoder).join(); - final decoded = jsonDecode(body) as Map; + final decoded = jsonDecode(body); + if (decoded is! Map) { + return const _PubPackageInfo(reachable: false, versions: {}); + } final versions = {}; - final versionEntries = decoded['versions']; - if (versionEntries is List) { - for (final entry in versionEntries) { - if (entry is Map) { - final version = entry['version']; - if (version is String) { - versions.add(version); - } + final entries = decoded['versions']; + if (entries is List) { + for (final entry in entries) { + if (entry is Map && entry['version'] is String) { + versions.add(entry['version'] as String); } } } - final latest = decoded['latest']; - String? latestVersion; - if (latest is Map) { - final latestValue = latest['version']; - if (latestValue is String) { - latestVersion = latestValue; - } - } - return _PubPackageInfo(latestVersion, versions); - } catch (_) { - return null; + return _PubPackageInfo(reachable: true, versions: versions); + } on Object { + return const _PubPackageInfo(reachable: false, versions: {}); } finally { client.close(force: true); } } - -class _PubspecInfo { - const _PubspecInfo({required this.name, required this.version}); - - final String? name; - final String? version; -} - -class _PubPackageInfo { - const _PubPackageInfo(this.latestVersion, this.versions); - - final String? latestVersion; - final Set versions; -} diff --git a/tool/stage_workspace.dart b/tool/stage_workspace.dart new file mode 100644 index 00000000..ef334fa1 --- /dev/null +++ b/tool/stage_workspace.dart @@ -0,0 +1,169 @@ +import 'dart:io'; + +import 'package:yaml/yaml.dart'; + +Future main(List args) async { + final packageName = _option(args, '--package'); + final outputPath = _option(args, '--output'); + if (packageName == null || outputPath == null) { + stderr.writeln( + 'Usage: dart run tool/stage_workspace.dart ' + '--package --output ', + ); + exitCode = 64; + return; + } + + final sourceRoot = Directory.current; + final sourcePackages = Directory('${sourceRoot.path}/packages'); + final packageEntries = {}; + for (final entity in sourcePackages.listSync()) { + if (entity is! Directory) continue; + final pubspec = File('${entity.path}/pubspec.yaml'); + if (!pubspec.existsSync()) continue; + final yaml = loadYaml(pubspec.readAsStringSync()); + final name = yaml['name']; + if (name is String) { + packageEntries[name] = _PackageEntry(name: name, source: entity); + } + } + + final selectedSource = Directory('${sourceRoot.path}/$packageName'); + final selectedPubspec = File('${selectedSource.path}/pubspec.yaml'); + if (!selectedSource.existsSync() || !selectedPubspec.existsSync()) { + stderr.writeln('Unknown package path: $packageName'); + exitCode = 64; + return; + } + final selectedEntry = packageEntries.values.firstWhere( + (entry) => entry.source.path == selectedSource.path, + ); + final requiredPackages = {selectedEntry.name}; + final pendingPackages = [selectedEntry.name]; + while (pendingPackages.isNotEmpty) { + final current = pendingPackages.removeLast(); + final yaml = loadYaml( + File( + '${packageEntries[current]!.source.path}/pubspec.yaml', + ).readAsStringSync(), + ); + for (final dependency in _localDependencyNames(yaml, packageEntries)) { + if (requiredPackages.add(dependency)) { + pendingPackages.add(dependency); + } + } + } + + final outputRoot = Directory(outputPath); + if (outputRoot.existsSync()) { + if (outputRoot.listSync().isNotEmpty) { + throw StateError('Staging output directory is not empty: $outputPath'); + } + } else { + outputRoot.createSync(recursive: true); + } + final stagedPackages = Directory('${outputRoot.path}/packages') + ..createSync(recursive: true); + + final directoryByName = {}; + for (final entry in packageEntries.values.where( + (entry) => requiredPackages.contains(entry.name), + )) { + final destination = Directory('${stagedPackages.path}/${entry.sourceName}'); + _copyDirectory(entry.source, destination); + directoryByName[entry.name] = entry.sourceName; + final pubspec = File('${destination.path}/pubspec.yaml'); + final contents = pubspec.readAsStringSync().replaceFirst( + RegExp(r'^resolution:\s*workspace\s*\n', multiLine: true), + '', + ); + pubspec.writeAsStringSync(contents); + } + + for (final entry in packageEntries.values.where( + (entry) => requiredPackages.contains(entry.name), + )) { + final stagedDirectory = Directory( + '${stagedPackages.path}/${entry.sourceName}', + ); + final sourceYaml = loadYaml( + File('${entry.source.path}/pubspec.yaml').readAsStringSync(), + ); + final localDependencies = {}; + for (final name in _localDependencyNames(sourceYaml, packageEntries)) { + final sourceName = directoryByName[name]; + if (sourceName != null && sourceName != entry.sourceName) { + localDependencies[name] = '../$sourceName'; + } + } + if (localDependencies.isEmpty) continue; + + final overrides = StringBuffer('dependency_overrides:\n'); + for (final dependency in localDependencies.entries) { + overrides + ..writeln(' ${dependency.key}:') + ..writeln(' path: ${dependency.value}'); + } + File( + '${stagedDirectory.path}/pubspec_overrides.yaml', + ).writeAsStringSync(overrides.toString()); + } + + final selectedDirectory = Directory( + '${stagedPackages.path}/${selectedEntry.sourceName}', + ); + stdout.writeln(selectedDirectory.path); +} + +Iterable _localDependencyNames( + dynamic yaml, + Map packageEntries, +) sync* { + for (final section in const [ + 'dependencies', + 'dev_dependencies', + 'dependency_overrides', + ]) { + final dependencies = yaml[section]; + if (dependencies is! YamlMap) continue; + for (final dependency in dependencies.keys) { + final name = dependency.toString(); + if (packageEntries.containsKey(name)) yield name; + } + } +} + +String? _option(List args, String name) { + final index = args.indexOf(name); + if (index == -1 || index + 1 >= args.length) return null; + return args[index + 1]; +} + +void _copyDirectory(Directory source, Directory destination) { + destination.createSync(recursive: true); + for (final entity in source.listSync()) { + final name = entity.path + .split(Platform.pathSeparator) + .where((part) => part.isNotEmpty) + .last; + if (name == '.dart_tool' || name == 'build' || name == 'example') { + continue; + } + if (entity is Directory) { + _copyDirectory(entity, Directory('${destination.path}/$name')); + } else if (entity is File) { + File( + '${destination.path}/$name', + ).writeAsBytesSync(entity.readAsBytesSync()); + } + } +} + +final class _PackageEntry { + _PackageEntry({required this.name, required this.source}); + + final String name; + final Directory source; + + String get sourceName => source.path.split(Platform.pathSeparator).last; +}