From 5e1c5a534d4b8112ee13a53a094cc415cfd88ce7 Mon Sep 17 00:00:00 2001 From: Carol Xu Date: Sun, 27 Sep 2026 20:44:32 -0500 Subject: [PATCH 01/13] Add JSRPC passthroughs on R2Bucket and R2MultipartUpload --- src/workerd/api/r2-bucket.c++ | 946 ++++++++- src/workerd/api/r2-bucket.h | 289 ++- src/workerd/api/r2-multipart.c++ | 136 +- src/workerd/api/r2-multipart.h | 61 +- src/workerd/api/r2-rpc.c++ | 115 +- src/workerd/api/r2-rpc.h | 83 + src/workerd/api/r2.h | 28 +- src/workerd/api/tests/BUILD.bazel | 60 + .../api/tests/r2-body-length-test.wd-test | 41 + .../api/tests/r2-body-length-ts-test.wd-test | 25 + .../api/tests/r2-instrumentation-test.js | 197 +- .../tests/r2-jsrpc-legacy-list-test.wd-test | 24 + .../api/tests/r2-jsrpc-multipart-test.wd-test | 35 + .../api/tests/r2-jsrpc-range-test.wd-test | 37 + .../api/tests/r2-jsrpc-response-test.wd-test | 33 + .../api/tests/r2-jsrpc-serialization-test.js | 176 ++ .../tests/r2-jsrpc-serialization-test.wd-test | 46 + src/workerd/api/tests/r2-jsrpc-test.wd-test | 49 + src/workerd/api/tests/r2-test.js | 1864 ++++++++++++++++- src/workerd/api/tests/r2-test.wd-test | 4 + src/workerd/io/compatibility-date.capnp | 7 + src/workerd/io/worker-interface.capnp | 12 + 22 files changed, 4048 insertions(+), 220 deletions(-) create mode 100644 src/workerd/api/tests/r2-body-length-test.wd-test create mode 100644 src/workerd/api/tests/r2-body-length-ts-test.wd-test create mode 100644 src/workerd/api/tests/r2-jsrpc-legacy-list-test.wd-test create mode 100644 src/workerd/api/tests/r2-jsrpc-multipart-test.wd-test create mode 100644 src/workerd/api/tests/r2-jsrpc-range-test.wd-test create mode 100644 src/workerd/api/tests/r2-jsrpc-response-test.wd-test create mode 100644 src/workerd/api/tests/r2-jsrpc-serialization-test.js create mode 100644 src/workerd/api/tests/r2-jsrpc-serialization-test.wd-test create mode 100644 src/workerd/api/tests/r2-jsrpc-test.wd-test diff --git a/src/workerd/api/r2-bucket.c++ b/src/workerd/api/r2-bucket.c++ index bddb57422f6..1af64e88af1 100644 --- a/src/workerd/api/r2-bucket.c++ +++ b/src/workerd/api/r2-bucket.c++ @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -69,11 +70,46 @@ kj::Own R2Bucket::getHttpClient(IoContext& context, TraceContext KJ_UNREACHABLE; } +TraceContext R2Bucket::makeR2TraceContext(kj::LiteralStringConst spanName, + kj::LiteralStringConst operation, + kj::Maybe key) { + TraceContext traceContext = IoContext::current().makeUserTraceSpan(spanName); + traceContext.setTag("cloudflare.binding.type"_kjc, "r2"_kjc); + KJ_IF_SOME(name, bindingName()) { + traceContext.setTag("cloudflare.binding.name"_kjc, name); + } + traceContext.setTag("cloudflare.r2.operation"_kjc, operation); + KJ_IF_SOME(name, bucketName()) { + traceContext.setTag("cloudflare.r2.bucket"_kjc, name); + } + KJ_IF_SOME(value, key) { + traceContext.setTag("cloudflare.r2.request.key"_kjc, value); + } + return traceContext; +} + static bool isWholeNumber(double x) { double intpart; return modf(x, &intpart) == 0; } +static jsg::Optional> copyBytes( + const jsg::Optional>& bytes) { + return bytes.map([](const kj::Array& value) { return kj::heapArray(value); }); +} + +static jsg::Dict copyStringDict(const jsg::Dict& dict) { + return { + .fields = + KJ_MAP(field, dict.fields) { + return jsg::Dict::Field{ + .name = kj::str(field.name), + .value = kj::str(field.value), + }; + }, + }; +} + // TODO(perf): Would be nice to expose the v8 internals for parsing a date/stringifying it as // something an embedder can call directly rather than doing this rigamarole. It would also avoid // concerns about the user overriding the methods we're invoking. @@ -232,6 +268,30 @@ static kj::Maybe> parseObjectMetadata(jsg::Lock& js, namespace { +class R2GetBodyStream final: public ReadableStreamSource { + public: + R2GetBodyStream(kj::Own inner, kj::Promise pump) + : inner(kj::mv(inner)), + pump(kj::mv(pump).catch_([](kj::Exception&&) {}).eagerlyEvaluate(nullptr)) {} + + kj::Promise tryRead(void* buffer, size_t minBytes, size_t maxBytes) override { + return inner->tryRead(buffer, minBytes, maxBytes); + } + + kj::Maybe tryGetLength(StreamEncoding encoding) override { + return inner->tryGetLength(encoding); + } + + void cancel(kj::Exception reason) override { + pump = nullptr; + inner->cancel(kj::mv(reason)); + } + + private: + kj::Own inner; + kj::Promise pump; +}; + void addEtagsToBuilder( capnp::List::Builder etagListBuilder, kj::ArrayPtr etagArray) { R2Bucket::Etag* currentEtag = etagArray.begin(); @@ -272,6 +332,31 @@ kj::String buildEtagsString(kj::ArrayPtr etagArray) { }, ", "); } +R2Bucket::SerializableConditional makeSerializableConditional( + jsg::Lock& js, R2Bucket::UnwrappedConditional& conditional, TraceContext& traceContext) { + R2Bucket::SerializableConditional rpcConditional{ + .secondsGranularity = conditional.secondsGranularity}; + KJ_IF_SOME(etags, conditional.etagMatches) { + auto value = buildEtagsString(etags); + traceContext.setTag("cloudflare.r2.request.only_if.etag_matches"_kjc, value.asPtr()); + rpcConditional.etagMatches = kj::mv(value); + } + KJ_IF_SOME(etags, conditional.etagDoesNotMatch) { + auto value = buildEtagsString(etags); + traceContext.setTag("cloudflare.r2.request.only_if.etag_does_not_match"_kjc, value.asPtr()); + rpcConditional.etagDoesNotMatch = kj::mv(value); + } + KJ_IF_SOME(date, conditional.uploadedBefore) { + traceContext.setTag("cloudflare.r2.request.only_if.uploaded_before"_kjc, toISOString(js, date)); + rpcConditional.uploadedBefore = date; + } + KJ_IF_SOME(date, conditional.uploadedAfter) { + traceContext.setTag("cloudflare.r2.request.only_if.uploaded_after"_kjc, toISOString(js, date)); + rpcConditional.uploadedAfter = date; + } + return rpcConditional; +} + template void initOnlyIf(TraceContext& traceContext, jsg::Lock& js, Builder& builder, Options& o) { KJ_IF_SOME(i, o.onlyIf) { @@ -319,6 +404,7 @@ void initOnlyIf(TraceContext& traceContext, jsg::Lock& js, Builder& builder, Opt } } +// Validates and performs string conversion on the SSE-C key. kj::Maybe buildSsecKey( kj::Maybe, kj::String>> maybeRawSsecKey) { KJ_IF_SOME(rawSsecKey, maybeRawSsecKey) { @@ -338,6 +424,51 @@ kj::Maybe buildSsecKey( return kj::none; } +static R2Bucket::HttpMetadata normalizeHttpMetadata( + jsg::Lock& js, kj::OneOf> metadata) { + KJ_SWITCH_ONEOF(metadata) { + KJ_CASE_ONEOF(value, R2Bucket::HttpMetadata) { + return kj::mv(value); + } + KJ_CASE_ONEOF(headers, jsg::Ref) { + return R2Bucket::HttpMetadata::fromRequestHeaders(js, *headers); + } + } + KJ_UNREACHABLE; +} + +/***************************** + * + * Helper functions to set various tags on traceContext. + * - addHttpMetadataRequestSpanTags + * - addR2ResponseSpanTags + * - addHeadResultSpanTags + * + ******************************/ +static void addHttpMetadataRequestSpanTags( + TraceContext& traceContext, const R2Bucket::HttpMetadata& metadata) { + KJ_IF_SOME(value, metadata.contentType) { + traceContext.setTag("cloudflare.r2.request.http_metadata.content_type"_kjc, value.asPtr()); + } + KJ_IF_SOME(value, metadata.contentEncoding) { + traceContext.setTag("cloudflare.r2.request.http_metadata.content_encoding"_kjc, value.asPtr()); + } + KJ_IF_SOME(value, metadata.contentDisposition) { + traceContext.setTag( + "cloudflare.r2.request.http_metadata.content_disposition"_kjc, value.asPtr()); + } + KJ_IF_SOME(value, metadata.contentLanguage) { + traceContext.setTag("cloudflare.r2.request.http_metadata.content_language"_kjc, value.asPtr()); + } + KJ_IF_SOME(value, metadata.cacheControl) { + traceContext.setTag("cloudflare.r2.request.http_metadata.cache_control"_kjc, value.asPtr()); + } + KJ_IF_SOME(value, metadata.cacheExpiry) { + traceContext.setTag("cloudflare.r2.request.http_metadata.cache_expiry"_kjc, + (value - kj::UNIX_EPOCH) / kj::MILLISECONDS); + } +} + static void addR2ResponseSpanTags(TraceContext& traceContext, R2Result& r2Result) { traceContext.setTag("cloudflare.r2.response.success"_kjc, r2Result.success()); KJ_IF_SOME(e, r2Result.getR2ErrorMessage()) { @@ -401,6 +532,17 @@ void addHeadResultSpanTags( } } +static void addListResultSpanTags(TraceContext& traceContext, const R2Bucket::ListResult& result) { + KJ_IF_SOME(cursor, result.cursor) { + traceContext.setTag("cloudflare.r2.response.cursor"_kjc, cursor.asPtr()); + } + traceContext.setTag( + "cloudflare.r2.response.returned_objects"_kjc, static_cast(result.objects.size())); + traceContext.setTag("cloudflare.r2.response.delimited_prefixes"_kjc, + static_cast(result.delimitedPrefixes.size())); + traceContext.setTag("cloudflare.r2.response.truncated"_kjc, result.truncated); +} + template void initGetOptions(TraceContext& traceContext, jsg::Lock& js, Builder& builder, Options& o) { initOnlyIf(traceContext, js, builder, o); @@ -469,17 +611,7 @@ jsg::Promise>> R2Bucket::head(jsg::Lock CompatibilityFlags::Reader flags) { return js.evalNow([&] { auto& context = IoContext::current(); - TraceContext traceContext = context.makeUserTraceSpan("r2_head"_kjc); - - traceContext.setTag("cloudflare.binding.type"_kjc, "r2"_kjc); - KJ_IF_SOME(b, this->bindingName()) { - traceContext.setTag("cloudflare.binding.name"_kjc, b); - } - traceContext.setTag("cloudflare.r2.operation"_kjc, "HeadObject"_kjc); - KJ_IF_SOME(b, this->bucketName()) { - traceContext.setTag("cloudflare.r2.bucket"_kjc, b); - } - traceContext.setTag("cloudflare.r2.request.key"_kjc, key.asPtr()); + TraceContext traceContext = makeR2TraceContext("r2_head"_kjc, "HeadObject"_kjc, key.asPtr()); capnp::JsonCodec json; json.handleByAnnotation(); @@ -512,6 +644,536 @@ jsg::Promise>> R2Bucket::head(jsg::Lock }); } +// Retrieves the RPC method with name methodName. +jsg::Ref R2Bucket::getRpcMethod(jsg::Lock& js, kj::StringPtr methodName) { + auto fetcher = [&]() -> jsg::Ref { + KJ_SWITCH_ONEOF(clientChannel) { + KJ_CASE_ONEOF(channel, uint) { + return js.alloc( + channel, Fetcher::RequiresHostAndProtocol::NO, true /* isInHouse */); + } + KJ_CASE_ONEOF(channel, IoOwn) { + return js.alloc(IoContext::current().addObject(kj::addRef(*channel)), + Fetcher::RequiresHostAndProtocol::NO, true /* isInHouse */); + } + } + KJ_UNREACHABLE; + }(); + + // getRpcMethodInternal skips the `rpc` compatibility gate, which matters because R2 bindings must + // keep working on compat dates older than that flag. The lookup is lazy and never fails for a + // real method name -- whether the entrypoint actually implements it is only discovered when the + // call reaches the far side. + return KJ_ASSERT_NONNULL(fetcher->getRpcMethodInternal(js, kj::str(methodName))); +} + +// Validates HeadResult shape from JSRPC call. +jsg::Ref headResultFromSerializable( + jsg::Lock& js, R2Bucket::SerializableHeadResult rpc, MissingMetadataPolicy policy) { + KJ_ASSERT(std::isfinite(rpc.size) && rpc.size >= 0 && isWholeNumber(rpc.size), + "Malformed R2 RPC result: size must be a non-negative integer."); + KJ_IF_SOME(range, rpc.range) { + KJ_IF_SOME(offset, range.offset) { + KJ_ASSERT(std::isfinite(offset) && offset >= 0 && isWholeNumber(offset), + "Malformed R2 RPC result: range offset must be a non-negative integer."); + } + KJ_IF_SOME(length, range.length) { + KJ_ASSERT(std::isfinite(length) && length >= 0 && isWholeNumber(length), + "Malformed R2 RPC result: range length must be a non-negative integer."); + } + KJ_IF_SOME(suffix, range.suffix) { + KJ_ASSERT(range.offset == kj::none && range.length == kj::none, + "Malformed R2 RPC result: range suffix is incompatible with offset or length."); + KJ_ASSERT(std::isfinite(suffix) && suffix >= 0 && isWholeNumber(suffix), + "Malformed R2 RPC result: range suffix must be a non-negative integer."); + } + } + + jsg::Optional httpMetadata; + jsg::Optional> customMetadata; + switch (policy) { + case MissingMetadataPolicy::EMPTY: + httpMetadata = kj::mv(rpc.httpMetadata).orDefault(R2Bucket::HttpMetadata{}); + customMetadata = kj::mv(rpc.customMetadata).orDefault(jsg::Dict{}); + break; + case MissingMetadataPolicy::ABSENT: + KJ_IF_SOME(metadata, rpc.httpMetadata) { + httpMetadata = kj::mv(metadata); + } + KJ_IF_SOME(metadata, rpc.customMetadata) { + customMetadata = kj::mv(metadata); + } + break; + } + + return js.alloc(kj::mv(rpc.key), kj::mv(rpc.version), rpc.size, + kj::mv(rpc.etag), + js.alloc(kj::mv(rpc.checksums.md5), kj::mv(rpc.checksums.sha1), + kj::mv(rpc.checksums.sha256), kj::mv(rpc.checksums.sha384), kj::mv(rpc.checksums.sha512)), + rpc.uploaded, kj::mv(httpMetadata), kj::mv(customMetadata), kj::mv(rpc.range), + kj::mv(rpc.storageClass), kj::mv(rpc.ssecKeyMd5)); +} + +static jsg::Ref getResultFromSerializable( + jsg::Lock& js, R2Bucket::SerializableGetResult rpc) { + KJ_ASSERT(rpc.kind == "body", "Malformed R2 get RPC result: expected a body result."); + auto body = KJ_ASSERT_NONNULL( + kj::mv(rpc.body), "Malformed R2 get RPC result: body result did not have a body."); + auto cancelReader = kj::defer( + [&] { body.forceCancel(js, js.error("Malformed R2 get RPC result.")).markAsHandled(js); }); + auto object = headResultFromSerializable(js, kj::mv(rpc.object), MissingMetadataPolicy::EMPTY); + + // Retrieve body length from range/size metadata to avoid depending on the stream to report it. + auto bodyLength = object->getSize(); + KJ_IF_SOME(range, object->getRange()) { + bodyLength = range.length.orDefault(bodyLength); + } + KJ_ASSERT(bodyLength <= 9007199254740991ull, + "Malformed R2 get RPC result: body length must be a safe integer."); + auto& context = IoContext::current(); + auto pipe = newIdentityPipe(static_cast(bodyLength)); + auto pump = context.waitForDeferredProxy(body.pumpTo(js, kj::mv(pipe.out), EndStream::YES)); + body = JsReadableStream::create( + js, context, kj::heap(kj::mv(pipe.in), kj::mv(pump))); + cancelReader.cancel(); + + auto customMetadata = object->getCustomMetadata(); + return js.alloc(kj::str(object->getName()), kj::str(object->getVersion()), + object->getSize(), kj::str(object->getEtag()), object->getChecksums(), object->getUploaded(), + object->getHttpMetadata(), kj::mv(customMetadata), object->getRange(), + kj::str(object->getStorageClass()), mapCopyString(object->getSSECKeyMd5()), kj::mv(body)); +} + +jsg::Ref headResultFromBackend( + jsg::Lock& js, R2Bucket::HeadBackendResult result, MissingMetadataPolicy policy) { + KJ_SWITCH_ONEOF(result) { + KJ_CASE_ONEOF(native, jsg::Ref) { + if (dynamic_cast(native.get()) != nullptr) { + auto body = static_cast(native.get())->getBody(js); + body.forceCancel(js, js.error("Malformed R2 RPC result: expected object metadata.")) + .markAsHandled(js); + KJ_FAIL_ASSERT( + "Malformed R2 RPC result: expected object metadata, received an object body."); + } + if (policy == MissingMetadataPolicy::EMPTY) { + bool hasHttpMetadata = native->getHttpMetadata() != kj::none; + bool hasCustomMetadata = native->getCustomMetadata() != kj::none; + KJ_ASSERT(hasHttpMetadata, + "Malformed R2 RPC result: object metadata did not include HTTP metadata."); + KJ_ASSERT(hasCustomMetadata, + "Malformed R2 RPC result: object metadata did not include custom metadata."); + } + return kj::mv(native); + } + KJ_CASE_ONEOF(rpc, R2Bucket::SerializableHeadResult) { + return headResultFromSerializable(js, kj::mv(rpc), policy); + } + } + KJ_UNREACHABLE; +} + +jsg::Promise>> R2Bucket::headRpc(jsg::Lock& js, + kj::String key, + const jsg::TypeHandler>& rpcPropHandler, + const jsg::TypeHandler>& headFnHandler, + const jsg::TypeHandler>>& headResultHandler) { + return js.evalNow([&] { + TraceContext traceContext = makeR2TraceContext("r2_head"_kjc, "HeadObject"_kjc, key.asPtr()); + + return callR2RpcMethod>(js, getRpcMethod(js, "head"_kj), + rpcPropHandler, headFnHandler, headResultHandler, kj::mv(key)) + .then(js, + [traceContext = kj::mv(traceContext)](jsg::Lock& js, + kj::Maybe parsed) mutable -> kj::Maybe> { + // A missing object is null, not an error: the gateway maps the 404 that + // R2Result::objectNotFound() used to represent onto a null return. + KJ_IF_SOME(backend, parsed) { + auto result = headResultFromBackend(js, kj::mv(backend)); + addHeadResultSpanTags(js, traceContext, *result.get()); + return kj::mv(result); + } + return kj::none; + }); + }); +} + +jsg::Promise>, jsg::Ref>> +R2Bucket::getRpc(jsg::Lock& js, + kj::String key, + jsg::Optional options, + const jsg::TypeHandler>& rpcPropHandler, + const jsg::TypeHandler< + jsg::Function)>>& getFnHandler, + const jsg::TypeHandler>>& getResultHandler) { + return js.evalNow([&] { + TraceContext traceContext = makeR2TraceContext("r2_get"_kjc, "GetObject"_kjc, key.asPtr()); + + jsg::Optional rpcOptions; + KJ_IF_SOME(o, options) { + SerializableGetOptions normalized; + + KJ_IF_SOME(i, o.onlyIf) { + UnwrappedConditional conditional = [&] { + KJ_SWITCH_ONEOF(i) { + KJ_CASE_ONEOF(value, Conditional) { + return UnwrappedConditional(value); + } + KJ_CASE_ONEOF(headers, jsg::Ref) { + return UnwrappedConditional(js, *headers); + } + } + KJ_UNREACHABLE; + }(); + + auto rpcConditional = makeSerializableConditional(js, conditional, traceContext); + + KJ_SWITCH_ONEOF(i) { + KJ_CASE_ONEOF(conditional, Conditional) { + normalized.onlyIf = kj::mv(rpcConditional); + } + KJ_CASE_ONEOF(headers, jsg::Ref) { + normalized.onlyIf = kj::mv(headers); + } + } + } + + KJ_IF_SOME(range, o.range) { + KJ_SWITCH_ONEOF(range) { + KJ_CASE_ONEOF(value, Range) { + kj::Vector::Field> fields; + KJ_IF_SOME(offset, value.offset) { + JSG_REQUIRE(offset >= 0, RangeError, "Invalid range. Starting offset (", offset, + ") must be greater than or equal to 0."); + JSG_REQUIRE(isWholeNumber(offset), RangeError, "Invalid range. Starting offset (", + offset, ") must be an integer, not floating point."); + traceContext.setTag( + "cloudflare.r2.request.range.offset"_kjc, static_cast(offset)); + fields.add(jsg::Dict::Field{.name = kj::str("offset"), .value = offset}); + } + KJ_IF_SOME(length, value.length) { + JSG_REQUIRE(length >= 0, RangeError, "Invalid range. Length (", length, + ") must be greater than or equal to 0."); + JSG_REQUIRE(isWholeNumber(length), RangeError, "Invalid range. Length (", length, + ") must be an integer, not floating point."); + traceContext.setTag( + "cloudflare.r2.request.range.length"_kjc, static_cast(length)); + fields.add(jsg::Dict::Field{.name = kj::str("length"), .value = length}); + } + KJ_IF_SOME(suffix, value.suffix) { + JSG_REQUIRE( + value.offset == kj::none, TypeError, "Suffix is incompatible with offset."); + JSG_REQUIRE( + value.length == kj::none, TypeError, "Suffix is incompatible with length."); + JSG_REQUIRE(suffix >= 0, RangeError, "Invalid suffix. Suffix (", suffix, + ") must be greater than or equal to 0."); + JSG_REQUIRE(isWholeNumber(suffix), RangeError, "Invalid range. Suffix (", suffix, + ") must be an integer, not floating point."); + traceContext.setTag( + "cloudflare.r2.request.range.suffix"_kjc, static_cast(suffix)); + fields.add(jsg::Dict::Field{.name = kj::str("suffix"), .value = suffix}); + } + normalized.range = jsg::Dict{.fields = fields.releaseAsArray()}; + } + KJ_CASE_ONEOF(headers, jsg::Ref) { + KJ_IF_SOME(value, headers->getCommon(js, capnp::CommonHeaderName::RANGE)) { + traceContext.setTag("cloudflare.r2.request.range"_kjc, value.asPtr()); + normalized.range = kj::mv(headers); + } + } + } + } + + KJ_IF_SOME(ssecKey, buildSsecKey(kj::mv(o.ssecKey))) { + traceContext.setTag("cloudflare.r2.request.ssec_key"_kjc, true); + normalized.ssecKey = kj::mv(ssecKey); + } + rpcOptions = kj::mv(normalized); + } + + return callR2RpcMethod>(js, getRpcMethod(js, "get"_kj), + rpcPropHandler, getFnHandler, getResultHandler, kj::mv(key), kj::mv(rpcOptions)) + .then(js, + [traceContext = kj::mv(traceContext)]( + jsg::Lock& js, kj::Maybe parsed) mutable + -> kj::OneOf>, jsg::Ref> { + KJ_IF_SOME(backend, parsed) { + KJ_SWITCH_ONEOF(backend) { + KJ_CASE_ONEOF(native, jsg::Ref) { + addHeadResultSpanTags(js, traceContext, *native.get()); + return kj::Maybe>(kj::mv(native)); + } + KJ_CASE_ONEOF(native, jsg::Ref) { + auto result = headResultFromBackend( + js, HeadBackendResult(kj::mv(native)), MissingMetadataPolicy::EMPTY); + addHeadResultSpanTags(js, traceContext, *result.get()); + return kj::mv(result); + } + KJ_CASE_ONEOF(rpc, SerializableGetResult) { + if (rpc.kind == "metadata") { + KJ_IF_SOME(body, rpc.body) { + body.forceCancel( + js, js.error("Malformed R2 get RPC result: metadata result had a body.")); + KJ_FAIL_ASSERT("Malformed R2 get RPC result: metadata result had a body."); + } + auto result = + headResultFromSerializable(js, kj::mv(rpc.object), MissingMetadataPolicy::EMPTY); + addHeadResultSpanTags(js, traceContext, *result.get()); + return kj::mv(result); + } + if (rpc.kind == "body") { + auto result = getResultFromSerializable(js, kj::mv(rpc)); + addHeadResultSpanTags(js, traceContext, *result.get()); + return kj::Maybe>(kj::mv(result)); + } + + KJ_IF_SOME(body, rpc.body) { + body.forceCancel(js, js.error("Malformed R2 get RPC result: unknown result kind.")); + } + KJ_FAIL_ASSERT("Malformed R2 get RPC result: unknown result kind ", rpc.kind, "."); + } + } + } + return kj::Maybe>(kj::none); + }); + }); +} + +jsg::Promise R2Bucket::deleteRpc(jsg::Lock& js, + kj::OneOf> keys, + const jsg::TypeHandler>& rpcPropHandler, + const jsg::TypeHandler>)>>& + deleteFnHandler, + const jsg::TypeHandler>& deleteResultHandler) { + return js.evalNow([&] { + auto& context = IoContext::current(); + TraceContext traceContext = makeR2TraceContext("r2_delete"_kjc, "DeleteObject"_kjc); + traceContext.setTag("cloudflare.r2.request.keys"_kjc, kj::str(keys)); + + // The result is discarded, matching delete_: a missing key is success, and per-key failures in + // a batch delete are reported in a body the binding has never read. + auto promise = callR2RpcMethod(js, getRpcMethod(js, "delete"_kj), rpcPropHandler, + deleteFnHandler, deleteResultHandler, kj::mv(keys)); + return context.attachSpans(js, kj::mv(promise), kj::mv(traceContext)); + }); +} + +jsg::Promise>> R2Bucket::putRpc(jsg::Lock& js, + kj::String key, + kj::Maybe value, + jsg::Optional options, + const jsg::TypeHandler>& rpcPropHandler, + const jsg::TypeHandler, + jsg::Optional, + double)>>& putFnHandler, + const jsg::TypeHandler>>& putResultHandler) { + return js.evalNow([&] { + auto cancelReader = kj::defer([&] { + KJ_IF_SOME(v, value) { + KJ_SWITCH_ONEOF(v) { + KJ_CASE_ONEOF(stream, JsReadableStream) { + stream.cancel(js, + js.error( + "Stream cancelled because the associated put operation encountered an error.")); + } + KJ_CASE_ONEOF_DEFAULT {} + } + } + }); + + TraceContext traceContext = makeR2TraceContext("r2_put"_kjc, "PutObject"_kjc, key.asPtr()); + + bool hashAlreadySpecified = false; + const auto prepareChecksum = + [&](auto checksum, kj::StringPtr algorithm, kj::StringPtr tag, size_t byteLength, + size_t hexLength) -> jsg::Optional, kj::String>> { + KJ_IF_SOME(c, checksum) { + JSG_REQUIRE( + !hashAlreadySpecified, TypeError, "You cannot specify multiple hashing algorithms."); + hashAlreadySpecified = true; + KJ_SWITCH_ONEOF(c) { + KJ_CASE_ONEOF(buffer, jsg::JsRef) { + auto bytes = buffer.getHandle(js); + JSG_REQUIRE(bytes.size() == byteLength, TypeError, algorithm, " is ", byteLength, + " bytes, not ", bytes.size()); + auto value = kj::heapArray(bytes.asArrayPtr()); + traceContext.setTag("cloudflare.r2.request.checksum.type"_kjc, tag); + traceContext.setTag( + "cloudflare.r2.request.checksum.value"_kjc, kj::encodeHex(value.asPtr())); + return kj::mv(value); + } + KJ_CASE_ONEOF(text, jsg::NonCoercible) { + JSG_REQUIRE(text.value.size() == hexLength, TypeError, algorithm, " is ", hexLength, + " hex characters, not ", text.value.size()); + auto decoded = kj::decodeHex(text.value); + JSG_REQUIRE(!decoded.hadErrors, TypeError, "Provided ", algorithm, + " wasn't a valid hex string"); + traceContext.setTag("cloudflare.r2.request.checksum.type"_kjc, tag); + traceContext.setTag("cloudflare.r2.request.checksum.value"_kjc, text.value.asPtr()); + return kj::heapArray(decoded); + } + } + } + return kj::none; + }; + + jsg::Optional rpcOptions; + KJ_IF_SOME(o, options) { + jsg::Optional>> onlyIf; + KJ_IF_SOME(condition, o.onlyIf) { + UnwrappedConditional conditional = [&] { + KJ_SWITCH_ONEOF(condition) { + KJ_CASE_ONEOF(value, Conditional) { + return UnwrappedConditional(value); + } + KJ_CASE_ONEOF(headers, jsg::Ref) { + return UnwrappedConditional(js, *headers); + } + } + KJ_UNREACHABLE; + }(); + + auto rpcConditional = makeSerializableConditional(js, conditional, traceContext); + + KJ_SWITCH_ONEOF(condition) { + KJ_CASE_ONEOF(conditional, Conditional) { + onlyIf = kj::mv(rpcConditional); + } + KJ_CASE_ONEOF(headers, jsg::Ref) { + onlyIf = kj::mv(headers); + } + } + } + + jsg::Optional>> httpMetadata; + KJ_IF_SOME(metadata, o.httpMetadata) { + auto normalized = normalizeHttpMetadata(js, kj::mv(metadata)); + addHttpMetadataRequestSpanTags(traceContext, normalized); + httpMetadata = kj::mv(normalized); + } + KJ_IF_SOME(_, o.customMetadata) { + traceContext.setTag("cloudflare.r2.request.custom_metadata"_kjc, true); + } + KJ_IF_SOME(storageClass, o.storageClass) { + traceContext.setTag("cloudflare.r2.request.storage_class"_kjc, storageClass.asPtr()); + } + jsg::Optional, kj::String>> ssecKey; + KJ_IF_SOME(key, buildSsecKey(kj::mv(o.ssecKey))) { + traceContext.setTag("cloudflare.r2.request.ssec_key"_kjc, true); + ssecKey = kj::mv(key); + } + + rpcOptions = SerializablePutOptions{ + .onlyIf = kj::mv(onlyIf), + .httpMetadata = kj::mv(httpMetadata), + .customMetadata = kj::mv(o.customMetadata), + .md5 = prepareChecksum(kj::mv(o.md5), "MD5"_kj, "md5"_kj, 16, 32), + .sha1 = prepareChecksum(kj::mv(o.sha1), "SHA-1"_kj, "sha1"_kj, 20, 40), + .sha256 = prepareChecksum(kj::mv(o.sha256), "SHA-256"_kj, "sha256"_kj, 32, 64), + .sha384 = prepareChecksum(kj::mv(o.sha384), "SHA-384"_kj, "sha384"_kj, 48, 96), + .sha512 = prepareChecksum(kj::mv(o.sha512), "SHA-512"_kj, "sha512"_kj, 64, 128), + .storageClass = kj::mv(o.storageClass), + .ssecKey = kj::mv(ssecKey), + }; + } + + auto rpcMethod = getRpcMethod(js, "put"_kj); + auto wrappedMethod = rpcPropHandler.wrap(js, kj::mv(rpcMethod)); + auto putFn = KJ_ASSERT_NONNULL(putFnHandler.tryUnwrap(js, wrappedMethod)); + kj::Maybe rpcValue; + double valueSize = 0; + KJ_IF_SOME(v, value) { + auto prepared = prepareR2RpcBody(js, v); + rpcValue = kj::mv(prepared.value); + valueSize = prepared.size; + } + traceContext.setTag("cloudflare.r2.request.size"_kjc, static_cast(valueSize)); + cancelReader.cancel(); + + auto rpcPromise = putFn(js, kj::mv(key), kj::mv(rpcValue), kj::mv(rpcOptions), valueSize); + auto promise = + unwrapR2RpcPromise>(js, kj::mv(rpcPromise), putResultHandler); + return promise.then(js, + [traceContext = kj::mv(traceContext)](jsg::Lock& js, + kj::Maybe parsed) mutable -> kj::Maybe> { + KJ_IF_SOME(backend, parsed) { + auto result = headResultFromBackend(js, kj::mv(backend)); + addHeadResultSpanTags(js, traceContext, *result.get()); + return kj::mv(result); + } + return kj::none; + }); + }); +} + +jsg::Promise> R2Bucket::createMultipartUploadRpc(jsg::Lock& js, + kj::String key, + jsg::Optional options, + const jsg::TypeHandler>& rpcPropHandler, + const jsg::TypeHandler)>>& + createFnHandler, + const jsg::TypeHandler>& uploadResultHandler) { + return js.evalNow([&] { + TraceContext traceContext = makeR2TraceContext( + "r2_createMultipartUpload"_kjc, "CreateMultipartUpload"_kjc, key.asPtr()); + + R2MultipartUpload::Metadata metadata; + KJ_IF_SOME(o, options) { + KJ_IF_SOME(metadata, o.httpMetadata) { + auto normalized = normalizeHttpMetadata(js, kj::mv(metadata)); + addHttpMetadataRequestSpanTags(traceContext, normalized); + o.httpMetadata = kj::mv(normalized); + } + KJ_IF_SOME(_, o.customMetadata) { + traceContext.setTag("cloudflare.r2.request.custom_metadata"_kjc, true); + } + KJ_IF_SOME(storageClass, o.storageClass) { + traceContext.setTag("cloudflare.r2.request.storage_class"_kjc, storageClass.asPtr()); + } + KJ_IF_SOME(ssecKey, buildSsecKey(kj::mv(o.ssecKey))) { + traceContext.setTag("cloudflare.r2.request.ssec_key"_kjc, true); + o.ssecKey = kj::mv(ssecKey); + } + } + + KJ_IF_SOME(o, options) { + KJ_IF_SOME(httpMetadata, o.httpMetadata) { + metadata.httpMetadata = httpMetadata.get().clone(); + } + KJ_IF_SOME(customMetadata, o.customMetadata) { + metadata.customMetadata.fields = KJ_MAP(field, customMetadata.fields) { + return jsg::Dict::Field{ + .name = kj::str(field.name), .value = kj::str(field.value)}; + }; + } + } + + auto uploadPromise = callR2RpcMethod(js, + getRpcMethod(js, "createMultipartUpload"_kj), rpcPropHandler, createFnHandler, + uploadResultHandler, kj::str(key), kj::mv(options)); + + return uploadPromise.then(js, + [bucket = JSG_THIS, key = kj::mv(key), metadata = kj::mv(metadata), + traceContext = kj::mv(traceContext)]( + jsg::Lock& js, MultipartUploadBackendResult backend) mutable { + kj::String uploadId; + KJ_SWITCH_ONEOF(backend) { + KJ_CASE_ONEOF(id, kj::String) { + uploadId = kj::mv(id); + } + KJ_CASE_ONEOF(handle, SerializableMultipartUploadHandle) { + KJ_ASSERT(handle.key == key, + "Malformed R2 createMultipartUpload RPC result: handle key did not match request."); + uploadId = kj::mv(handle.uploadId); + } + } + traceContext.setTag("cloudflare.r2.response.upload_id"_kjc, uploadId.asPtr()); + return js.alloc( + kj::mv(key), kj::mv(uploadId), kj::mv(bucket), kj::mv(metadata)); + }); + }); +} + R2Bucket::FeatureFlags::FeatureFlags(CompatibilityFlags::Reader featureFlags) : listHonorsIncludes(featureFlags.getR2ListHonorIncludeFields()) {} @@ -543,6 +1205,7 @@ jsg::Ref R2Bucket::deserialize( } } +//////////////////////////////////////////////////////////////////////////////////////////////// jsg::Promise>, jsg::Ref>> R2Bucket::get(jsg::Lock& js, kj::String key, @@ -551,17 +1214,7 @@ R2Bucket::get(jsg::Lock& js, CompatibilityFlags::Reader flags) { return js.evalNow([&] { auto& context = IoContext::current(); - TraceContext traceContext = context.makeUserTraceSpan("r2_get"_kjc); - - traceContext.setTag("cloudflare.binding.type"_kjc, "r2"_kjc); - KJ_IF_SOME(b, this->bindingName()) { - traceContext.setTag("cloudflare.binding.name"_kjc, b); - } - traceContext.setTag("cloudflare.r2.operation"_kjc, "GetObject"_kjc); - KJ_IF_SOME(b, this->bucketName()) { - traceContext.setTag("cloudflare.r2.bucket"_kjc, b); - } - traceContext.setTag("cloudflare.r2.request.key"_kjc, key.asPtr()); + TraceContext traceContext = makeR2TraceContext("r2_get"_kjc, "GetObject"_kjc, key.asPtr()); capnp::JsonCodec json; json.handleByAnnotation(); @@ -638,17 +1291,7 @@ jsg::Promise>> R2Bucket::put(jsg::Lock& }); auto& context = IoContext::current(); - TraceContext traceContext = context.makeUserTraceSpan("r2_put"_kjc); - - traceContext.setTag("cloudflare.binding.type"_kjc, "r2"_kjc); - KJ_IF_SOME(b, this->bindingName()) { - traceContext.setTag("cloudflare.binding.name"_kjc, b); - } - traceContext.setTag("cloudflare.r2.operation"_kjc, "PutObject"_kjc); - KJ_IF_SOME(b, this->bucketName()) { - traceContext.setTag("cloudflare.r2.bucket"_kjc, b); - } - traceContext.setTag("cloudflare.r2.request.key"_kjc, key.asPtr()); + TraceContext traceContext = makeR2TraceContext("r2_put"_kjc, "PutObject"_kjc, key.asPtr()); capnp::JsonCodec json; json.handleByAnnotation(); @@ -907,17 +1550,8 @@ jsg::Promise> R2Bucket::createMultipartUpload(jsg::L const jsg::TypeHandler>& errorType) { return js.evalNow([&] { auto& context = IoContext::current(); - TraceContext traceContext = context.makeUserTraceSpan("r2_createMultipartUpload"_kjc); - - traceContext.setTag("cloudflare.binding.type"_kjc, "r2"_kjc); - KJ_IF_SOME(b, this->bindingName()) { - traceContext.setTag("cloudflare.binding.name"_kjc, b); - } - traceContext.setTag("cloudflare.r2.operation"_kjc, "CreateMultipartUpload"_kjc); - KJ_IF_SOME(b, this->bucketName()) { - traceContext.setTag("cloudflare.r2.bucket"_kjc, b); - } - traceContext.setTag("cloudflare.r2.request.key"_kjc, key.asPtr()); + TraceContext traceContext = makeR2TraceContext( + "r2_createMultipartUpload"_kjc, "CreateMultipartUpload"_kjc, key.asPtr()); capnp::JsonCodec json; json.handleByAnnotation(); @@ -1034,17 +1668,7 @@ jsg::Promise R2Bucket::delete_(jsg::Lock& js, const jsg::TypeHandler>& errorType) { return js.evalNow([&] { auto& context = IoContext::current(); - - TraceContext traceContext = context.makeUserTraceSpan("r2_delete"_kjc); - - traceContext.setTag("cloudflare.binding.type"_kjc, "r2"_kjc); - KJ_IF_SOME(b, this->bindingName()) { - traceContext.setTag("cloudflare.binding.name"_kjc, b); - } - traceContext.setTag("cloudflare.r2.operation"_kjc, "DeleteObject"_kjc); - KJ_IF_SOME(b, this->bucketName()) { - traceContext.setTag("cloudflare.r2.bucket"_kjc, b); - } + TraceContext traceContext = makeR2TraceContext("r2_delete"_kjc, "DeleteObject"_kjc); capnp::JsonCodec json; json.handleByAnnotation(); @@ -1089,22 +1713,96 @@ jsg::Promise R2Bucket::delete_(jsg::Lock& js, }); } +jsg::Promise R2Bucket::listRpc(jsg::Lock& js, + jsg::Optional options, + const jsg::TypeHandler>& rpcPropHandler, + const jsg::TypeHandler)>>& + listFnHandler, + const jsg::TypeHandler>& listResultHandler, + CompatibilityFlags::Reader flags) { + return js.evalNow([&] { + TraceContext traceContext = makeR2TraceContext("r2_list"_kjc, "ListObjects"_kjc); + + const bool honorsIncludes = flags.getR2ListHonorIncludeFields(); + jsg::Optional rpcOptions; + if (options != kj::none || !honorsIncludes) { + SerializableListOptions normalized; + kj::Vector includes; + + KJ_IF_SOME(o, options) { + KJ_IF_SOME(limit, o.limit) { + normalized.limit = limit; + traceContext.setTag("cloudflare.r2.request.limit"_kjc, static_cast(limit)); + } + KJ_IF_SOME(prefix, o.prefix) { + traceContext.setTag("cloudflare.r2.request.prefix"_kjc, prefix.value.asPtr()); + normalized.prefix = kj::mv(prefix.value); + } + KJ_IF_SOME(cursor, o.cursor) { + traceContext.setTag("cloudflare.r2.request.cursor"_kjc, cursor.value.asPtr()); + normalized.cursor = kj::mv(cursor.value); + } + KJ_IF_SOME(delimiter, o.delimiter) { + traceContext.setTag("cloudflare.r2.request.delimiter"_kjc, delimiter.value.asPtr()); + normalized.delimiter = kj::mv(delimiter.value); + } + KJ_IF_SOME(startAfter, o.startAfter) { + traceContext.setTag("cloudflare.r2.request.start_after"_kjc, startAfter.value.asPtr()); + normalized.startAfter = kj::mv(startAfter.value); + } + KJ_IF_SOME(requestedIncludes, o.include) { + for (auto& requested: requestedIncludes) { + if (requested.value == "httpMetadata") { + traceContext.setTag("cloudflare.r2.request.include.http_metadata"_kjc, true); + } else if (requested.value == "customMetadata") { + traceContext.setTag("cloudflare.r2.request.include.custom_metadata"_kjc, true); + } else { + JSG_FAIL_REQUIRE(RangeError, "Unsupported include value ", requested.value); + } + if (honorsIncludes) { + includes.add(kj::mv(requested.value)); + } + } + } + } + + if (!honorsIncludes) { + includes.add(kj::str("httpMetadata")); + includes.add(kj::str("customMetadata")); + } + normalized.include = includes.releaseAsArray(); + rpcOptions = kj::mv(normalized); + } + + return callR2RpcMethod(js, getRpcMethod(js, "list"_kj), rpcPropHandler, + listFnHandler, listResultHandler, kj::mv(rpcOptions)) + .then(js, + [traceContext = kj::mv(traceContext)]( + jsg::Lock& js, ListBackendResult backend) mutable -> ListResult { + ListResult result; + result.objects = KJ_MAP(object, backend.objects) { + return headResultFromBackend(js, kj::mv(object), MissingMetadataPolicy::ABSENT); + }; + result.truncated = backend.truncated; + KJ_IF_SOME(cursor, backend.cursor) { + result.cursor = kj::mv(cursor); + } + result.delimitedPrefixes = + kj::mv(backend.delimitedPrefixes).orDefault(kj::heapArray(0)); + + addListResultSpanTags(traceContext, result); + return kj::mv(result); + }); + }); +} + jsg::Promise R2Bucket::list(jsg::Lock& js, jsg::Optional options, const jsg::TypeHandler>& errorType, CompatibilityFlags::Reader flags) { return js.evalNow([&] { auto& context = IoContext::current(); - TraceContext traceContext = context.makeUserTraceSpan("r2_list"_kjc); - - traceContext.setTag("cloudflare.binding.type"_kjc, "r2"_kjc); - KJ_IF_SOME(b, this->bindingName()) { - traceContext.setTag("cloudflare.binding.name"_kjc, b); - } - traceContext.setTag("cloudflare.r2.operation"_kjc, "ListObjects"_kjc); - KJ_IF_SOME(b, this->bucketName()) { - traceContext.setTag("cloudflare.r2.bucket"_kjc, b); - } + TraceContext traceContext = makeR2TraceContext("r2_list"_kjc, "ListObjects"_kjc); capnp::JsonCodec json; json.handleByAnnotation(); @@ -1223,19 +1921,13 @@ jsg::Promise R2Bucket::list(jsg::Lock& js, result.truncated = responseBuilder.getTruncated(); if (responseBuilder.hasCursor()) { result.cursor = kj::str(responseBuilder.getCursor()); - traceContext.setTag( - "cloudflare.r2.response.cursor"_kjc, KJ_ASSERT_NONNULL(result.cursor).asPtr()); } if (responseBuilder.hasDelimitedPrefixes()) { result.delimitedPrefixes = KJ_MAP(e, responseBuilder.getDelimitedPrefixes()) { return kj::str(e); }; } - traceContext.setTag("cloudflare.r2.response.returned_objects"_kjc, - static_cast(result.objects.size())); - traceContext.setTag("cloudflare.r2.response.delimited_prefixes"_kjc, - static_cast(result.delimitedPrefixes.size())); - traceContext.setTag("cloudflare.r2.response.truncated"_kjc, result.truncated); + addListResultSpanTags(traceContext, result); return kj::mv(result); }); }); @@ -1482,6 +2174,116 @@ jsg::Promise> R2Bucket::GetResult::blob(jsg::Lock& js) { }); } +R2Bucket::SerializableChecksums R2Bucket::Checksums::toSerializable() const { + return { + .md5 = copyBytes(md5), + .sha1 = copyBytes(sha1), + .sha256 = copyBytes(sha256), + .sha384 = copyBytes(sha384), + .sha512 = copyBytes(sha512), + }; +} + +void R2Bucket::Checksums::serialize(jsg::Lock& js, + jsg::Serializer& serializer, + const jsg::TypeHandler& checksumsHandler) { + requireR2RpcSerializer(serializer); + serializer.write(js, jsg::JsValue(checksumsHandler.wrap(js, toSerializable()))); +} + +jsg::Ref R2Bucket::Checksums::deserialize(jsg::Lock& js, + rpc::SerializationTag tag, + jsg::Deserializer& deserializer, + const jsg::TypeHandler& checksumsHandler) { + requireR2RpcDeserializer(deserializer); + auto checksums = KJ_UNWRAP_OR(checksumsHandler.tryUnwrap(js, deserializer.readValue(js)), { + JSG_FAIL_REQUIRE(DOMDataCloneError, "Deserialization failed: invalid R2 checksums payload"); + }); + return js.alloc(kj::mv(checksums.md5), kj::mv(checksums.sha1), + kj::mv(checksums.sha256), kj::mv(checksums.sha384), kj::mv(checksums.sha512)); +} + +R2Bucket::SerializableHeadResult R2Bucket::HeadResult::toSerializable() const { + kj::Maybe rpcHttpMetadata; + KJ_IF_SOME(metadata, httpMetadata) { + rpcHttpMetadata = metadata.clone(); + } + + kj::Maybe> rpcCustomMetadata; + KJ_IF_SOME(metadata, customMetadata) { + rpcCustomMetadata = copyStringDict(metadata); + } + + kj::Maybe rpcRange; + KJ_IF_SOME(value, range) { + rpcRange = Range{ + .offset = value.offset, + .length = value.length, + .suffix = value.suffix, + }; + } + + return { + .key = kj::str(name), + .version = kj::str(version), + .size = size, + .etag = kj::str(etag), + .uploaded = uploaded, + .storageClass = kj::str(storageClass), + .checksums = checksums->toSerializable(), + .httpMetadata = kj::mv(rpcHttpMetadata), + .customMetadata = kj::mv(rpcCustomMetadata), + .range = kj::mv(rpcRange), + .ssecKeyMd5 = mapCopyString(ssecKeyMd5), + }; +} + +void R2Bucket::HeadResult::serialize(jsg::Lock& js, + jsg::Serializer& serializer, + const jsg::TypeHandler& headResultHandler) { + requireR2RpcSerializer(serializer); + serializer.write(js, jsg::JsValue(headResultHandler.wrap(js, toSerializable()))); +} + +jsg::Ref R2Bucket::HeadResult::deserialize(jsg::Lock& js, + rpc::SerializationTag tag, + jsg::Deserializer& deserializer, + const jsg::TypeHandler& headResultHandler) { + requireR2RpcDeserializer(deserializer); + auto headResult = KJ_UNWRAP_OR(headResultHandler.tryUnwrap(js, deserializer.readValue(js)), { + JSG_FAIL_REQUIRE(DOMDataCloneError, "Deserialization failed: invalid R2 object payload"); + }); + return headResultFromSerializable(js, kj::mv(headResult), MissingMetadataPolicy::ABSENT); +} + +void R2Bucket::GetResult::serialize(jsg::Lock& js, + jsg::Serializer& serializer, + const jsg::TypeHandler& getResultHandler) { + requireR2RpcSerializer(serializer); + serializer.write(js, + jsg::JsValue(getResultHandler.wrap(js, + SerializableGetResult{ + .kind = kj::str("body"), + .object = toSerializable(), + .body = body.addRef(js), + }))); +} + +jsg::Ref R2Bucket::GetResult::deserialize(jsg::Lock& js, + rpc::SerializationTag tag, + jsg::Deserializer& deserializer, + const jsg::TypeHandler& getResultHandler) { + requireR2RpcDeserializer(deserializer); + auto getResult = KJ_UNWRAP_OR(getResultHandler.tryUnwrap(js, deserializer.readValue(js)), { + JSG_FAIL_REQUIRE(DOMDataCloneError, "Deserialization failed: invalid R2 object body payload"); + }); + JSG_REQUIRE(getResult.kind == "body", DOMDataCloneError, + "Deserialization failed: R2 object body payload has an invalid kind"); + JSG_REQUIRE(getResult.body != kj::none, DOMDataCloneError, + "Deserialization failed: R2 object body payload is missing its body"); + return getResultFromSerializable(js, kj::mv(getResult)); +} + R2Bucket::StringChecksums R2Bucket::Checksums::toJSON() { return { .md5 = this->md5.map(kj::encodeHex), diff --git a/src/workerd/api/r2-bucket.h b/src/workerd/api/r2-bucket.h index a57770965f9..e2f728f6a52 100644 --- a/src/workerd/api/r2-bucket.h +++ b/src/workerd/api/r2-bucket.h @@ -7,7 +7,9 @@ #include "r2-rpc.h" #include +#include #include +#include namespace workerd::api { class Headers; @@ -36,6 +38,9 @@ kj::Own r2GetClient(IoContext& context, uint subrequestChannel, kj::ArrayPtr fillR2Path( kj::StringPtr pathStorage[1], const kj::Maybe& bucket); +kj::Maybe buildSsecKey( + kj::Maybe, kj::String>> maybeRawSsecKey); + class R2MultipartUpload; // A capability to an R2 Bucket. @@ -126,6 +131,8 @@ class R2Bucket: public jsg::Object { JSG_STRUCT_TS_OVERRIDE(R2StringChecksums); }; + struct SerializableChecksums; + class Checksums: public jsg::Object { public: Checksums(jsg::Optional> md5, @@ -146,6 +153,7 @@ class R2Bucket: public jsg::Object { jsg::Optional getSha512(jsg::Lock& js); StringChecksums toJSON(); + SerializableChecksums toSerializable() const; JSG_RESOURCE_TYPE(Checksums) { JSG_LAZY_READONLY_INSTANCE_PROPERTY(md5, getMd5); @@ -163,6 +171,16 @@ class R2Bucket: public jsg::Object { }); } + void serialize(jsg::Lock& js, + jsg::Serializer& serializer, + const jsg::TypeHandler& checksumsHandler); + static jsg::Ref deserialize(jsg::Lock& js, + rpc::SerializationTag tag, + jsg::Deserializer& deserializer, + const jsg::TypeHandler& checksumsHandler); + + JSG_SERIALIZABLE(rpc::SerializationTag::R2_CHECKSUMS); + jsg::Optional> md5; jsg::Optional> sha1; jsg::Optional> sha256; @@ -232,6 +250,58 @@ class R2Bucket: public jsg::Object { JSG_STRUCT_TS_OVERRIDE(R2PutOptions); }; + // Serializable* types: the intermediate representation exchanged with the R2 gateway worker + // (R2GW) over JSRPC. They mirror the shape of the parameters the R2GW expects and the values + // it returns, and exist only so those values can be serialized across the JSRPC boundary. + // + // These are NOT customer-facing types and must never be returned to user code. The public + // counterparts (Conditional, GetOptions, HeadResult, GetResult, ListResult, ...) validate + // input with NonCoercible strings and expose methods / lazy accessors, none of which can be + // wrapped back to JavaScript or serialized. Public inputs are normalized into a Serializable* + // type before calling the R2GW, and Serializable* results are converted back into the public + // types before being handed to the caller. For the same reason these carry no TS overrides. + struct SerializableConditional { + jsg::Optional etagMatches; + jsg::Optional etagDoesNotMatch; + jsg::Optional uploadedBefore; + jsg::Optional uploadedAfter; + jsg::Optional secondsGranularity; + + JSG_STRUCT(etagMatches, etagDoesNotMatch, uploadedBefore, uploadedAfter, secondsGranularity); + }; + + struct SerializablePutOptions { + jsg::Optional>> onlyIf; + jsg::Optional>> httpMetadata; + jsg::Optional> customMetadata; + jsg::Optional, kj::String>> md5; + jsg::Optional, kj::String>> sha1; + jsg::Optional, kj::String>> sha256; + jsg::Optional, kj::String>> sha384; + jsg::Optional, kj::String>> sha512; + jsg::Optional storageClass; + jsg::Optional, kj::String>> ssecKey; + + JSG_STRUCT(onlyIf, + httpMetadata, + customMetadata, + md5, + sha1, + sha256, + sha384, + sha512, + storageClass, + ssecKey); + }; + + struct SerializableGetOptions { + jsg::Optional>> onlyIf; + jsg::Optional, jsg::Ref>> range; + jsg::Optional ssecKey; + + JSG_STRUCT(onlyIf, range, ssecKey); + }; + struct MultipartOptions { jsg::Optional>> httpMetadata; jsg::Optional> customMetadata; @@ -242,6 +312,87 @@ class R2Bucket: public jsg::Object { JSG_STRUCT_TS_OVERRIDE(R2MultipartOptions); }; + // Object metadata as returned by the R2GW. HeadResult / Checksums are built from these. + struct SerializableChecksums { + jsg::Optional> md5; + jsg::Optional> sha1; + jsg::Optional> sha256; + jsg::Optional> sha384; + jsg::Optional> sha512; + + JSG_STRUCT(md5, sha1, sha256, sha384, sha512); + }; + + // Field names match the gateway's R2ObjectRpc, not HeadResult's members: the + // key arrives as `key` where HeadResult stores it as `name`. + // + // `kj::Maybe` rather than `jsg::Optional` throughout, because jsg::Optional + // accepts `undefined` but not `null`, and only kj::Maybe tolerates both. The + // gateway omits absent fields today, but that is an unenforced cross-repo + // invariant and a null would otherwise be a hard unwrap failure. + struct SerializableHeadResult { + kj::String key; + kj::String version; + double size; + kj::String etag; + kj::Date uploaded; + kj::String storageClass; + SerializableChecksums checksums; + kj::Maybe httpMetadata; + kj::Maybe> customMetadata; + kj::Maybe range; + kj::Maybe ssecKeyMd5; + + JSG_STRUCT(key, + version, + size, + etag, + uploaded, + storageClass, + checksums, + httpMetadata, + customMetadata, + range, + ssecKeyMd5); + }; + + struct SerializableListOptions { + jsg::Optional limit; + jsg::Optional prefix; + jsg::Optional cursor; + jsg::Optional delimiter; + jsg::Optional startAfter; + kj::Array include; + + JSG_STRUCT(limit, prefix, cursor, delimiter, startAfter, include); + }; + + struct SerializableListResult { + kj::Array objects; + bool truncated; + kj::Maybe cursor; + kj::Maybe> delimitedPrefixes; + + JSG_STRUCT(objects, truncated, cursor, delimitedPrefixes); + }; + + struct SerializableGetResult { + kj::String kind; + SerializableHeadResult object; + kj::Maybe body; + + JSG_STRUCT(kind, object, body); + }; + + struct SerializableMultipartUploadHandle { + kj::String key; + kj::String uploadId; + + JSG_STRUCT(key, uploadId); + }; + + using MultipartUploadBackendResult = kj::OneOf; + class HeadResult: public jsg::Object { public: HeadResult(kj::String name, @@ -314,6 +465,8 @@ class R2Bucket: public jsg::Object { return range; } + SerializableHeadResult toSerializable() const; + void writeHttpMetadata(jsg::Lock& js, Headers& headers); JSG_RESOURCE_TYPE(HeadResult) { @@ -333,6 +486,16 @@ class R2Bucket: public jsg::Object { JSG_TS_OVERRIDE(R2Object); } + void serialize(jsg::Lock& js, + jsg::Serializer& serializer, + const jsg::TypeHandler& headResultHandler); + static jsg::Ref deserialize(jsg::Lock& js, + rpc::SerializationTag tag, + jsg::Deserializer& deserializer, + const jsg::TypeHandler& headResultHandler); + + JSG_SERIALIZABLE(rpc::SerializationTag::R2_OBJECT); + void visitForMemoryInfo(jsg::MemoryTracker& tracker) const { tracker.trackField("name", name); tracker.trackField("version", version); @@ -421,6 +584,16 @@ class R2Bucket: public jsg::Object { }); } + void serialize(jsg::Lock& js, + jsg::Serializer& serializer, + const jsg::TypeHandler& getResultHandler); + static jsg::Ref deserialize(jsg::Lock& js, + rpc::SerializationTag tag, + jsg::Deserializer& deserializer, + const jsg::TypeHandler& getResultHandler); + + JSG_SERIALIZABLE(rpc::SerializationTag::R2_OBJECT_BODY); + void visitForMemoryInfo(jsg::MemoryTracker& tracker) const { body.visitForMemoryInfo(tracker); } @@ -433,6 +606,10 @@ class R2Bucket: public jsg::Object { } }; + using HeadBackendResult = kj::OneOf, SerializableHeadResult>; + using GetBackendResult = + kj::OneOf, jsg::Ref, SerializableGetResult>; + struct ListResult { kj::Array> objects; bool truncated; @@ -449,6 +626,15 @@ class R2Bucket: public jsg::Object { )); }; + struct ListBackendResult { + kj::Array objects; + bool truncated; + kj::Maybe cursor; + kj::Maybe> delimitedPrefixes; + + JSG_STRUCT(objects, truncated, cursor, delimitedPrefixes); + }; + struct ListOptions { jsg::Optional limit; jsg::Optional> prefix; @@ -490,20 +676,87 @@ class R2Bucket: public jsg::Object { jsg::Promise delete_(jsg::Lock& js, kj::OneOf> keys, const jsg::TypeHandler>& errorType); + + // JSRPC equivalents of the above, selected by JSG_RESOURCE_TYPE when the r2_binding_jsrpc compatibility flag is + // on. It dispatches to the gateway's R2BindingEntrypoint instead of + // synthesising an HTTP request, then rebuilds the public result types from the + // plain data JSRPC delivers. + // + // These keep ordinary typed signatures rather than taking a raw + // v8::FunctionCallbackInfo the way KvNamespace::deleteBulk does. A raw-args + // passthrough cannot work here: it returns the JsRpcPromise directly, so the + // caller receives the gateway's plain data and R2Object's methods and sync + // accessors are gone. Reconstruction needs the resolved value, which needs a + // real promise, which needs a typed return, which needs an injected + // TypeHandler -- and TypeHandlers are only injected into typed signatures. + jsg::Promise>> headRpc(jsg::Lock& js, + kj::String key, + const jsg::TypeHandler>& rpcPropHandler, + const jsg::TypeHandler>& headFnHandler, + const jsg::TypeHandler>>& headResultHandler); + jsg::Promise>, jsg::Ref>> getRpc( + jsg::Lock& js, + kj::String key, + jsg::Optional options, + const jsg::TypeHandler>& rpcPropHandler, + const jsg::TypeHandler)>>& getFnHandler, + const jsg::TypeHandler>>& getResultHandler); + jsg::Promise deleteRpc(jsg::Lock& js, + kj::OneOf> keys, + const jsg::TypeHandler>& rpcPropHandler, + const jsg::TypeHandler< + jsg::Function>)>>& deleteFnHandler, + const jsg::TypeHandler>& deleteResultHandler); + jsg::Promise>> putRpc(jsg::Lock& js, + kj::String key, + kj::Maybe value, + jsg::Optional options, + const jsg::TypeHandler>& rpcPropHandler, + const jsg::TypeHandler, + jsg::Optional, + double)>>& putFnHandler, + const jsg::TypeHandler>>& putResultHandler); + jsg::Promise> createMultipartUploadRpc(jsg::Lock& js, + kj::String key, + jsg::Optional options, + const jsg::TypeHandler>& rpcPropHandler, + const jsg::TypeHandler< + jsg::Function)>>& createFnHandler, + const jsg::TypeHandler>& uploadResultHandler); + jsg::Promise listRpc(jsg::Lock& js, + jsg::Optional options, + const jsg::TypeHandler>& rpcPropHandler, + const jsg::TypeHandler)>>& + listFnHandler, + const jsg::TypeHandler>& listResultHandler, + CompatibilityFlags::Reader flags); jsg::Promise list(jsg::Lock& js, jsg::Optional options, const jsg::TypeHandler>& errorType, CompatibilityFlags::Reader flags); JSG_RESOURCE_TYPE(R2Bucket, CompatibilityFlags::Reader flags) { - JSG_METHOD(head); - JSG_METHOD(get); - JSG_METHOD(put); - JSG_METHOD(createMultipartUpload); + // The compatibility flag restricts the new transport to allowlisted + // workers, because it is marked $experimental and EWC decides who + // may opt in. + if (flags.getR2BindingsJsrpc()) { + JSG_METHOD_NAMED(head, headRpc); + JSG_METHOD_NAMED(get, getRpc); + JSG_METHOD_NAMED(delete, deleteRpc); + JSG_METHOD_NAMED(put, putRpc); + JSG_METHOD_NAMED(createMultipartUpload, createMultipartUploadRpc); + JSG_METHOD_NAMED(list, listRpc); + } else { + JSG_METHOD(head); + JSG_METHOD(get); + JSG_METHOD_NAMED(delete, delete_); + JSG_METHOD(put); + JSG_METHOD(createMultipartUpload); + JSG_METHOD(list); + } JSG_METHOD(resumeMultipartUpload); - JSG_METHOD_NAMED(delete, delete_); - JSG_METHOD(list); - JSG_TS_ROOT(); JSG_TS_OVERRIDE({ // The order of these matters, since typescript tries to match function signatures in order @@ -600,9 +853,31 @@ class R2Bucket: public jsg::Object { kj::Own getHttpClient(IoContext& context, TraceContext& traceContext); + TraceContext makeR2TraceContext(kj::LiteralStringConst spanName, + kj::LiteralStringConst operation, + kj::Maybe key = kj::none); + + // Look up a method on the gateway's entrypoint over this binding's subrequest + // channel. + jsg::Ref getRpcMethod(jsg::Lock& js, kj::StringPtr methodName); + friend class R2MultipartUpload; }; +// Helper enum to indicate whether returned httpMetadata and customMetadata to be empty or absent. +// Use EMPTY for `head`, successful `get`, conditional `get`, `put`, and multipart completion. Use ABSENT for list fields that were not requested. +enum class MissingMetadataPolicy { + EMPTY, + ABSENT, +}; + +jsg::Ref headResultFromSerializable(jsg::Lock& js, + R2Bucket::SerializableHeadResult rpc, + MissingMetadataPolicy policy = MissingMetadataPolicy::EMPTY); +jsg::Ref headResultFromBackend(jsg::Lock& js, + R2Bucket::HeadBackendResult result, + MissingMetadataPolicy policy = MissingMetadataPolicy::EMPTY); + // Non-generic wrapper avoid moving the parseObjectMetadata implementation into this header file // by making use of dynamic dispatch. kj::Maybe> parseHeadResultWrapper(jsg::Lock& js, diff --git a/src/workerd/api/r2-multipart.c++ b/src/workerd/api/r2-multipart.c++ index ff30b115cd6..e28d04ced88 100644 --- a/src/workerd/api/r2-multipart.c++ +++ b/src/workerd/api/r2-multipart.c++ @@ -41,16 +41,7 @@ jsg::Promise R2MultipartUpload::uploadPart(jsg: "Part number must be between 1 and 10000 (inclusive). Actual value was: ", partNumber); auto& context = IoContext::current(); - TraceContext traceContext = context.makeUserTraceSpan("r2_uploadPart"_kjc); - - traceContext.setTag("cloudflare.binding.type"_kjc, "r2"_kjc); - KJ_IF_SOME(b, this->bucket->bindingName()) { - traceContext.setTag("cloudflare.binding.name"_kjc, b); - } - traceContext.setTag("cloudflare.r2.operation"_kjc, "UploadPart"_kjc); - KJ_IF_SOME(b, this->bucket->bucketName()) { - traceContext.setTag("cloudflare.r2.bucket"_kjc, b); - } + TraceContext traceContext = bucket->makeR2TraceContext("r2_uploadPart"_kjc, "UploadPart"_kjc); traceContext.setTag("cloudflare.r2.request.upload_id"_kjc, uploadId.asPtr()); traceContext.setTag("cloudflare.r2.request.part_number"_kjc, static_cast(partNumber)); traceContext.setTag("cloudflare.r2.request.key"_kjc, key.asPtr()); @@ -138,21 +129,63 @@ jsg::Promise R2MultipartUpload::uploadPart(jsg: }); } +R2MultipartUpload::Metadata R2MultipartUpload::Metadata::clone() const { + return {.httpMetadata = httpMetadata.clone(), + .customMetadata = {.fields = KJ_MAP(field, customMetadata.fields) { + return jsg::Dict::Field{.name = kj::str(field.name), .value = kj::str(field.value)}; + }}}; +} + +jsg::Promise R2MultipartUpload::uploadPartRpc(jsg::Lock& js, + int partNumber, + R2PutValue value, + jsg::Optional options, + const jsg::TypeHandler>& rpcPropHandler, + const jsg::TypeHandler, + double)>>& uploadPartFnHandler, + const jsg::TypeHandler>& uploadPartResultHandler) { + return js.evalNow([&] { + JSG_REQUIRE(partNumber >= 1 && partNumber <= 10000, TypeError, + "Part number must be between 1 and 10000 (inclusive). Actual value was: ", partNumber); + + TraceContext traceContext = bucket->makeR2TraceContext("r2_uploadPart"_kjc, "UploadPart"_kjc); + traceContext.setTag("cloudflare.r2.request.upload_id"_kjc, uploadId.asPtr()); + traceContext.setTag("cloudflare.r2.request.part_number"_kjc, static_cast(partNumber)); + traceContext.setTag("cloudflare.r2.request.key"_kjc, key.asPtr()); + + KJ_IF_SOME(o, options) { + KJ_IF_SOME(ssecKey, buildSsecKey(kj::mv(o.ssecKey))) { + traceContext.setTag("cloudflare.r2.request.ssec_key"_kjc, true); + o.ssecKey = kj::mv(ssecKey); + } + } + + auto prepared = prepareR2RpcBody(js, value); + traceContext.setTag("cloudflare.r2.request.size"_kjc, static_cast(prepared.size)); + + auto promise = callR2RpcMethod(js, bucket->getRpcMethod(js, "uploadPart"_kj), + rpcPropHandler, uploadPartFnHandler, uploadPartResultHandler, kj::str(key), + kj::str(uploadId), partNumber, kj::mv(prepared.value), kj::mv(options), prepared.size); + return promise.then(js, + [partNumber, traceContext = kj::mv(traceContext)]( + jsg::Lock& js, UploadedPart uploadedPart) mutable { + traceContext.setTag("cloudflare.r2.response.etag"_kjc, uploadedPart.etag.asPtr()); + return UploadedPart{.partNumber = partNumber, .etag = kj::mv(uploadedPart.etag)}; + }); + }); +} + jsg::Promise> R2MultipartUpload::complete(jsg::Lock& js, kj::Array uploadedParts, const jsg::TypeHandler>& errorType) { return js.evalNow([&] { auto& context = IoContext::current(); - TraceContext traceContext = context.makeUserTraceSpan("r2_completeMultipartUpload"_kjc); - - traceContext.setTag("cloudflare.binding.type"_kjc, "r2"_kjc); - KJ_IF_SOME(b, this->bucket->bindingName()) { - traceContext.setTag("cloudflare.binding.name"_kjc, b); - } - traceContext.setTag("cloudflare.r2.operation"_kjc, "CompleteMultipartUpload"_kjc); - KJ_IF_SOME(b, this->bucket->bucketName()) { - traceContext.setTag("cloudflare.r2.bucket"_kjc, b); - } + TraceContext traceContext = + bucket->makeR2TraceContext("r2_completeMultipartUpload"_kjc, "CompleteMultipartUpload"_kjc); traceContext.setTag("cloudflare.r2.request.upload_id"_kjc, uploadId.asPtr()); traceContext.setTag("cloudflare.r2.request.key"_kjc, key.asPtr()); kj::String partIds = @@ -207,20 +240,46 @@ jsg::Promise> R2MultipartUpload::complete(jsg::Lo }); } +jsg::Promise> R2MultipartUpload::completeRpc(jsg::Lock& js, + kj::Array uploadedParts, + const jsg::TypeHandler>& rpcPropHandler, + const jsg::TypeHandler)>>& completeFnHandler, + const jsg::TypeHandler>& completeResultHandler) { + return js.evalNow([&] { + TraceContext traceContext = + bucket->makeR2TraceContext("r2_completeMultipartUpload"_kjc, "CompleteMultipartUpload"_kjc); + traceContext.setTag("cloudflare.r2.request.upload_id"_kjc, uploadId.asPtr()); + traceContext.setTag("cloudflare.r2.request.key"_kjc, key.asPtr()); + kj::String partIds = + kj::strArray(KJ_MAP(part, uploadedParts) { return kj::str(part.partNumber); }, ", "); + traceContext.setTag("cloudflare.r2.request.uploaded_parts"_kjc, kj::mv(partIds)); + + for (auto& part: uploadedParts) { + JSG_REQUIRE(part.partNumber >= 1 && part.partNumber <= 10000, TypeError, + "Part number must be between 1 and 10000 (inclusive). Actual value was: ", + part.partNumber); + } + + auto promise = callR2RpcMethod(js, + bucket->getRpcMethod(js, "completeMultipartUpload"_kj), rpcPropHandler, completeFnHandler, + completeResultHandler, kj::str(key), kj::str(uploadId), kj::mv(uploadedParts)); + return promise.then(js, + [traceContext = kj::mv(traceContext)]( + jsg::Lock& js, R2Bucket::HeadBackendResult backend) mutable { + auto result = headResultFromBackend(js, kj::mv(backend)); + addHeadResultSpanTags(js, traceContext, *result.get()); + return result; + }); + }); +} + jsg::Promise R2MultipartUpload::abort( jsg::Lock& js, const jsg::TypeHandler>& errorType) { return js.evalNow([&] { auto& context = IoContext::current(); - TraceContext traceContext = context.makeUserTraceSpan("r2_abortMultipartUpload"_kjc); - - traceContext.setTag("cloudflare.binding.type"_kjc, "r2"_kjc); - KJ_IF_SOME(b, this->bucket->bindingName()) { - traceContext.setTag("cloudflare.binding.name"_kjc, b); - } - traceContext.setTag("cloudflare.r2.operation"_kjc, "AbortMultipartUpload"_kjc); - KJ_IF_SOME(b, this->bucket->bucketName()) { - traceContext.setTag("cloudflare.r2.bucket"_kjc, b); - } + TraceContext traceContext = + bucket->makeR2TraceContext("r2_abortMultipartUpload"_kjc, "AbortMultipartUpload"_kjc); traceContext.setTag("cloudflare.r2.request.upload_id"_kjc, uploadId.asPtr()); traceContext.setTag("cloudflare.r2.request.key"_kjc, key.asPtr()); @@ -255,4 +314,21 @@ jsg::Promise R2MultipartUpload::abort( }); }); } + +jsg::Promise R2MultipartUpload::abortRpc(jsg::Lock& js, + const jsg::TypeHandler>& rpcPropHandler, + const jsg::TypeHandler>& abortFnHandler, + const jsg::TypeHandler>& abortResultHandler) { + return js.evalNow([&] { + auto& context = IoContext::current(); + TraceContext traceContext = + bucket->makeR2TraceContext("r2_abortMultipartUpload"_kjc, "AbortMultipartUpload"_kjc); + traceContext.setTag("cloudflare.r2.request.upload_id"_kjc, uploadId.asPtr()); + traceContext.setTag("cloudflare.r2.request.key"_kjc, key.asPtr()); + + auto promise = callR2RpcMethod(js, bucket->getRpcMethod(js, "abortMultipartUpload"_kj), + rpcPropHandler, abortFnHandler, abortResultHandler, kj::str(key), kj::str(uploadId)); + return context.attachSpans(js, kj::mv(promise), kj::mv(traceContext)); + }); +} } // namespace workerd::api::public_beta diff --git a/src/workerd/api/r2-multipart.h b/src/workerd/api/r2-multipart.h index 693d39f7dd8..3528659b60b 100644 --- a/src/workerd/api/r2-multipart.h +++ b/src/workerd/api/r2-multipart.h @@ -26,10 +26,28 @@ class R2MultipartUpload: public jsg::Object { JSG_STRUCT_TS_OVERRIDE(R2UploadPartOptions); }; - R2MultipartUpload(kj::String key, kj::String uploadId, jsg::Ref bucket) + // The gateway's completion response omits create-time metadata. Created RPC uploads retain + // these values locally; resumed uploads use only the metadata returned by the gateway. + struct Metadata { + R2Bucket::HttpMetadata httpMetadata; + jsg::Dict customMetadata; + + Metadata clone() const; + + JSG_MEMORY_INFO(Metadata) { + tracker.trackField("httpMetadata", httpMetadata); + tracker.trackField("customMetadata", customMetadata); + } + }; + + R2MultipartUpload(kj::String key, + kj::String uploadId, + jsg::Ref bucket, + kj::Maybe metadata = kj::none) : key(kj::mv(key)), uploadId(kj::mv(uploadId)), - bucket(kj::mv(bucket)) {} + bucket(kj::mv(bucket)), + metadata(kj::mv(metadata)) {} kj::StringPtr getKey() const { return key; @@ -47,19 +65,48 @@ class R2MultipartUpload: public jsg::Object { jsg::Promise> complete(jsg::Lock& js, kj::Array uploadedParts, const jsg::TypeHandler>& errorType); + jsg::Promise uploadPartRpc(jsg::Lock& js, + int partNumber, + R2PutValue value, + jsg::Optional options, + const jsg::TypeHandler>& rpcPropHandler, + const jsg::TypeHandler, + double)>>& uploadPartFnHandler, + const jsg::TypeHandler>& uploadPartResultHandler); + jsg::Promise abortRpc(jsg::Lock& js, + const jsg::TypeHandler>& rpcPropHandler, + const jsg::TypeHandler>& abortFnHandler, + const jsg::TypeHandler>& abortResultHandler); + jsg::Promise> completeRpc(jsg::Lock& js, + kj::Array uploadedParts, + const jsg::TypeHandler>& rpcPropHandler, + const jsg::TypeHandler)>>& completeFnHandler, + const jsg::TypeHandler>& completeResultHandler); - JSG_RESOURCE_TYPE(R2MultipartUpload) { + JSG_RESOURCE_TYPE(R2MultipartUpload, CompatibilityFlags::Reader flags) { JSG_LAZY_READONLY_INSTANCE_PROPERTY(key, getKey); JSG_LAZY_READONLY_INSTANCE_PROPERTY(uploadId, getUploadId); - JSG_METHOD(uploadPart); - JSG_METHOD(abort); - JSG_METHOD(complete); + if (flags.getR2BindingsJsrpc()) { + JSG_METHOD_NAMED(uploadPart, uploadPartRpc); + JSG_METHOD_NAMED(abort, abortRpc); + JSG_METHOD_NAMED(complete, completeRpc); + } else { + JSG_METHOD(uploadPart); + JSG_METHOD(abort); + JSG_METHOD(complete); + } } void visitForMemoryInfo(jsg::MemoryTracker& tracker) const { tracker.trackField("key", key); tracker.trackField("uploadId", uploadId); tracker.trackField("bucket", bucket); + tracker.trackField("metadata", metadata); } protected: @@ -68,6 +115,8 @@ class R2MultipartUpload: public jsg::Object { jsg::Ref bucket; private: + kj::Maybe metadata; + void visitForGc(jsg::GcVisitor& visitor) { visitor.visit(bucket); } diff --git a/src/workerd/api/r2-rpc.c++ b/src/workerd/api/r2-rpc.c++ index 37f6038dc2d..4189034b047 100644 --- a/src/workerd/api/r2-rpc.c++ +++ b/src/workerd/api/r2-rpc.c++ @@ -5,6 +5,8 @@ #include "r2-rpc.h" #include +#include +#include #include #include #include @@ -15,6 +17,85 @@ #include namespace workerd::api { + +namespace { + +constexpr size_t R2_RPC_INLINE_BODY_LIMIT = 16u << 20; + +JsReadableStream makeR2RpcMemoryStream( + jsg::Lock& js, kj::ArrayPtr bytes, kj::Maybe> backing = kj::none) { + return JsReadableStream::create( + js, IoContext::current(), newMemorySource(bytes, kj::mv(backing))); +} + +} // namespace + +jsg::Promise normalizeR2RpcPromise(jsg::Lock& js, jsg::Value rpcPromise) { + auto paf = js.newPromiseAndResolver(); + paf.resolver.resolve(js, kj::mv(rpcPromise)); + return kj::mv(paf.promise); +} + +void requireR2RpcSerializer(jsg::Serializer& serializer) { + auto& handler = JSG_REQUIRE_NONNULL(serializer.getExternalHandler(), DOMDataCloneError, + "R2 resources can only be serialized for RPC."); + JSG_REQUIRE(dynamic_cast(&handler) != nullptr, DOMDataCloneError, + "R2 resources can only be serialized for RPC."); +} + +void requireR2RpcDeserializer(jsg::Deserializer& deserializer) { + auto& handler = JSG_REQUIRE_NONNULL(deserializer.getExternalHandler(), DOMDataCloneError, + "R2 resources can only be deserialized for RPC."); + JSG_REQUIRE(dynamic_cast(&handler) != nullptr, DOMDataCloneError, + "R2 resources can only be deserialized for RPC."); +} + +PreparedR2RpcBody prepareR2RpcBody(jsg::Lock& js, R2PutValue& value) { + KJ_SWITCH_ONEOF(value) { + KJ_CASE_ONEOF(stream, JsReadableStream) { + auto size = stream.tryGetLength(js, StreamEncoding::IDENTITY); + + JSG_REQUIRE(size != kj::none, TypeError, + "Provided readable stream must have a known length (request/response body or readable " + "half of FixedLengthStream)"); + auto exactSize = KJ_ASSERT_NONNULL(size); + JSG_REQUIRE(exactSize <= 9007199254740991ull, RangeError, + "Provided readable stream is too large to represent its length exactly"); + return {.value = kj::mv(stream), .size = static_cast(exactSize)}; + } + KJ_CASE_ONEOF(data, kj::Array) { + auto size = data.size(); + if (size > R2_RPC_INLINE_BODY_LIMIT) { + // Type-wrapper arrays may alias V8 memory, which cannot be pumped without the isolate lock. + auto owned = kj::heapArray(data.asPtr()); + auto view = owned.asPtr(); + return {.value = makeR2RpcMemoryStream(js, view, kj::heap(kj::mv(owned))), + .size = static_cast(size)}; + } + return {.value = kj::mv(data), .size = static_cast(size)}; + } + KJ_CASE_ONEOF(text, jsg::NonCoercible) { + auto size = text.value.size(); + if (size > R2_RPC_INLINE_BODY_LIMIT) { + auto bytes = text.value.asBytes(); + return {.value = makeR2RpcMemoryStream(js, bytes, kj::heap(kj::mv(text.value))), + .size = static_cast(size)}; + } + return {.value = kj::mv(text.value), .size = static_cast(size)}; + } + KJ_CASE_ONEOF(blob, jsg::Ref) { + auto size = blob->getSize(); + if (size > R2_RPC_INLINE_BODY_LIMIT) { + // Blob bytes are V8-backed, so newMemorySource() must copy them into native memory. + return { + .value = makeR2RpcMemoryStream(js, blob->getData()), .size = static_cast(size)}; + } + return {.value = kj::mv(blob), .size = static_cast(size)}; + } + } + KJ_UNREACHABLE; +} + static kj::Own toError(uint statusCode, kj::StringPtr responseBody) { capnp::JsonCodec json; json.handleByAnnotation(); @@ -29,6 +110,33 @@ jsg::JsValue R2Error::getStack(jsg::Lock& js) { return jsg::JsObject(KJ_ASSERT_NONNULL(errorForStack).Get(js.v8Isolate)).get(js, "stack"_kj); } +void R2Error::serialize(jsg::Lock& js, + jsg::Serializer& serializer, + const jsg::TypeHandler& payloadHandler) { + requireR2RpcSerializer(serializer); + serializer.write(js, + jsg::JsValue(payloadHandler.wrap(js, + SerializableR2Error{ + .code = v4Code, + .message = kj::str(message), + .action = kj::str(KJ_REQUIRE_NONNULL(action)), + }))); +} + +jsg::Ref R2Error::deserialize(jsg::Lock& js, + rpc::SerializationTag tag, + jsg::Deserializer& deserializer, + const jsg::TypeHandler& payloadHandler) { + requireR2RpcDeserializer(deserializer); + auto payload = KJ_UNWRAP_OR(payloadHandler.tryUnwrap(js, deserializer.readValue(js)), + { JSG_FAIL_REQUIRE(DOMDataCloneError, "Deserialization failed: invalid R2 error payload"); }); + auto result = js.alloc(payload.code, kj::mv(payload.message)); + result->action = kj::mv(payload.action); + result->errorForStack = v8::Global( + js.v8Isolate, v8::Exception::Error(v8::String::Empty(js.v8Isolate)).As()); + return result; +} + kj::Maybe R2Result::v4ErrorCode() { KJ_IF_SOME(e, toThrow) { return e->v4Code; @@ -46,10 +154,9 @@ kj::Maybe R2Result::getR2ErrorMessage() { void R2Result::throwIfError( kj::StringPtr action, const jsg::TypeHandler>& errorType) { KJ_IF_SOME(e, toThrow) { - // TODO(soon): Once jsg::JsPromise exists, switch to using that to tunnel out the exception. As - // it stands today, unfortunately, all we can send back to the user is a message. R2Error isn't - // a registered type in the runtime. When reenabling, make sure to update overrides/r2.d.ts to - // reenable the type + // Keep the existing generic error behavior for HTTP-backed bindings. R2Error is serializable so + // an already-structured error can cross JSRPC, but changing this throw path would be a public + // compatibility change. #if 0 auto isolate = IoContext::current().getCurrentLock().getIsolate(); (*e)->action = kj::str(action); diff --git a/src/workerd/api/r2-rpc.h b/src/workerd/api/r2-rpc.h index 1b6f16b459b..137a87b9f0e 100644 --- a/src/workerd/api/r2-rpc.h +++ b/src/workerd/api/r2-rpc.h @@ -6,6 +6,7 @@ #include #include +#include #include namespace kj { @@ -16,12 +17,73 @@ namespace workerd::api { class ReadableStreamSource; +// JsRpcPromise is a custom thenable. Resolving a fresh promise with it makes V8 adopt it even when +// the unwrap_custom_thenables compatibility flag is disabled. +jsg::Promise normalizeR2RpcPromise(jsg::Lock& js, jsg::Value rpcPromise); + +void requireR2RpcSerializer(jsg::Serializer& serializer); +void requireR2RpcDeserializer(jsg::Deserializer& deserializer); + +template +jsg::Value callR2RpcMethod(jsg::Lock& js, + jsg::Ref rpcProp, + const jsg::TypeHandler>& rpcPropHandler, + const jsg::TypeHandler>& fnHandler, + Args... args) { + auto wrappedProp = rpcPropHandler.wrap(js, kj::mv(rpcProp)); + auto fn = KJ_ASSERT_NONNULL(fnHandler.tryUnwrap(js, wrappedProp)); + return fn(js, kj::mv(args)...); +} + +template +jsg::Promise unwrapR2RpcPromise(jsg::Lock& js, + jsg::Value rpcPromise, + const jsg::TypeHandler>& resultPromiseHandler) { + auto normalizedPromise = normalizeR2RpcPromise(js, kj::mv(rpcPromise)); + if constexpr (kj::isSameType()) { + return KJ_ASSERT_NONNULL( + resultPromiseHandler.tryUnwrap(js, normalizedPromise.consumeHandle(js))); + } else { + // Decode only fulfilled responses so gateway rejections retain their public API errors. + // Failures converting a response into the internal wire types are gateway contract violations. + return normalizedPromise.then(js, [&resultPromiseHandler](jsg::Lock& js, jsg::Value value) { + auto fulfilled = js.resolvedPromise(kj::mv(value)); + auto parsed = + KJ_ASSERT_NONNULL(resultPromiseHandler.tryUnwrap(js, fulfilled.consumeHandle(js))); + return parsed.catch_(js, [](jsg::Lock& js, jsg::Value error) -> Result { + auto exception = js.exceptionToKj(kj::mv(error)); + KJ_FAIL_ASSERT("Malformed R2 RPC result.", exception); + }); + }); + } +} + +template +jsg::Promise callR2RpcMethod(jsg::Lock& js, + jsg::Ref rpcProp, + const jsg::TypeHandler>& rpcPropHandler, + const jsg::TypeHandler>& fnHandler, + const jsg::TypeHandler>& resultPromiseHandler, + Args... args) { + auto rpcPromise = + callR2RpcMethod(js, kj::mv(rpcProp), rpcPropHandler, fnHandler, kj::mv(args)...); + return unwrapR2RpcPromise(js, kj::mv(rpcPromise), resultPromiseHandler); +} + // NOTE: We don't currently actually use this as a structured object (hence the `kj::Own` // that we see pop up). // TODO(soon): Switch to structured objects and use jsg::Ref instead of kj::Own // to maintain ownership. class R2Error: public jsg::Object { public: + struct SerializableR2Error { + uint code; + kj::String message; + kj::String action; + + JSG_STRUCT(code, message, action); + }; + R2Error(uint v4Code, kj::String message): v4Code(v4Code), message(kj::mv(message)) {} constexpr kj::StringPtr getName() const { @@ -52,6 +114,16 @@ class R2Error: public jsg::Object { JSG_TS_ROOT(); } + void serialize(jsg::Lock& js, + jsg::Serializer& serializer, + const jsg::TypeHandler& payloadHandler); + static jsg::Ref deserialize(jsg::Lock& js, + rpc::SerializationTag tag, + jsg::Deserializer& deserializer, + const jsg::TypeHandler& payloadHandler); + + JSG_SERIALIZABLE(rpc::SerializationTag::R2_ERROR); + private: uint v4Code; kj::String message; @@ -66,6 +138,17 @@ class R2Error: public jsg::Object { using R2PutValue = kj::OneOf, jsg::NonCoercible, jsg::Ref>; +using SerializablePutValue = + kj::OneOf, kj::String, jsg::Ref>; + +struct PreparedR2RpcBody { + SerializablePutValue value; + double size; +}; + +// Prepares the transport value and the exact byte length passed alongside it. The length is an +// internal argument to the gateway's named RPC method, not part of the public R2 API. +PreparedR2RpcBody prepareR2RpcBody(jsg::Lock& js, R2PutValue& value); struct R2Result { uint httpStatus; diff --git a/src/workerd/api/r2.h b/src/workerd/api/r2.h index eb104bbef50..c6b4ec1e8af 100644 --- a/src/workerd/api/r2.h +++ b/src/workerd/api/r2.h @@ -9,14 +9,24 @@ namespace workerd::api::public_beta { #define EW_R2_PUBLIC_BETA_ISOLATE_TYPES \ - api::R2Error, api::public_beta::R2Bucket, api::public_beta::R2MultipartUpload, \ - api::public_beta::R2MultipartUpload::UploadedPart, api::public_beta::R2Bucket::HeadResult, \ - api::public_beta::R2Bucket::GetResult, api::public_beta::R2Bucket::Range, \ - api::public_beta::R2Bucket::Conditional, api::public_beta::R2Bucket::GetOptions, \ - api::public_beta::R2Bucket::PutOptions, api::public_beta::R2Bucket::MultipartOptions, \ - api::public_beta::R2Bucket::Checksums, api::public_beta::R2Bucket::StringChecksums, \ - api::public_beta::R2Bucket::HttpMetadata, api::public_beta::R2Bucket::ListOptions, \ - api::public_beta::R2Bucket::ListResult, \ - api::public_beta::R2MultipartUpload::UploadPartOptions + api::R2Error, api::R2Error::SerializableR2Error, api::public_beta::R2Bucket, \ + api::public_beta::R2MultipartUpload, api::public_beta::R2MultipartUpload::UploadedPart, \ + api::public_beta::R2Bucket::HeadResult, api::public_beta::R2Bucket::GetResult, \ + api::public_beta::R2Bucket::Range, api::public_beta::R2Bucket::Conditional, \ + api::public_beta::R2Bucket::GetOptions, api::public_beta::R2Bucket::PutOptions, \ + api::public_beta::R2Bucket::MultipartOptions, api::public_beta::R2Bucket::Checksums, \ + api::public_beta::R2Bucket::StringChecksums, api::public_beta::R2Bucket::HttpMetadata, \ + api::public_beta::R2Bucket::ListOptions, api::public_beta::R2Bucket::ListResult, \ + api::public_beta::R2MultipartUpload::UploadPartOptions, \ + api::public_beta::R2Bucket::SerializableChecksums, \ + api::public_beta::R2Bucket::SerializableHeadResult, \ + api::public_beta::R2Bucket::SerializableConditional, \ + api::public_beta::R2Bucket::SerializablePutOptions, \ + api::public_beta::R2Bucket::SerializableGetOptions, \ + api::public_beta::R2Bucket::SerializableGetResult, \ + api::public_beta::R2Bucket::SerializableListOptions, \ + api::public_beta::R2Bucket::SerializableListResult, \ + api::public_beta::R2Bucket::ListBackendResult, \ + api::public_beta::R2Bucket::SerializableMultipartUploadHandle // The list of r2 types that are added to worker.c++'s JSG_DECLARE_ISOLATE_TYPE } // namespace workerd::api::public_beta diff --git a/src/workerd/api/tests/BUILD.bazel b/src/workerd/api/tests/BUILD.bazel index 27c17237d72..fcbfedfa42f 100644 --- a/src/workerd/api/tests/BUILD.bazel +++ b/src/workerd/api/tests/BUILD.bazel @@ -545,6 +545,66 @@ wd_test( ], ) +wd_test( + src = "r2-body-length-test.wd-test", + args = ["--experimental"], + data = ["r2-test.js"], +) + +wd_test( + src = "r2-body-length-ts-test.wd-test", + args = ["--experimental"], + data = [ + "r2-body-length-test.wd-test", + "r2-test.js", + ], + lint = False, +) + +wd_test( + src = "r2-jsrpc-multipart-test.wd-test", + args = ["--experimental"], + data = ["r2-test.js"], +) + +wd_test( + src = "r2-jsrpc-range-test.wd-test", + args = ["--experimental"], + data = ["r2-test.js"], +) + +wd_test( + src = "r2-jsrpc-response-test.wd-test", + args = ["--experimental"], + data = ["r2-test.js"], +) + +wd_test( + src = "r2-jsrpc-serialization-test.wd-test", + args = ["--experimental"], + data = [ + "r2-jsrpc-serialization-test.js", + "r2-test.js", + ], +) + +wd_test( + src = "r2-jsrpc-test.wd-test", + args = ["--experimental"], + data = [ + "instrumentation-tail-worker.js", + "r2-instrumentation-test.js", + "r2-test.js", + ], +) + +wd_test( + src = "r2-jsrpc-legacy-list-test.wd-test", + args = ["--experimental"], + data = ["r2-test.js"], + generate_all_compat_flags_variant = False, +) + wd_test( src = "r2-test.wd-test", args = ["--experimental"], diff --git a/src/workerd/api/tests/r2-body-length-test.wd-test b/src/workerd/api/tests/r2-body-length-test.wd-test new file mode 100644 index 00000000000..669ff35145e --- /dev/null +++ b/src/workerd/api/tests/r2-body-length-test.wd-test @@ -0,0 +1,41 @@ +using Workerd = import "/workerd/workerd.capnp"; + +# Run the same body-length checks against one mock gateway using two callers: HTTP and JSRPC. +# The JSRPC caller also has a plain service binding to separate RPC behavior from R2 parsing. +# Only the body-length tests are exported here, keeping this matrix separate from other tests. +const unitTests :Workerd.Config = ( + services = [ + (name = "http", worker = ( + modules = .testModules, + bindings = [ + (name = "BUCKET", r2Bucket = (name = "gateway", entrypoint = "R2BodyLengthEntrypoint")), + (name = "R2_BODY_LENGTH_TRANSPORT", text = "http"), + ], + compatibilityFlags = ["nodejs_compat", "streams_enable_constructors"], + )), + (name = "jsrpc", worker = ( + modules = .testModules, + bindings = [ + (name = "BUCKET", r2Bucket = (name = "gateway", entrypoint = "R2BodyLengthEntrypoint")), + (name = "SERVICE", service = (name = "gateway", entrypoint = "R2BodyLengthEntrypoint")), + (name = "R2_BODY_LENGTH_TRANSPORT", text = "jsrpc"), + ], + compatibilityFlags = ["experimental", "nodejs_compat", "streams_enable_constructors", + "r2_binding_jsrpc", "rpc", "fetcher_no_get_put_delete"], + )), + (name = "gateway", worker = .gatewayWorker), + ], +); + +const gatewayWorker :Workerd.Worker = ( + modules = [ + (name = "worker", esModule = "export { R2BodyLengthEntrypoint } from './r2-test.js';"), + (name = "r2-test.js", esModule = embed "r2-test.js"), + ], + compatibilityFlags = ["nodejs_compat", "streams_enable_constructors"], +); + +const testModules :List(Workerd.Worker.Module) = [ + (name = "worker", esModule = "export { r2BodyLengthTests } from './r2-test.js';"), + (name = "r2-test.js", esModule = embed "r2-test.js"), +]; diff --git a/src/workerd/api/tests/r2-body-length-ts-test.wd-test b/src/workerd/api/tests/r2-body-length-ts-test.wd-test new file mode 100644 index 00000000000..5387564b58b --- /dev/null +++ b/src/workerd/api/tests/r2-body-length-ts-test.wd-test @@ -0,0 +1,25 @@ +using Workerd = import "/workerd/workerd.capnp"; +using R2BodyLength = import "r2-body-length-test.wd-test"; + +# The TS backend needs bootstrap/hydration autogates. A separate config lets the normal +# R2 tests also exercise the legacy receiver with those autogates disabled. +const unitTests :Workerd.Config = ( + services = [ + (name = "jsrpc-ts", worker = ( + modules = R2BodyLength.testModules, + bindings = [ + (name = "BUCKET", r2Bucket = (name = "gateway", entrypoint = "R2BodyLengthEntrypoint")), + (name = "SERVICE", service = (name = "gateway", entrypoint = "R2BodyLengthEntrypoint")), + (name = "R2_BODY_LENGTH_TRANSPORT", text = "jsrpc-ts"), + ], + compatibilityFlags = ["experimental", "nodejs_compat", "streams_enable_constructors", + "r2_binding_jsrpc", "rpc", "fetcher_no_get_put_delete", + "typescript_implemented_streams"], + )), + (name = "gateway", worker = R2BodyLength.gatewayWorker), + ], + autogates = [ + "workerd-autogate-per-isolate-javascript-bootstrap", + "workerd-autogate-rpc-externals-hydration", + ], +); diff --git a/src/workerd/api/tests/r2-instrumentation-test.js b/src/workerd/api/tests/r2-instrumentation-test.js index 406303129fa..43936219dab 100644 --- a/src/workerd/api/tests/r2-instrumentation-test.js +++ b/src/workerd/api/tests/r2-instrumentation-test.js @@ -12,13 +12,39 @@ import { export default testTailHandler; export const test = { - async test() { + async test(_ctrl, env) { // Wait for all the tailStream executions to finish await Promise.allSettled(invocationPromises); // Recorded streaming tail worker events, in insertion order. let received = Array.from(spans.values()); + // Calls using the must-not-call key produce spans even though validation rejects them before + // reaching the backend. These spans are validated here because they are not included in the + // `expected` array below. + const validationSpans = received.filter( + (span) => span['cloudflare.r2.request.key'] === 'must-not-call' + ); + assert.deepStrictEqual( + validationSpans.map((span) => span.name), + [ + ...Array(14).fill('r2_get'), + ...Array(4).fill('r2_put'), + 'r2_createMultipartUpload', + 'r2_uploadPart', + 'r2_completeMultipartUpload', + ] + ); + for (const span of validationSpans) { + assert.strictEqual(span['cloudflare.binding.type'], 'r2'); + assert.strictEqual(span['cloudflare.binding.name'], 'BUCKET'); + assert.strictEqual(span['cloudflare.r2.bucket'], 'r2-test'); + assert.strictEqual(span.closed, true); + } + received = received.filter( + (span) => span['cloudflare.r2.request.key'] !== 'must-not-call' + ); + // spans emitted by r2-test.js in execution order let expected = [ { @@ -31,7 +57,7 @@ export const test = { 'cloudflare.r2.request.size': 7n, 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.custom_metadata': true, @@ -62,7 +88,7 @@ export const test = { 'cloudflare.r2.request.key': 'basicKey', 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.custom_metadata': true, @@ -77,7 +103,7 @@ export const test = { 'cloudflare.r2.request.key': 'basicKey', 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.custom_metadata': true, @@ -92,7 +118,7 @@ export const test = { 'cloudflare.r2.request.key': 'basicKey', 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.custom_metadata': true, @@ -145,7 +171,7 @@ export const test = { 'cloudflare.r2.request.uploaded_parts': '1', 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.custom_metadata': true, @@ -201,7 +227,7 @@ export const test = { 'cloudflare.r2.request.range.length': 3n, 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.custom_metadata': true, @@ -217,7 +243,7 @@ export const test = { 'cloudflare.r2.request.range.suffix': 2n, 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.custom_metadata': true, @@ -255,7 +281,7 @@ export const test = { 'cloudflare.r2.request.size': 7n, 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.custom_metadata': true, @@ -275,7 +301,7 @@ export const test = { 'cloudflare.r2.request.size': 7n, 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.custom_metadata': true, @@ -294,7 +320,44 @@ export const test = { '2000-01-01T00:00:00.000Z', 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, + 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', + 'cloudflare.r2.response.storage_class': 'Standard', + 'cloudflare.r2.response.custom_metadata': true, + closed: true, + }, + { + name: 'r2_put', + 'cloudflare.binding.type': 'r2', + 'cloudflare.binding.name': 'BUCKET', + 'cloudflare.r2.operation': 'PutObject', + 'cloudflare.r2.bucket': 'r2-test', + 'cloudflare.r2.request.key': 'onlyIfMultipleEtags', + 'cloudflare.r2.request.only_if.etag_matches': 'strongEtag, weakEtag', + 'cloudflare.r2.request.only_if.etag_does_not_match': + 'firstEtag, secondEtag', + 'cloudflare.r2.request.size': 7n, + 'cloudflare.r2.response.success': true, + 'cloudflare.r2.response.etag': 'objectEtag', + 'cloudflare.r2.response.size': 7, + 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', + 'cloudflare.r2.response.storage_class': 'Standard', + 'cloudflare.r2.response.custom_metadata': true, + closed: true, + }, + { + name: 'r2_get', + 'cloudflare.binding.type': 'r2', + 'cloudflare.binding.name': 'BUCKET', + 'cloudflare.r2.operation': 'GetObject', + 'cloudflare.r2.bucket': 'r2-test', + 'cloudflare.r2.request.key': 'onlyIfMultipleEtags', + 'cloudflare.r2.request.only_if.etag_matches': 'strongEtag, weakEtag', + 'cloudflare.r2.request.only_if.etag_does_not_match': + 'firstEtag, secondEtag', + 'cloudflare.r2.response.success': true, + 'cloudflare.r2.response.etag': 'objectEtag', + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.custom_metadata': true, @@ -313,7 +376,7 @@ export const test = { '2000-01-01T00:00:00.000Z', 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.custom_metadata': true, @@ -336,7 +399,7 @@ export const test = { 'cloudflare.r2.request.size': 7n, 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.content_type': 'text/plain', @@ -358,7 +421,7 @@ export const test = { 'cloudflare.r2.request.key': 'httpMetadata', 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.content_type': 'text/plain', @@ -380,7 +443,7 @@ export const test = { 'cloudflare.r2.request.key': 'httpMetadata', 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.content_type': 'text/plain', @@ -436,16 +499,9 @@ export const test = { 'cloudflare.r2.request.uploaded_parts': '', 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', - 'cloudflare.r2.response.content_type': 'text/plain', - 'cloudflare.r2.response.content_encoding': 'utf-8', - 'cloudflare.r2.response.content_disposition': - 'attachment; filename = "basicKey.txt"', - 'cloudflare.r2.response.content_language': 'en-US', - 'cloudflare.r2.response.cache_control': 'no-store', - 'cloudflare.r2.response.cache_expiry': '1970-01-01T00:00:01.000Z', 'cloudflare.r2.response.custom_metadata': true, closed: true, }, @@ -466,7 +522,7 @@ export const test = { 'cloudflare.r2.request.size': 7n, 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.content_type': 'text/plain', @@ -488,7 +544,7 @@ export const test = { 'cloudflare.r2.request.key': 'httpMetadata', 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.content_type': 'text/plain', @@ -510,7 +566,7 @@ export const test = { 'cloudflare.r2.request.key': 'httpMetadata', 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.content_type': 'text/plain', @@ -566,16 +622,9 @@ export const test = { 'cloudflare.r2.request.uploaded_parts': '', 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', - 'cloudflare.r2.response.content_type': 'text/plain', - 'cloudflare.r2.response.content_encoding': 'utf-8', - 'cloudflare.r2.response.content_disposition': - 'attachment; filename = "basicKey.txt"', - 'cloudflare.r2.response.content_language': 'en-US', - 'cloudflare.r2.response.cache_control': 'no-store', - 'cloudflare.r2.response.cache_expiry': '1970-01-01T00:00:01.000Z', 'cloudflare.r2.response.custom_metadata': true, closed: true, }, @@ -590,7 +639,7 @@ export const test = { 'cloudflare.r2.request.size': 7n, 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.custom_metadata': true, @@ -605,7 +654,7 @@ export const test = { 'cloudflare.r2.request.key': 'customMetadata', 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.custom_metadata': true, @@ -620,7 +669,7 @@ export const test = { 'cloudflare.r2.request.key': 'customMetadata', 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.custom_metadata': true, @@ -663,7 +712,7 @@ export const test = { 'cloudflare.r2.request.uploaded_parts': '', 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.custom_metadata': true, @@ -679,7 +728,7 @@ export const test = { 'cloudflare.r2.request.size': 7n, 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.custom_metadata': true, @@ -694,7 +743,7 @@ export const test = { 'cloudflare.r2.request.key': 'classDefault', 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.custom_metadata': true, @@ -709,7 +758,7 @@ export const test = { 'cloudflare.r2.request.key': 'classDefault', 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.custom_metadata': true, @@ -737,7 +786,7 @@ export const test = { 'cloudflare.r2.request.uploaded_parts': '', 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.custom_metadata': true, @@ -754,7 +803,7 @@ export const test = { 'cloudflare.r2.request.size': 7n, 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.custom_metadata': true, @@ -769,7 +818,7 @@ export const test = { 'cloudflare.r2.request.key': 'classStandard', 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.custom_metadata': true, @@ -784,7 +833,7 @@ export const test = { 'cloudflare.r2.request.key': 'classStandard', 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.custom_metadata': true, @@ -813,7 +862,7 @@ export const test = { 'cloudflare.r2.request.uploaded_parts': '', 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.custom_metadata': true, @@ -830,7 +879,7 @@ export const test = { 'cloudflare.r2.request.size': 7n, 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'InfrequentAccess', 'cloudflare.r2.response.custom_metadata': true, @@ -845,7 +894,7 @@ export const test = { 'cloudflare.r2.request.key': 'classInfrequentAccess', 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'InfrequentAccess', 'cloudflare.r2.response.custom_metadata': true, @@ -860,7 +909,7 @@ export const test = { 'cloudflare.r2.request.key': 'classInfrequentAccess', 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'InfrequentAccess', 'cloudflare.r2.response.custom_metadata': true, @@ -889,7 +938,7 @@ export const test = { 'cloudflare.r2.request.uploaded_parts': '', 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'InfrequentAccess', 'cloudflare.r2.response.custom_metadata': true, @@ -906,7 +955,7 @@ export const test = { 'cloudflare.r2.request.size': 7n, 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.ssec_key': true, @@ -923,7 +972,7 @@ export const test = { 'cloudflare.r2.request.ssec_key': true, 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.ssec_key': true, @@ -939,7 +988,7 @@ export const test = { 'cloudflare.r2.request.key': 'ssec', 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.ssec_key': true, @@ -984,7 +1033,7 @@ export const test = { 'cloudflare.r2.request.uploaded_parts': '1', 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.ssec_key': true, @@ -1002,7 +1051,7 @@ export const test = { 'cloudflare.r2.request.size': 7n, 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.ssec_key': true, @@ -1019,7 +1068,7 @@ export const test = { 'cloudflare.r2.request.ssec_key': true, 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.ssec_key': true, @@ -1035,7 +1084,7 @@ export const test = { 'cloudflare.r2.request.key': 'ssec', 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.ssec_key': true, @@ -1080,7 +1129,7 @@ export const test = { 'cloudflare.r2.request.uploaded_parts': '1', 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.ssec_key': true, @@ -1100,7 +1149,7 @@ export const test = { 'cloudflare.r2.request.size': 7n, 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.checksum.md5': '9a0364b9e99bb480dd25e1f0284c8555', @@ -1121,7 +1170,7 @@ export const test = { 'cloudflare.r2.request.key': 'multipleChecksums', 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.checksum.md5': '9a0364b9e99bb480dd25e1f0284c8555', @@ -1133,8 +1182,34 @@ export const test = { 'cloudflare.r2.response.custom_metadata': true, closed: true, }, + { + name: 'r2_delete', + 'cloudflare.binding.type': 'r2', + 'cloudflare.binding.name': 'BUCKET', + 'cloudflare.r2.operation': 'DeleteObject', + 'cloudflare.r2.bucket': 'r2-test', + 'cloudflare.r2.request.keys': 'existing, missing', + 'cloudflare.r2.response.success': true, + closed: true, + }, ]; - assert.deepStrictEqual(received, expected); + const projectSharedTags = (span) => + Object.fromEntries( + Object.entries(span).filter( + ([name]) => name !== 'cloudflare.r2.response.success' + ) + ); + + const sharedReceived = received + .filter((span) => span['cloudflare.binding.type'] === 'r2') + // The JSRPC target runs transport-specific tests after the canonical suite. + .slice(0, expected.length) + .map(projectSharedTags); + assert.deepStrictEqual(sharedReceived, expected.map(projectSharedTags)); + + if (env.R2_TRACE_TRANSPORT === 'http') { + assert.deepStrictEqual(received, expected); + } }, }; diff --git a/src/workerd/api/tests/r2-jsrpc-legacy-list-test.wd-test b/src/workerd/api/tests/r2-jsrpc-legacy-list-test.wd-test new file mode 100644 index 00000000000..280225de3a4 --- /dev/null +++ b/src/workerd/api/tests/r2-jsrpc-legacy-list-test.wd-test @@ -0,0 +1,24 @@ +using Workerd = import "/workerd/workerd.capnp"; + +# Compatibility-sensitive coverage for list() before r2_list_honor_include. This lives separately +# because a future compatibility date necessarily enables the behavior under test. +const unitTests :Workerd.Config = ( + services = [ + ( name = "r2-jsrpc-legacy-list-test", + worker = ( + modules = [ + ( name = "worker", esModule = embed "r2-test.js" ) + ], + bindings = [ + ( name = "BUCKET", + r2Bucket = (name = "r2-jsrpc-legacy-list-test", entrypoint = "R2BindingEntrypoint") ), + ( name = "R2_TRANSPORT", text = "jsrpc" ), + ( name = "R2_LIST_HONOR_INCLUDE", text = "false" ), + ], + compatibilityFlags = ["experimental", "nodejs_compat", "service_binding_extra_handlers", + "streams_enable_constructors", "r2_binding_jsrpc", + "disable_fast_jsg_struct"], + ) + ), + ], +); diff --git a/src/workerd/api/tests/r2-jsrpc-multipart-test.wd-test b/src/workerd/api/tests/r2-jsrpc-multipart-test.wd-test new file mode 100644 index 00000000000..6849030515d --- /dev/null +++ b/src/workerd/api/tests/r2-jsrpc-multipart-test.wd-test @@ -0,0 +1,35 @@ +using Workerd = import "/workerd/workerd.capnp"; + +# Exercise the bucket multipart RPC contract with both struct-wrapping modes. +const unitTests :Workerd.Config = ( + services = [ + ( name = "fast-structs", + worker = ( + modules = [ + (name = "worker", esModule = "export { R2BindingEntrypoint, jsrpcMultipartTests } from './r2-test.js';"), + (name = "r2-test.js", esModule = embed "r2-test.js"), + ], + bindings = [ + (name = "BUCKET", r2Bucket = (name = "fast-structs", entrypoint = "R2BindingEntrypoint")), + (name = "R2_TRANSPORT", text = "jsrpc"), + ], + compatibilityFlags = ["experimental", "nodejs_compat", "service_binding_extra_handlers", + "streams_enable_constructors", "r2_binding_jsrpc", "enable_fast_jsg_struct"], + ), + ), + ( name = "slow-structs", + worker = ( + modules = [ + (name = "worker", esModule = "export { R2BindingEntrypoint, jsrpcMultipartTests } from './r2-test.js';"), + (name = "r2-test.js", esModule = embed "r2-test.js"), + ], + bindings = [ + (name = "BUCKET", r2Bucket = (name = "slow-structs", entrypoint = "R2BindingEntrypoint")), + (name = "R2_TRANSPORT", text = "jsrpc"), + ], + compatibilityFlags = ["experimental", "nodejs_compat", "service_binding_extra_handlers", + "streams_enable_constructors", "r2_binding_jsrpc", "disable_fast_jsg_struct"], + ), + ), + ], +); diff --git a/src/workerd/api/tests/r2-jsrpc-range-test.wd-test b/src/workerd/api/tests/r2-jsrpc-range-test.wd-test new file mode 100644 index 00000000000..9ee470ef40e --- /dev/null +++ b/src/workerd/api/tests/r2-jsrpc-range-test.wd-test @@ -0,0 +1,37 @@ +using Workerd = import "/workerd/workerd.capnp"; + +# The main R2 JSRPC suite disables fast structs. Exercise both modes here because fast structs +# materialize absent optional fields as undefined, while the gateway identifies suffix ranges +# by property presence. Offset/length ranges must reach the gateway without a suffix property. +const unitTests :Workerd.Config = ( + services = [ + ( name = "fast-structs", + worker = ( + modules = [ + (name = "worker", esModule = "export { R2BindingEntrypoint, jsrpcRangeTests } from './r2-test.js';"), + (name = "r2-test.js", esModule = embed "r2-test.js"), + ], + bindings = [ + (name = "BUCKET", r2Bucket = (name = "fast-structs", entrypoint = "R2BindingEntrypoint")), + (name = "R2_TRANSPORT", text = "jsrpc"), + ], + compatibilityFlags = ["experimental", "nodejs_compat", "service_binding_extra_handlers", + "r2_binding_jsrpc", "enable_fast_jsg_struct"], + ), + ), + ( name = "slow-structs", + worker = ( + modules = [ + (name = "worker", esModule = "export { R2BindingEntrypoint, jsrpcRangeTests } from './r2-test.js';"), + (name = "r2-test.js", esModule = embed "r2-test.js"), + ], + bindings = [ + (name = "BUCKET", r2Bucket = (name = "slow-structs", entrypoint = "R2BindingEntrypoint")), + (name = "R2_TRANSPORT", text = "jsrpc"), + ], + compatibilityFlags = ["experimental", "nodejs_compat", "service_binding_extra_handlers", + "r2_binding_jsrpc", "disable_fast_jsg_struct"], + ), + ), + ], +); diff --git a/src/workerd/api/tests/r2-jsrpc-response-test.wd-test b/src/workerd/api/tests/r2-jsrpc-response-test.wd-test new file mode 100644 index 00000000000..5c866746613 --- /dev/null +++ b/src/workerd/api/tests/r2-jsrpc-response-test.wd-test @@ -0,0 +1,33 @@ +using Workerd = import "/workerd/workerd.capnp"; + +# Response decoding must report internal errors with either struct-wrapping implementation. +const unitTests :Workerd.Config = ( + services = [ + ( name = "fast-structs", + worker = ( + modules = [ + (name = "worker", esModule = "import { R2JsrpcResponseEntrypoint, testInvalidR2Responses } from './r2-test.js'; export { R2JsrpcResponseEntrypoint }; export const InvalidR2ResponseTests = { test: testInvalidR2Responses };"), + (name = "r2-test.js", esModule = embed "r2-test.js"), + ], + bindings = [ + (name = "BUCKET", r2Bucket = (name = "fast-structs", entrypoint = "R2JsrpcResponseEntrypoint")), + ], + compatibilityFlags = ["experimental", "nodejs_compat", "service_binding_extra_handlers", + "streams_enable_constructors", "r2_binding_jsrpc", "enable_fast_jsg_struct"], + ), + ), + ( name = "slow-structs", + worker = ( + modules = [ + (name = "worker", esModule = "import { R2JsrpcResponseEntrypoint, testInvalidR2Responses } from './r2-test.js'; export { R2JsrpcResponseEntrypoint }; export const InvalidR2ResponseTests = { test: testInvalidR2Responses };"), + (name = "r2-test.js", esModule = embed "r2-test.js"), + ], + bindings = [ + (name = "BUCKET", r2Bucket = (name = "slow-structs", entrypoint = "R2JsrpcResponseEntrypoint")), + ], + compatibilityFlags = ["experimental", "nodejs_compat", "service_binding_extra_handlers", + "streams_enable_constructors", "r2_binding_jsrpc", "disable_fast_jsg_struct"], + ), + ), + ], +); diff --git a/src/workerd/api/tests/r2-jsrpc-serialization-test.js b/src/workerd/api/tests/r2-jsrpc-serialization-test.js new file mode 100644 index 00000000000..1cab6500300 --- /dev/null +++ b/src/workerd/api/tests/r2-jsrpc-serialization-test.js @@ -0,0 +1,176 @@ +// Copyright (c) 2017-2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +import assert from 'node:assert'; +import { WorkerEntrypoint } from 'cloudflare:workers'; + +export class ObservedR2Binding extends WorkerEntrypoint { + fetch() { + throw new Error('R2 platform binding tests must not use HTTP'); + } + + async head(...args) { + if (args[0] === 'body-as-head') { + return this.env.REAL_BUCKET.get('rpc-json'); + } + if (args[0] === 'list-object-as-head') { + const listed = await this.env.REAL_BUCKET.list({ + prefix: 'httpMeta', + include: [], + }); + return listed.objects[0]; + } + return this.env.REAL_BUCKET.head(...args); + } + + async get(...args) { + if (args[0] === 'list-object-as-get') { + const listed = await this.env.REAL_BUCKET.list({ + prefix: 'httpMeta', + include: [], + }); + return listed.objects[0]; + } + return this.env.REAL_BUCKET.get(...args); + } + + put(...args) { + return this.env.REAL_BUCKET.put(...args); + } + + list(...args) { + return this.env.REAL_BUCKET.list(...args); + } + + async checksums() { + return (await this.env.REAL_BUCKET.head('multipleChecksums')).checksums; + } + + getNative(key) { + return this.env.REAL_BUCKET.get(key); + } + + async createMultipartUpload(...args) { + const upload = await this.env.REAL_BUCKET.createMultipartUpload(...args); + return { key: upload.key, uploadId: upload.uploadId }; + } + + uploadPart(key, uploadId, partNumber, value, options) { + return this.env.REAL_BUCKET.resumeMultipartUpload(key, uploadId).uploadPart( + partNumber, + value, + options + ); + } + + abortMultipartUpload(key, uploadId) { + return this.env.REAL_BUCKET.resumeMultipartUpload(key, uploadId).abort(); + } + + completeMultipartUpload(key, uploadId, uploadedParts) { + return this.env.REAL_BUCKET.resumeMultipartUpload(key, uploadId).complete( + uploadedParts + ); + } +} + +export const nativeR2ResultTests = { + async test(_controller, env) { + const checksums = await env.PLATFORM.checksums(); + assert.throws(() => structuredClone(checksums), { name: 'DataCloneError' }); + const checksumJson = checksums.toJSON(); + assert.strictEqual(checksumJson.md5, '9a0364b9e99bb480dd25e1f0284c8555'); + assert.strictEqual( + checksumJson.sha1, + '2a0364b9e99bb480dd25e1f0284c855511223344' + ); + assert.strictEqual( + checksumJson.sha256, + '3a0364b9e99bb480dd25e1f0284c8555112233445566778899aabbccddeeff00' + ); + + const head = await env.BUCKET.head('multipleChecksums'); + assert.throws(() => structuredClone(head), { name: 'DataCloneError' }); + assert.strictEqual(head.key, 'basicKey'); + assert.strictEqual(typeof head.writeHttpMetadata, 'function'); + const headChecksumJson = head.checksums.toJSON(); + assert.strictEqual( + headChecksumJson.md5, + '9a0364b9e99bb480dd25e1f0284c8555' + ); + assert.strictEqual( + headChecksumJson.sha1, + '2a0364b9e99bb480dd25e1f0284c855511223344' + ); + assert.strictEqual( + headChecksumJson.sha256, + '3a0364b9e99bb480dd25e1f0284c8555112233445566778899aabbccddeeff00' + ); + await assert.rejects(() => env.BUCKET.head('body-as-head'), { + message: /^internal error; reference = \S+$/, + }); + await assert.rejects(() => env.BUCKET.head('list-object-as-head'), { + message: /^internal error; reference = \S+$/, + }); + + const directGet = await env.PLATFORM.getNative('rpc-json'); + assert.throws(() => structuredClone(directGet), { name: 'DataCloneError' }); + assert.deepStrictEqual(await directGet.json(), { ok: true }); + + const object = await env.BUCKET.get('rpc-json'); + assert.strictEqual(typeof object.text, 'function'); + assert.deepStrictEqual(await object.json(), { ok: true }); + + const metadataOnly = await env.BUCKET.get('rpc-conditional-metadata'); + assert.strictEqual(metadataOnly.key, 'basicKey'); + assert.strictEqual(metadataOnly.body, undefined); + await assert.rejects(() => env.BUCKET.get('list-object-as-get'), { + message: /^internal error; reference = \S+$/, + }); + + const put = await env.BUCKET.put('platform-put', 'content'); + assert.strictEqual(put.key, 'basicKey'); + assert.strictEqual(typeof put.writeHttpMetadata, 'function'); + + const listed = await env.BUCKET.list({ + prefix: 'rpc-metadata', + include: ['httpMetadata', 'customMetadata'], + }); + assert.strictEqual(listed.objects.length, 1); + assert.strictEqual(listed.objects[0].key, 'basicKey'); + assert.strictEqual(typeof listed.objects[0].writeHttpMetadata, 'function'); + + const listedWithoutMetadata = await env.BUCKET.list({ + prefix: 'httpMeta', + include: [], + }); + assert.strictEqual( + listedWithoutMetadata.objects[0].httpMetadata, + undefined + ); + assert.strictEqual( + listedWithoutMetadata.objects[0].customMetadata, + undefined + ); + + const upload = await env.BUCKET.createMultipartUpload( + 'rpc-multipart-platform' + ); + assert.strictEqual(upload.key, 'rpc-multipart-platform'); + assert.strictEqual(upload.uploadId, 'rpc-multipart-platform-id'); + const part = await upload.uploadPart(1, 'content'); + assert.strictEqual( + part.etag, + 'rpc-multipart-platform/rpc-multipart-platform-id/1' + ); + const completed = await upload.complete([part]); + assert.strictEqual(completed.key, 'rpc-multipart-platform'); + assert.strictEqual(completed.version, 'rpc-multipart-platform-id'); + + const aborted = await env.BUCKET.createMultipartUpload( + 'rpc-multipart-abort-platform' + ); + await aborted.abort(); + }, +}; diff --git a/src/workerd/api/tests/r2-jsrpc-serialization-test.wd-test b/src/workerd/api/tests/r2-jsrpc-serialization-test.wd-test new file mode 100644 index 00000000000..578799f61b0 --- /dev/null +++ b/src/workerd/api/tests/r2-jsrpc-serialization-test.wd-test @@ -0,0 +1,46 @@ +using Workerd = import "/workerd/workerd.capnp"; + +# The user-facing R2 binding targets a platform Worker. That Worker calls a separate real R2 +# binding whose fetcher targets the mock R2GW entrypoint. Keeping this topology in a dedicated +# config prevents the native-resource round trips from being conflated with the HTTP transport. +const unitTests :Workerd.Config = ( + services = [ + (name = "user", worker = ( + modules = [ + (name = "worker", esModule = "export { nativeR2ResultTests } from './test.js';"), + (name = "test.js", esModule = embed "r2-jsrpc-serialization-test.js"), + ], + bindings = [ + (name = "BUCKET", + r2Bucket = (name = "platform", entrypoint = "ObservedR2Binding")), + (name = "PLATFORM", + service = (name = "platform", entrypoint = "ObservedR2Binding")), + ], + compatibilityFlags = ["experimental", "nodejs_compat", "streams_enable_constructors", + "r2_list_honor_include", "r2_binding_jsrpc", "rpc", + "fetcher_no_get_put_delete"], + )), + (name = "platform", worker = ( + modules = [ + (name = "worker", esModule = "export { ObservedR2Binding } from './test.js';"), + (name = "test.js", esModule = embed "r2-jsrpc-serialization-test.js"), + ], + bindings = [ + (name = "REAL_BUCKET", + r2Bucket = (name = "gateway", entrypoint = "R2BindingEntrypoint")), + ], + compatibilityFlags = ["experimental", "nodejs_compat", "streams_enable_constructors", + "r2_list_honor_include", "r2_binding_jsrpc"], + )), + (name = "gateway", worker = ( + modules = [ + (name = "worker", esModule = "export { R2BindingEntrypoint } from './r2-test.js';"), + (name = "r2-test.js", esModule = embed "r2-test.js"), + ], + bindings = [ + (name = "R2_LIST_HONOR_INCLUDE", text = "true"), + ], + compatibilityFlags = ["nodejs_compat", "streams_enable_constructors"], + )), + ], +); diff --git a/src/workerd/api/tests/r2-jsrpc-test.wd-test b/src/workerd/api/tests/r2-jsrpc-test.wd-test new file mode 100644 index 00000000000..6b70dc8cca7 --- /dev/null +++ b/src/workerd/api/tests/r2-jsrpc-test.wd-test @@ -0,0 +1,49 @@ +using Workerd = import "/workerd/workerd.capnp"; + +# Runs the canonical R2 binding suite (r2-test.js) with the JSRPC transport enabled. +# The named entrypoint rejects every HTTP request to detect any transport fallback. +# +# Reaching the JSRPC method table needs two things to line up: +# - the r2_binding_jsrpc compatibility flag, which is $experimental and so also needs +# "experimental" in the flag list here +# - a binding that targets the named entrypoint the gateway exposes +# +# The `entrypoint` below is the point of this file. A bare-string designator would target the +# default export and would not exercise named-entrypoint dispatch, which is the production shape. +const unitTests :Workerd.Config = ( + services = [ + ( name = "r2-test", + worker = ( + modules = [ + ( name = "worker", esModule = embed "r2-test.js" ) + ], + bindings = [ + ( name = "BUCKET", + r2Bucket = (name = "r2-test", entrypoint = "R2BindingEntrypoint") ), + ( name = "R2_TRANSPORT", text = "jsrpc" ), + ( name = "R2_LIST_HONOR_INCLUDE", text = "true" ), + ], + compatibilityFlags = ["experimental", "nodejs_compat", "service_binding_extra_handlers", + "streams_enable_constructors", "r2_list_honor_include", + "r2_binding_jsrpc", "disable_fast_jsg_struct"], + streamingTails = ["tail"], + ) + ), + (name = "tail", worker = .tailWorker), + ], + extensions = [( + modules = [ + (name = "test:instrumentation-tail", esModule = embed "instrumentation-tail-worker.js"), + ], + )], +); + +const tailWorker :Workerd.Worker = ( + modules = [ + (name = "worker", esModule = embed "r2-instrumentation-test.js"), + ], + bindings = [ + (name = "R2_TRACE_TRANSPORT", text = "jsrpc"), + ], + compatibilityFlags = ["experimental", "nodejs_compat", "disable_fast_jsg_struct"], +); diff --git a/src/workerd/api/tests/r2-test.js b/src/workerd/api/tests/r2-test.js index 7b18aec2053..405968c7ddd 100644 --- a/src/workerd/api/tests/r2-test.js +++ b/src/workerd/api/tests/r2-test.js @@ -3,9 +3,21 @@ // https://opensource.org/licenses/Apache-2.0 import assert from 'node:assert'; +import { WorkerEntrypoint } from 'cloudflare:workers'; const key = 'basicKey'; const body = 'content'; +const rpcStreamBody = 'café'; +const rpcInlineBodyLimit = 16 << 20; +const largeRpcBodySize = rpcInlineBodyLimit + 2; +const perKeyErrorDeleteKeys = ['existing', 'missing']; +const rpcRanges = { + offset: { offset: 1 }, + length: { length: 3 }, + bounded: { offset: 0, length: 3 }, + suffix: { suffix: 2 }, + empty: {}, +}; const httpMetaObj = { contentType: 'text/plain', contentLanguage: 'en-US', @@ -59,7 +71,7 @@ const sha256Buffer = new Uint8Array([ const objResponse = { name: key, version: 'objectVersion', - size: '123', + size: '7', etag: 'objectEtag', uploaded: '1724767257918', storageClass: 'Standard', @@ -80,10 +92,112 @@ const HeadObject = { }, httpEtag: '"objectEtag"', etag: 'objectEtag', - size: 123, + size: 7, version: 'objectVersion', key, }; +const object = { + key: 'key', + version: 'version', + size: 0, + etag: 'etag', + uploaded: new Date(0), + storageClass: 'Standard', + checksums: {}, +}; +const invalidMetadata = { + negativeSize: { size: -1 }, + fractionalSize: { size: 0.5 }, + infiniteSize: { size: Infinity }, + nanSize: { size: NaN }, + negativeOffset: { range: { offset: -1 } }, + fractionalOffset: { range: { offset: 0.5 } }, + negativeLength: { range: { length: -1 } }, + fractionalLength: { range: { length: 0.5 } }, + negativeSuffix: { range: { suffix: -1 } }, + fractionalSuffix: { range: { suffix: 0.5 } }, + suffixWithOffset: { range: { suffix: 1, offset: 0 } }, + suffixWithLength: { range: { suffix: 1, length: 1 } }, + invalidDate: { uploaded: new Date(NaN) }, + invalidChecksum: { checksums: { md5: {} } }, +}; + +function buildRpcHead(requestKey, multipartOptions) { + const result = { + key, + version: objResponse.version, + size: Number(objResponse.size), + etag: objResponse.etag, + uploaded: new Date(Number(objResponse.uploaded)), + storageClass: multipartOptions?.storageClass ?? objResponse.storageClass, + checksums: {}, + httpMetadata: {}, + customMetadata: multipartOptions?.customMetadata ?? {}, + }; + + if (multipartOptions?.httpMetadata !== undefined) { + result.httpMetadata = + multipartOptions.httpMetadata instanceof Headers + ? httpMetaObj + : multipartOptions.httpMetadata; + } + + switch (requestKey) { + case 'httpMetadata': + result.httpMetadata = httpMetaObj; + break; + case 'customMetadata': + result.customMetadata = customMetadata; + break; + case 'classInfrequentAccess': + result.storageClass = 'InfrequentAccess'; + break; + case 'ssec': + case 'ssecMultipart': + result.ssecKeyMd5 = keyMd5; + break; + case 'multipleChecksums': + result.checksums = { + md5: md5Buffer.buffer, + sha1: sha1Buffer.buffer, + sha256: sha256Buffer.buffer, + }; + break; + case 'ranged': + result.range = { offset: 10, length: 20 }; + break; + } + + return result; +} + +async function assertLargeRpcBody(requestKey, value, valueSize) { + assert(value instanceof ReadableStream); + assert.strictEqual(valueSize, largeRpcBodySize); + const uploaded = new Uint8Array(await new Response(value).arrayBuffer()); + assert.strictEqual(uploaded.byteLength, largeRpcBodySize); + + switch (requestKey) { + case 'largeBuffer': + assert.strictEqual(uploaded[0], 0x11); + assert.strictEqual(uploaded[1], 0x41); + assert.strictEqual(uploaded.at(-2), 0x41); + assert.strictEqual(uploaded.at(-1), 0x22); + break; + case 'largeString': + assert.strictEqual(uploaded[0], 0xc3); + assert.strictEqual(uploaded[1], 0xa9); + assert.strictEqual(uploaded.at(-2), 0xc3); + assert.strictEqual(uploaded.at(-1), 0xa9); + break; + case 'largeBlob': + assert.strictEqual(uploaded[0], 0x5a); + assert.strictEqual(uploaded.at(-1), 0x5a); + break; + default: + assert.fail(`unexpected large RPC body key: ${requestKey}`); + } +} function buildGetResponse({ head, body, isList } = {}) { const encoder = new TextEncoder(); @@ -114,6 +228,7 @@ function buildGetResponse({ head, body, isList } = {}) { }, }); } + async function compareResponse(res, { head, body } = {}, bytes) { // Destructuring syntax looks ugly, but gets around needing to construct HeadResponse objects(somehow?) const { ...obj } = await res; @@ -136,7 +251,11 @@ async function compareResponse(res, { head, body } = {}, bytes) { } } -export default { +// TODO: Let each test configure expected calls and responses shared by both transports. +// For example, head('missing') -> not found should produce HTTP 404 or JSRPC null. +// Each adapter should verify the method and arguments, reject unexpected calls, and encode +// the configured response in its transport's format. +const testWorker = { // Handler for HTTP request binding makes to R2 async fetch(request, env, ctx) { // We only expect PUT/Get @@ -158,6 +277,9 @@ export default { reader.releaseLock(); const jsonRequest = JSON.parse(new TextDecoder().decode(value)); + if (jsonRequest.object === 'must-not-call') { + assert.fail('R2 backend must not be called for invalid input'); + } // The rest of the body is the object payload. const decoder = new TextDecoder(); @@ -172,6 +294,25 @@ export default { if (jsonRequest.method === 'delete') { if (jsonRequest.objects) { + // A batch delete can report individual key failures in a successful HTTP response. + // R2Bucket::delete_() reads this body from the legacy transport but does not expose it. + if (jsonRequest.objects[0] === perKeyErrorDeleteKeys[0]) { + assert.deepEqual(jsonRequest.objects, perKeyErrorDeleteKeys); + return Response.json( + [ + { name: 'existing' }, + { + name: 'missing', + error: { + version: 1, + v4code: 10007, + message: 'The specified key does not exist.', + }, + }, + ], + { status: 200 } + ); + } assert.deepEqual(jsonRequest.objects, [key, key + '2']); } else { assert.deepEqual(jsonRequest.object, key); @@ -238,6 +379,19 @@ export default { }); break; } + case 'onlyIfMultipleEtags': { + assert.deepStrictEqual(jsonRequest.onlyIf, { + etagMatches: [ + { value: 'strongEtag', type: 'strong' }, + { value: 'weakEtag', type: 'weak' }, + ], + etagDoesNotMatch: [ + { value: 'firstEtag', type: 'strong' }, + { value: 'secondEtag', type: 'strong' }, + ], + }); + break; + } case 'httpMetadata': { if (jsonRequest.method !== 'completeMultipartUpload') { assert.deepEqual(jsonRequest.httpFields, httpFields); @@ -254,7 +408,7 @@ export default { uploadId: 'multipartId', }); case 'completeMultipartUpload': { - return Response.json(head); + return Response.json(objResponse); } } } @@ -275,7 +429,7 @@ export default { uploadId: 'multipartId', }); case 'completeMultipartUpload': - return Response.json(head); + return Response.json(objResponse); } } // falls through @@ -406,6 +560,9 @@ export default { const rawHeader = request.headers.get('cf-r2-request'); const jsonRequest = JSON.parse(rawHeader); assert.strictEqual(jsonRequest.version, 1); + if (jsonRequest.object === 'must-not-call') { + assert.fail('R2 backend must not be called for invalid input'); + } if (jsonRequest.method === 'list') { switch (jsonRequest.prefix) { case 'basic': { @@ -491,7 +648,7 @@ export default { return buildGetResponse({ head: { range: { - offset: '6', + offset: '5', length: '2', }, }, @@ -532,6 +689,19 @@ export default { }); return buildGetResponse({ body }); } + case 'onlyIfMultipleEtags': { + assert.deepStrictEqual(jsonRequest.onlyIf, { + etagMatches: [ + { value: 'strongEtag', type: 'strong' }, + { value: 'weakEtag', type: 'weak' }, + ], + etagDoesNotMatch: [ + { value: 'firstEtag', type: 'strong' }, + { value: 'secondEtag', type: 'strong' }, + ], + }); + return buildGetResponse({ body }); + } case 'httpMetadata': { const head = { httpFields, @@ -630,7 +800,10 @@ export default { // GetObject(.bytes()) await compareResponse(env.BUCKET.get(key), { body }, true); // HeadObject - await compareResponse(env.BUCKET.head(key)); + const headObject = await env.BUCKET.head(key); + await compareResponse(headObject); + assert.strictEqual(typeof headObject.writeHttpMetadata, 'function'); + assert.strictEqual(typeof headObject.checksums.toJSON, 'function'); // MultipartUploads { // CreateMultipartUpload @@ -672,8 +845,11 @@ export default { } // DeleteObject { - await env.BUCKET.delete(key); - await env.BUCKET.delete([key, 'basicKey2']); + assert.strictEqual(await env.BUCKET.delete(key), undefined); + assert.strictEqual( + await env.BUCKET.delete([key, 'basicKey2']), + undefined + ); } } // Ranged Reads @@ -704,7 +880,7 @@ export default { { head: { range: { - offset: 6, + offset: 5, length: 2, }, }, @@ -714,26 +890,22 @@ export default { } // Conditionals { - try { - await env.BUCKET.put('throwOnInvalidEtag', body, { + await assert.rejects( + env.BUCKET.put('throwOnInvalidEtag', body, { onlyIf: new Headers({ 'if-match': 'strongEtag', }), - }); - throw new Error('This should have thrown'); - } catch { - // intentionally empty - } - try { - await env.BUCKET.put('throwOnInvalidEtag', body, { + }), + { message: 'Invalid ETag in if-match header' } + ); + await assert.rejects( + env.BUCKET.put('throwOnInvalidEtag', body, { onlyIf: new Headers({ 'if-none-match': 'strongEtag', }), - }); - throw new Error('This should have thrown'); - } catch { - // intentionally empty - } + }), + { message: 'Invalid ETag in if-none-match header' } + ); await env.BUCKET.put('onlyIfStrongEtag', body, { onlyIf: { etagMatches: 'strongEtag', @@ -755,6 +927,16 @@ export default { uploadedBefore: new Date('0'), }, }); + const multipleEtagHeaders = new Headers({ + 'if-match': '"strongEtag", W/"weakEtag"', + 'if-none-match': '"firstEtag", "secondEtag"', + }); + await env.BUCKET.put('onlyIfMultipleEtags', body, { + onlyIf: multipleEtagHeaders, + }); + await env.BUCKET.get('onlyIfMultipleEtags', { + onlyIf: multipleEtagHeaders, + }); await env.BUCKET.get('onlyIfWildcard', { onlyIf: { etagMatches: '*', @@ -799,9 +981,16 @@ export default { }); list.objects[0] = { ...list.objects[0] }; list.objects[0].checksums = { ...list.objects[0].checksums }; + const expected = { ...HeadObject, ...head }; + if ( + env.R2_TRANSPORT === 'jsrpc' && + env.R2_LIST_HONOR_INCLUDE === 'true' + ) { + expected.customMetadata = undefined; + } assert.deepEqual(list, { delimitedPrefixes: [], - objects: [{ ...HeadObject, ...head }], + objects: [expected], truncated: false, }); } @@ -811,8 +1000,7 @@ export default { await env.BUCKET.createMultipartUpload('httpMetadata', { httpMetadata, }) - ).complete([]), - { head } + ).complete([]) ); } // customMetadata @@ -844,9 +1032,16 @@ export default { }); list.objects[0] = { ...list.objects[0] }; list.objects[0].checksums = { ...list.objects[0].checksums }; + const expected = { ...HeadObject, ...head }; + if ( + env.R2_TRANSPORT === 'jsrpc' && + env.R2_LIST_HONOR_INCLUDE === 'true' + ) { + expected.httpMetadata = undefined; + } assert.deepEqual(list, { delimitedPrefixes: [], - objects: [{ ...HeadObject, ...head }], + objects: [expected], truncated: false, }); } @@ -856,8 +1051,7 @@ export default { await env.BUCKET.createMultipartUpload('customMetadata', { customMetadata, }) - ).complete([]), - { head } + ).complete([]) ); } } @@ -962,7 +1156,1615 @@ export default { // Also test HEAD operation to verify checksum tags const headResp = await env.BUCKET.head('multipleChecksums'); - assert.ok(headResp); + assert.deepStrictEqual(new Uint8Array(headResp.checksums.md5), md5Buffer); + assert.deepStrictEqual( + new Uint8Array(headResp.checksums.sha1), + sha1Buffer + ); + assert.deepStrictEqual( + new Uint8Array(headResp.checksums.sha256), + sha256Buffer + ); + assert.deepStrictEqual(headResp.checksums.toJSON(), { + md5: '9a0364b9e99bb480dd25e1f0284c8555', + sha1: '2a0364b9e99bb480dd25e1f0284c855511223344', + sha256: + '3a0364b9e99bb480dd25e1f0284c8555112233445566778899aabbccddeeff00', + }); } }, }; + +const finishedRpcUploads = new Set(); + +// Checks that the key and upload ids match test expectations +function assertMultipartIdentity(requestKey, uploadId, action) { + assert.strictEqual(typeof requestKey, 'string'); + assert.strictEqual(typeof uploadId, 'string'); + if (requestKey.startsWith('rpc-multipart-')) { + assert( + [`${requestKey}-id`, 'resumed-a', 'resumed-b', 'missing'].includes( + uploadId + ) + ); + const missing = + uploadId === 'missing' || + finishedRpcUploads.has(JSON.stringify([requestKey, uploadId])); + if (missing && action !== 'abortMultipartUpload') { + throw new Error(`${action}: Upload not found (10024)`); + } + return; + } + const expected = { + 'must-not-call': 'invalidPartUploadId', + rpcStream: 'streamedId', + largeBuffer: 'largeUploadId', + 'rpc-wrong-part-number': 'wrongPartUploadId', + }[requestKey]; + if (expected !== undefined) { + assert.strictEqual(uploadId, expected); + } else { + assert( + uploadId === 'multipartId' || + (requestKey === key && uploadId === 'resumedId') + ); + } +} + +// This entrypoint rejects HTTP requests so the JSRPC tests detect any transport fallback. +// The HTTP tests use the default export's backend. +export class R2BindingEntrypoint extends WorkerEntrypoint { + fetch() { + throw new Error('R2 JSRPC tests must not use HTTP'); + } + + head(requestKey) { + if (requestKey === 'missing') { + return null; + } + if (requestKey === 'boom') { + throw new Error('head: no such bucket (10006)'); + } + + const result = buildRpcHead(requestKey); + if (requestKey === 'rpc-malformed-size') { + result.size = -1; + } + if (requestKey === 'rpc-malformed-range') { + result.range = { offset: -1, length: 1 }; + } + if (requestKey === 'rpc-malformed-date') { + result.uploaded = new Date(NaN); + } + return result; + } + + get(requestKey, options) { + if (requestKey.startsWith('rpc-range-')) { + const range = options?.range; + const rangeKey = requestKey.slice('rpc-range-'.length); + if (rangeKey === 'headers') { + assert(range instanceof Headers); + assert.strictEqual(range.get('range'), 'bytes=1-3'); + assert.strictEqual(range.get('x-test'), 'preserved'); + } else if (rangeKey === 'no-range') { + assert.strictEqual(range, undefined); + } else if (Object.hasOwn(rpcRanges, rangeKey)) { + assert.deepStrictEqual(range, rpcRanges[rangeKey]); + assert.strictEqual('suffix' in range, rangeKey === 'suffix'); + } else { + assert.fail( + 'Invalid ranges must be rejected before calling the gateway' + ); + } + return null; + } + if (requestKey === 'missing') { + return null; + } + if (requestKey === 'rpc-get-boom') { + throw new Error('get: no such bucket (10006)'); + } + if (requestKey === 'rpc-malformed-kind') { + return { kind: 'other', object: buildRpcHead(requestKey) }; + } + if (requestKey === 'rpc-malformed-metadata-body') { + return { + kind: 'metadata', + object: buildRpcHead(requestKey), + body: new ReadableStream(), + }; + } + if (requestKey === 'rpc-malformed-missing-body') { + return { kind: 'body', object: buildRpcHead(requestKey) }; + } + if (requestKey === 'rpc-options') { + assert.deepStrictEqual(options, { + onlyIf: { + etagMatches: 'strongEtag', + uploadedBefore: new Date(0), + secondsGranularity: false, + }, + range: { offset: 1, length: 3 }, + ssecKey: hexKey, + }); + } + if (requestKey === 'rpc-header-options') { + assert(options.onlyIf instanceof Headers); + assert.strictEqual( + options.onlyIf.get('if-match'), + '"strongEtag", W/"weakEtag"' + ); + assert.strictEqual( + options.onlyIf.get('if-none-match'), + '"firstEtag", "secondEtag"' + ); + assert.strictEqual( + options.onlyIf.get('if-modified-since'), + new Date(0).toUTCString() + ); + assert(options.range instanceof Headers); + assert.strictEqual(options.range.get('range'), 'bytes=1-3'); + } + if (requestKey === 'onlyIfMultipleEtags') { + assert(options.onlyIf instanceof Headers); + assert.strictEqual( + options.onlyIf.get('if-match'), + '"strongEtag", W/"weakEtag"' + ); + assert.strictEqual( + options.onlyIf.get('if-none-match'), + '"firstEtag", "secondEtag"' + ); + } + if (requestKey === 'must-not-call') { + throw new Error('get RPC method must not be called'); + } + + const object = buildRpcHead(requestKey); + if (requestKey === 'rangeOffLen') { + assert.deepStrictEqual(options, { range: { offset: 1, length: 3 } }); + object.range = { offset: 1, length: 3 }; + } else if (requestKey === 'rangeSuff') { + assert.deepStrictEqual(options, { range: { suffix: 2 } }); + object.range = { offset: 5, length: 2 }; + } + + if (requestKey === 'rpc-conditional-metadata') { + return { kind: 'metadata', object }; + } + + const responseBody = + requestKey === 'rangeOffLen' + ? 'ont' + : requestKey === 'rangeSuff' + ? 'nt' + : requestKey === 'rpc-json' + ? JSON.stringify({ ok: true }) + : body; + if (object.range === undefined) { + object.size = new TextEncoder().encode(responseBody).byteLength; + } + let sent = false; + return { + kind: 'body', + object, + body: new ReadableStream({ + pull(controller) { + assert.strictEqual(sent, false); + sent = true; + controller.enqueue(new TextEncoder().encode(responseBody)); + controller.close(); + }, + }), + }; + } + + delete(keys) { + if (keys === 'boom') { + throw new Error('delete: bad keys (10021)'); + } + if (Array.isArray(keys)) { + if (keys[0] === perKeyErrorDeleteKeys[0]) { + // R2BindingEntrypoint has already discarded the successful HTTP batch response body before + // deleteRpc() receives this void RPC result. + assert.deepEqual(keys, perKeyErrorDeleteKeys); + } else { + assert.deepEqual(keys, [key, key + '2']); + } + } else { + assert.strictEqual(keys, key); + } + return rpcSuccess(null); + } + + async put(requestKey, value, options, valueSize) { + if (requestKey === 'must-not-call') { + throw new Error('put RPC method must not be called'); + } + if (requestKey === 'rpc-null-value') { + assert.strictEqual(value, null); + assert.strictEqual(valueSize, 0); + return buildRpcHead(requestKey); + } + if (requestKey === 'rpc-conditional-null') { + assert.deepStrictEqual(options.onlyIf, { + etagMatches: 'strongEtag', + secondsGranularity: false, + }); + return null; + } + if (requestKey === 'rpc-put-options') { + assert.deepStrictEqual(options.onlyIf, { + etagMatches: 'strongEtag', + uploadedBefore: new Date(0), + secondsGranularity: false, + }); + assert.deepStrictEqual(options.httpMetadata, httpMetaObj); + assert.deepStrictEqual(options.customMetadata, customMetadata); + assert.deepStrictEqual(new Uint8Array(options.md5), md5Buffer); + assert.strictEqual(options.storageClass, 'InfrequentAccess'); + assert.strictEqual(options.ssecKey, hexKey); + } + if (requestKey === 'rpc-put-header-options') { + assert(options.onlyIf instanceof Headers); + assert.strictEqual( + options.onlyIf.get('if-match'), + '"strongEtag", W/"weakEtag"' + ); + assert.strictEqual( + options.onlyIf.get('if-none-match'), + '"firstEtag", "secondEtag"' + ); + } + if (requestKey === 'onlyIfMultipleEtags') { + assert(options.onlyIf instanceof Headers); + assert.strictEqual( + options.onlyIf.get('if-match'), + '"strongEtag", W/"weakEtag"' + ); + assert.strictEqual( + options.onlyIf.get('if-none-match'), + '"firstEtag", "secondEtag"' + ); + } + if (requestKey.startsWith('large')) { + await assertLargeRpcBody(requestKey, value, valueSize); + return buildRpcHead(requestKey, options); + } + const uploaded = await new Response(value).text(); + const expected = requestKey === 'rpcStream' ? rpcStreamBody : body; + assert.strictEqual(uploaded, expected); + assert.strictEqual( + valueSize, + new TextEncoder().encode(expected).byteLength + ); + return buildRpcHead(requestKey, options); + } + + list(options) { + const honorsIncludes = this.env.R2_LIST_HONOR_INCLUDE === 'true'; + const effectiveIncludes = honorsIncludes + ? (options?.include ?? []) + : ['httpMetadata', 'customMetadata']; + + if (options?.prefix === 'rpc-boom') { + throw new Error('list: no such bucket (10006)'); + } + if (options?.prefix === 'rpc-malformed') { + return { objects: [{}], truncated: false }; + } + if (options?.prefix === 'rpc-options') { + assert.deepStrictEqual(options, { + limit: 2, + prefix: 'rpc-options', + cursor: 'cursor-in', + delimiter: '/', + startAfter: 'after', + include: effectiveIncludes, + }); + return { + objects: [], + truncated: true, + cursor: 'cursor-out', + delimitedPrefixes: ['rpc-options/'], + }; + } + + if (options?.prefix === 'basic') { + assert.deepStrictEqual(options, { + limit: 1, + prefix: 'basic', + cursor: 'ai', + delimiter: '/', + include: effectiveIncludes, + }); + return { + objects: [listRpcHead('basic', effectiveIncludes)], + truncated: true, + cursor: 'ai', + }; + } + if (options?.prefix === 'httpMeta') { + return { + objects: [listRpcHead('httpMetadata', effectiveIncludes)], + truncated: false, + }; + } + if (options?.prefix === 'customMeta') { + return { + objects: [listRpcHead('customMetadata', effectiveIncludes)], + truncated: false, + }; + } + if (options?.prefix === 'rpc-metadata') { + return { + objects: [listRpcHead('basic', effectiveIncludes)], + truncated: false, + }; + } + + if (honorsIncludes) { + assert.strictEqual(options, undefined); + } else { + assert.deepStrictEqual(options, { + include: ['httpMetadata', 'customMetadata'], + }); + } + return { objects: [], truncated: false }; + } + + createMultipartUpload(requestKey, options) { + if (requestKey === 'must-not-call') { + throw new Error('createMultipartUpload RPC method must not be called'); + } + if (requestKey === 'rpc-create-options') { + assert.deepStrictEqual(options.httpMetadata, httpMetaObj); + assert.deepStrictEqual(options.customMetadata, customMetadata); + assert.strictEqual(options.storageClass, 'InfrequentAccess'); + assert.strictEqual(options.ssecKey, hexKey); + } + assert.strictEqual(typeof requestKey, 'string'); + return requestKey.startsWith('rpc-multipart-') + ? `${requestKey}-id` + : 'multipartId'; + } + + async uploadPart( + requestKey, + uploadId, + partNumber, + value, + options, + valueSize + ) { + assertMultipartIdentity(requestKey, uploadId, 'uploadPart'); + assert(partNumber >= 1 && partNumber <= 10000); + if (requestKey === 'must-not-call') { + throw new Error('uploadPart RPC method must not be called'); + } + if (requestKey === 'largeBuffer') { + await assertLargeRpcBody(requestKey, value, valueSize); + return { partNumber, etag: 'partEtag' }; + } + const uploaded = await new Response(value).text(); + const expected = + requestKey === 'ssecMultipart' + ? 'hey' + : requestKey === 'rpcStream' + ? rpcStreamBody + : body; + assert.strictEqual(uploaded, expected); + assert.strictEqual( + valueSize, + new TextEncoder().encode(expected).byteLength + ); + if (requestKey === 'ssecMultipart') { + assert.strictEqual(options?.ssecKey, hexKey); + } + return { + partNumber: requestKey === 'rpc-wrong-part-number' ? 9999 : partNumber, + etag: requestKey.startsWith('rpc-multipart-') + ? `${requestKey}/${uploadId}/${partNumber}` + : uploadId === 'resumedId' + ? 'resumedRpcPartEtag' + : 'partEtag', + }; + } + + abortMultipartUpload(requestKey, uploadId) { + assertMultipartIdentity(requestKey, uploadId, 'abortMultipartUpload'); + if (requestKey.startsWith('rpc-multipart-')) { + finishedRpcUploads.add(JSON.stringify([requestKey, uploadId])); + } + } + + completeMultipartUpload(requestKey, uploadId, uploadedParts) { + assertMultipartIdentity(requestKey, uploadId, 'completeMultipartUpload'); + if (requestKey === 'must-not-call') { + throw new Error('completeMultipartUpload RPC method must not be called'); + } + for (const part of uploadedParts) { + assert(part.partNumber >= 1 && part.partNumber <= 10000); + assert.strictEqual(typeof part.etag, 'string'); + if (requestKey.startsWith('rpc-multipart-')) { + assert.strictEqual( + part.etag, + `${requestKey}/${uploadId}/${part.partNumber}` + ); + } + } + const result = buildRpcHead(requestKey); + // The backend completion response omits create-time HTTP and custom metadata. + result.httpMetadata = {}; + result.customMetadata = {}; + if (requestKey.startsWith('rpc-multipart-')) { + finishedRpcUploads.add(JSON.stringify([requestKey, uploadId])); + result.key = requestKey; + result.version = uploadId; + } + if (uploadId === 'resumedId') { + result.version = 'resumedRpcObjectVersion'; + } + return result; + } +} + +function listRpcHead(requestKey, includes) { + const result = buildRpcHead(requestKey); + if (!includes.includes('httpMetadata')) { + delete result.httpMetadata; + } + if (!includes.includes('customMetadata')) { + delete result.customMetadata; + } + return result; +} + +// Test that documents the behaviour of multi-key deletion where one fails and the other succeeds +export const deletePerKeyErrorParityTests = { + async test(_ctrl, env) { + if (env.R2_TRANSPORT !== 'http' && env.R2_TRANSPORT !== 'jsrpc') { + return; + } + + // Both transports implement Promise: the legacy path discards the successful HTTP body, + // while the RPC path receives the corresponding void result from R2BindingEntrypoint. + const deletion = env.BUCKET.delete(perKeyErrorDeleteKeys); + assert(deletion instanceof Promise); + assert.strictEqual(await deletion, undefined); + }, +}; + +export const r2ValidationTests = { + async test(ctrl, env) { + await assert.rejects( + env.BUCKET.get('must-not-call', { range: { offset: -1 } }), + { + message: + 'Invalid range. Starting offset (-1) must be greater than or equal to 0.', + } + ); + await assert.rejects( + env.BUCKET.get('must-not-call', { range: { suffix: 1, offset: 0 } }), + { name: 'TypeError', message: 'Suffix is incompatible with offset.' } + ); + await assert.rejects( + env.BUCKET.get('must-not-call', { range: { suffix: 1, length: 1 } }), + { name: 'TypeError', message: 'Suffix is incompatible with length.' } + ); + for (const field of ['offset', 'length', 'suffix']) { + for (const value of [-1, 0.5, NaN]) { + await assert.rejects( + env.BUCKET.get('must-not-call', { range: { [field]: value } }), + { name: 'RangeError' } + ); + } + } + await assert.rejects(env.BUCKET.get('must-not-call', { ssecKey: 'bad' }), { + message: 'SSE-C Key must be 32 bytes in length', + }); + await assert.rejects( + env.BUCKET.get('must-not-call', { + onlyIf: { etagMatches: '"quoted"' }, + }), + { + message: 'Conditional ETag should not be wrapped in quotes ("quoted").', + } + ); + + await assert.rejects( + env.BUCKET.put('must-not-call', body, { md5: new Uint8Array(1) }), + { message: 'MD5 is 16 bytes, not 1' } + ); + await assert.rejects( + env.BUCKET.put('must-not-call', body, { + md5: 'zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz', + }), + { message: "Provided MD5 wasn't a valid hex string" } + ); + await assert.rejects( + env.BUCKET.put('must-not-call', body, { + md5: md5Buffer, + sha1: sha1Buffer, + }), + { message: 'You cannot specify multiple hashing algorithms.' } + ); + + const invalidOptionsStream = new FixedLengthStream(1); + const invalidOptionsWriter = invalidOptionsStream.writable.getWriter(); + const invalidOptionsWriting = invalidOptionsWriter.write( + new Uint8Array([1]) + ); + await assert.rejects( + env.BUCKET.put('must-not-call', invalidOptionsStream.readable, { + onlyIf: { etagMatches: '"quoted"' }, + }), + { + message: 'Conditional ETag should not be wrapped in quotes ("quoted").', + } + ); + await assert.rejects(invalidOptionsWriting, { + message: + 'Stream cancelled because the associated put operation encountered an error.', + }); + + await assert.rejects( + env.BUCKET.createMultipartUpload('must-not-call', { + ssecKey: 'bad', + }), + { message: 'SSE-C Key must be 32 bytes in length' } + ); + const invalidPartUpload = env.BUCKET.resumeMultipartUpload( + 'must-not-call', + 'invalidPartUploadId' + ); + await assert.rejects( + invalidPartUpload.uploadPart(1, body, { ssecKey: 'bad' }), + { message: 'SSE-C Key must be 32 bytes in length' } + ); + await assert.rejects( + invalidPartUpload.complete([{ partNumber: 0, etag: 'invalid' }]), + { + message: + 'Part number must be between 1 and 10000 (inclusive). Actual value was: 0', + } + ); + }, +}; + +// These cases cover RPC boundary behavior that the HTTP-oriented fake cannot observe directly. +// The canonical API suite remains identical for both transport configurations. +export const jsrpcTransportTests = { + async test(ctrl, env, ctx) { + if (env.R2_TRANSPORT !== 'jsrpc') { + return; + } + + assert.strictEqual(await env.BUCKET.head('missing'), null); + + const ranged = await env.BUCKET.head('ranged'); + assert.deepStrictEqual(ranged.range, { offset: 10, length: 20 }); + + await assert.rejects(env.BUCKET.head('boom'), (err) => { + assert.strictEqual(err.message, 'head: no such bucket (10006)'); + assert.strictEqual(err.code, undefined); + return true; + }); + await assert.rejects(env.BUCKET.head('rpc-malformed-size'), { + message: /^internal error; reference = \S+$/, + }); + await assert.rejects(env.BUCKET.head('rpc-malformed-range'), { + message: /^internal error; reference = \S+$/, + }); + await assert.rejects(env.BUCKET.head('rpc-malformed-date'), { + message: /^internal error; reference = \S+$/, + }); + + await assert.rejects(env.BUCKET.delete('boom'), { + message: 'delete: bad keys (10021)', + }); + + const coerced = await env.BUCKET.head(12345); + assert.strictEqual(coerced.key, key); + + assert.strictEqual(await env.BUCKET.get('missing'), null); + + const conditional = await env.BUCKET.get('rpc-conditional-metadata', { + onlyIf: { etagMatches: 'objectEtag' }, + }); + assert.strictEqual(conditional.body, undefined); + assert.deepStrictEqual(conditional.httpMetadata, {}); + assert.deepStrictEqual(conditional.customMetadata, {}); + assert.strictEqual(typeof conditional.writeHttpMetadata, 'function'); + + const lazyBody = await env.BUCKET.get('rpc-lazy-body'); + assert.strictEqual(lazyBody.bodyUsed, false); + assert(lazyBody.body instanceof ReadableStream); + assert.strictEqual(await lazyBody.text(), body); + assert.strictEqual(lazyBody.bodyUsed, true); + + const bytesResult = await env.BUCKET.get('rpc-bytes'); + assert.deepStrictEqual( + await bytesResult.bytes(), + new TextEncoder().encode(body) + ); + assert.strictEqual(bytesResult.bodyUsed, true); + const bufferResult = await env.BUCKET.get('rpc-array-buffer'); + assert.deepStrictEqual( + new Uint8Array(await bufferResult.arrayBuffer()), + new TextEncoder().encode(body) + ); + const jsonResult = await env.BUCKET.get('rpc-json'); + assert.deepStrictEqual(await jsonResult.json(), { ok: true }); + const blobResult = await env.BUCKET.get('rpc-blob'); + assert.strictEqual(await (await blobResult.blob()).text(), body); + + await env.BUCKET.get('rpc-options', { + onlyIf: { + etagMatches: 'strongEtag', + uploadedBefore: new Date(0), + }, + range: { offset: 1, length: 3 }, + ssecKey: bufferKey, + }); + await env.BUCKET.get('rpc-header-options', { + onlyIf: new Headers({ + 'if-match': '"strongEtag", W/"weakEtag"', + 'if-none-match': '"firstEtag", "secondEtag"', + 'if-modified-since': new Date(0).toUTCString(), + }), + range: new Headers({ range: 'bytes=1-3' }), + }); + + await assert.rejects(env.BUCKET.get('rpc-malformed-kind'), { + message: /^internal error; reference = \S+$/, + }); + await assert.rejects(env.BUCKET.get('rpc-malformed-metadata-body'), { + message: /^internal error; reference = \S+$/, + }); + // The missing-body assertion logs the details and exposes only an internal error reference. + await assert.rejects(env.BUCKET.get('rpc-malformed-missing-body'), { + message: /^internal error; reference = \S+$/, + }); + await assert.rejects(env.BUCKET.get('rpc-get-boom'), { + message: 'get: no such bucket (10006)', + }); + + const emptyList = await env.BUCKET.list(); + assert.deepStrictEqual(emptyList, { + objects: [], + truncated: false, + delimitedPrefixes: [], + }); + + const optionList = await env.BUCKET.list({ + limit: 2, + prefix: 'rpc-options', + cursor: 'cursor-in', + delimiter: '/', + startAfter: 'after', + include: [], + }); + assert.deepStrictEqual(optionList, { + objects: [], + truncated: true, + cursor: 'cursor-out', + delimitedPrefixes: ['rpc-options/'], + }); + + if (env.R2_LIST_HONOR_INCLUDE === 'true') { + for (const include of [ + [], + ['httpMetadata'], + ['customMetadata'], + ['httpMetadata', 'customMetadata'], + ]) { + const listed = await env.BUCKET.list({ + prefix: 'rpc-metadata', + include, + }); + const object = listed.objects[0]; + assert.strictEqual(typeof object.writeHttpMetadata, 'function'); + assert.strictEqual(typeof object.checksums.toJSON, 'function'); + assert.strictEqual( + object.httpMetadata === undefined, + !include.includes('httpMetadata') + ); + assert.strictEqual( + object.customMetadata === undefined, + !include.includes('customMetadata') + ); + if (include.includes('httpMetadata')) { + assert.deepStrictEqual(object.httpMetadata, {}); + } + if (include.includes('customMetadata')) { + assert.deepStrictEqual(object.customMetadata, {}); + } + } + } + + await assert.rejects(env.BUCKET.list({ prefix: 'rpc-boom' }), { + message: 'list: no such bucket (10006)', + }); + await assert.rejects(env.BUCKET.list({ prefix: 'rpc-malformed' }), { + message: /^internal error; reference = \S+$/, + }); + + const nullPut = await env.BUCKET.put('rpc-null-value', null); + assert.strictEqual(nullPut.key, key); + assert.strictEqual( + await env.BUCKET.put('rpc-conditional-null', body, { + onlyIf: { etagMatches: 'strongEtag' }, + }), + null + ); + await env.BUCKET.put('rpc-put-options', body, { + onlyIf: { + etagMatches: 'strongEtag', + uploadedBefore: new Date(0), + }, + httpMetadata: httpMetaHeaders, + customMetadata, + md5: md5Buffer, + storageClass: 'InfrequentAccess', + ssecKey: bufferKey, + }); + await env.BUCKET.put('rpc-put-header-options', body, { + onlyIf: new Headers({ + 'if-match': '"strongEtag", W/"weakEtag"', + 'if-none-match': '"firstEtag", "secondEtag"', + }), + }); + + const encodedStreamBody = new TextEncoder().encode(rpcStreamBody); + { + const { readable, writable } = new FixedLengthStream( + encodedStreamBody.byteLength + ); + const writer = writable.getWriter(); + const writing = writer + .write(encodedStreamBody) + .then(() => writer.close()); + const result = await env.BUCKET.put('rpcStream', readable); + await writing; + assert.strictEqual(result.size, Number(objResponse.size)); + } + + let unknownLengthCancelReason; + await assert.rejects( + env.BUCKET.put( + 'unknownLengthStream', + new ReadableStream({ + cancel(reason) { + unknownLengthCancelReason = reason; + }, + }) + ), + { + message: + 'Provided readable stream must have a known length (request/response body or readable half of FixedLengthStream)', + } + ); + assert.strictEqual( + unknownLengthCancelReason.message, + 'Stream cancelled because the associated put operation encountered an error.' + ); + + const largeBufferBacking = new Uint8Array(largeRpcBodySize + 2); + largeBufferBacking.fill(0x41); + const largeBuffer = largeBufferBacking.subarray(1, -1); + largeBuffer[0] = 0x11; + largeBuffer[largeBuffer.length - 1] = 0x22; + await env.BUCKET.put('largeBuffer', largeBuffer); + + const largeString = 'é'.repeat(largeRpcBodySize / 2); + await env.BUCKET.put('largeString', largeString); + + const largeBlobBytes = new Uint8Array(largeRpcBodySize); + largeBlobBytes.fill(0x5a); + await env.BUCKET.put('largeBlob', new Blob([largeBlobBytes])); + }, +}; + +export const jsrpcMultipartTests = { + async test(ctrl, env) { + if (env.R2_TRANSPORT !== 'jsrpc') { + return; + } + + const createOptionsUpload = await env.BUCKET.createMultipartUpload( + 'rpc-create-options', + { + httpMetadata: httpMetaHeaders, + customMetadata, + storageClass: 'InfrequentAccess', + ssecKey: bufferKey, + } + ); + assert.strictEqual(createOptionsUpload.uploadId, 'multipartId'); + + const encodedStreamBody = new TextEncoder().encode(rpcStreamBody); + // Use a view with a non-zero byte offset to verify that the RPC upload streams only the view's + // bytes, rather than the entire underlying ArrayBuffer. + const largeBufferBacking = new Uint8Array(largeRpcBodySize + 2); + largeBufferBacking.fill(0x41); + const largeBuffer = largeBufferBacking.subarray(1, -1); + largeBuffer[0] = 0x11; + largeBuffer[largeBuffer.length - 1] = 0x22; + + const resumed = env.BUCKET.resumeMultipartUpload(key, 'resumedId'); + assert.strictEqual(resumed.key, key); + assert.strictEqual(resumed.uploadId, 'resumedId'); + const resumedPart = await resumed.uploadPart(1, body); + assert.deepStrictEqual(resumedPart, { + partNumber: 1, + etag: 'resumedRpcPartEtag', + }); + const resumedObject = await resumed.complete([resumedPart]); + assert.strictEqual(resumedObject.key, key); + assert.strictEqual(resumedObject.version, 'resumedRpcObjectVersion'); + assert.strictEqual(resumedObject.etag, objResponse.etag); + assert.strictEqual(resumedObject.httpEtag, `"${objResponse.etag}"`); + assert.strictEqual(resumedObject.size, Number(objResponse.size)); + assert(resumedObject.uploaded instanceof Date); + assert.strictEqual(resumedObject.storageClass, objResponse.storageClass); + const resumedHeaders = new Headers(); + resumedObject.writeHttpMetadata(resumedHeaders); + assert.deepStrictEqual([...resumedHeaders], []); + assert.strictEqual(typeof resumedObject.checksums.toJSON, 'function'); + assert.strictEqual(JSON.stringify(resumedObject.checksums), '{}'); + + const streamedUpload = env.BUCKET.resumeMultipartUpload( + 'rpcStream', + 'streamedId' + ); + const { readable, writable } = new FixedLengthStream( + encodedStreamBody.byteLength + ); + const writer = writable.getWriter(); + const writing = writer.write(encodedStreamBody).then(() => writer.close()); + const streamedPart = await streamedUpload.uploadPart(2, readable); + await writing; + assert.deepStrictEqual(streamedPart, { + partNumber: 2, + etag: 'partEtag', + }); + + const largeUpload = env.BUCKET.resumeMultipartUpload( + 'largeBuffer', + 'largeUploadId' + ); + assert.deepStrictEqual(await largeUpload.uploadPart(1, largeBuffer), { + partNumber: 1, + etag: 'partEtag', + }); + + const wrongPartUpload = env.BUCKET.resumeMultipartUpload( + 'rpc-wrong-part-number', + 'wrongPartUploadId' + ); + assert.deepStrictEqual(await wrongPartUpload.uploadPart(3, body), { + partNumber: 3, + etag: 'partEtag', + }); + + // Each operation carries its key and upload ID, including interleaved calls on the same key. + const uploads = []; + for (const uploadKey of ['rpc-multipart-first', 'rpc-multipart-second']) { + const headers = new Headers(httpMetaHeaders); + const metadata = { uploadKey }; + const creating = env.BUCKET.createMultipartUpload(uploadKey, { + httpMetadata: headers, + customMetadata: metadata, + }); + headers.set('content-type', 'application/octet-stream'); + metadata.uploadKey = 'mutated'; + const upload = await creating; + assert.strictEqual(upload.key, uploadKey); + assert.strictEqual(upload.uploadId, `${uploadKey}-id`); + uploads.push(upload); + } + for (const uploadId of ['resumed-a', 'resumed-b']) { + const upload = env.BUCKET.resumeMultipartUpload( + 'rpc-multipart-first', + uploadId + ); + assert.strictEqual(upload.key, 'rpc-multipart-first'); + assert.strictEqual(upload.uploadId, uploadId); + uploads.push(upload); + } + const uploadedParts = await Promise.all( + uploads.map(async (upload) => { + const first = await upload.uploadPart(1, body); + const rest = await Promise.all([ + upload.uploadPart(2, body), + upload.uploadPart(3, body), + ]); + const parts = [first, ...rest]; + assert.deepStrictEqual( + parts, + [1, 2, 3].map((partNumber) => ({ + partNumber, + etag: `${upload.key}/${upload.uploadId}/${partNumber}`, + })) + ); + return parts; + }) + ); + + for (let i = 0; i < uploads.length; i++) { + const upload = uploads[i]; + const result = await upload.complete(uploadedParts[i]); + assert.strictEqual(result.key, upload.key); + assert.strictEqual(result.version, upload.uploadId); + // Created and resumed uploads both use the metadata returned by the gateway. + assert.deepStrictEqual(result.customMetadata, {}); + const resultHeaders = new Headers(); + result.writeHttpMetadata(resultHeaders); + assert.deepStrictEqual([...resultHeaders], []); + } + + const aborted = await env.BUCKET.createMultipartUpload( + 'rpc-multipart-aborted' + ); + await aborted.uploadPart(1, body); + // Abort must address the same upload when called through a separately resumed wrapper. + await env.BUCKET.resumeMultipartUpload( + aborted.key, + aborted.uploadId + ).abort(); + const missing = env.BUCKET.resumeMultipartUpload( + 'rpc-multipart-missing', + 'missing' + ); + for (const upload of [aborted, missing]) { + await assert.rejects(upload.uploadPart(2, body), { + message: 'uploadPart: Upload not found (10024)', + }); + await assert.rejects(upload.complete([]), { + message: 'completeMultipartUpload: Upload not found (10024)', + }); + await upload.abort(); + await upload.abort(); + } + }, +}; + +export const jsrpcRangeTests = { + async test(controller, env) { + if (env.R2_TRANSPORT !== 'jsrpc') { + return; + } + + for (const [rangeKey, range] of Object.entries(rpcRanges)) { + assert.strictEqual( + await env.BUCKET.get(`rpc-range-${rangeKey}`, { range }), + null + ); + } + assert.strictEqual( + await env.BUCKET.get('rpc-range-offset', { + range: { offset: 1, length: undefined, suffix: undefined }, + }), + null + ); + assert.strictEqual( + await env.BUCKET.get('rpc-range-suffix', { + range: { offset: undefined, length: undefined, suffix: 2 }, + }), + null + ); + assert.strictEqual( + await env.BUCKET.get('rpc-range-empty', { + range: { offset: undefined, length: undefined, suffix: undefined }, + }), + null + ); + + assert.strictEqual( + await env.BUCKET.get('rpc-range-headers', { + range: new Headers({ range: 'bytes=1-3', 'x-test': 'preserved' }), + }), + null + ); + assert.strictEqual(await env.BUCKET.get('rpc-range-no-range'), null); + assert.strictEqual(await env.BUCKET.get('rpc-range-no-range', {}), null); + assert.strictEqual( + await env.BUCKET.get('rpc-range-no-range', { range: new Headers() }), + null + ); + }, +}; + +// The source-kind matrix uses the same four bytes and reports object.size = 4. It checks whether +// the stream itself knows its byte count: PUT receives only the body, not object.size. +const bodyLengthBytes = new TextEncoder().encode('data'); +const bodyLengthCopyCases = { + empty: { content: '', size: 0 }, + utf8: { content: 'café🌍', size: 9 }, + chunks: { content: 'data', size: 4 }, + bounded: { + content: 'at', + size: 4, + options: { range: { offset: 1, length: 2 } }, + range: { offset: 1, length: 2 }, + }, + offset: { + content: 'ta', + size: 4, + options: { range: { offset: 2 } }, + range: { offset: 2, length: 2 }, + }, + suffix: { + content: 'ta', + size: 4, + options: { range: { suffix: 2 } }, + range: { offset: 2, length: 2 }, + }, + shortened: { + content: 'ta', + size: 4, + options: { range: { offset: 2, length: 99 } }, + range: { offset: 2, length: 2 }, + }, + remaining: { content: 'ata', size: 3 }, +}; +let bodyLengthReleased; +let bodyLengthCancellation; + +function bodyLengthUploadBytes(requestKey) { + return requestKey === 'destination' + ? bodyLengthBytes + : new TextEncoder().encode(bodyLengthCopyCases[requestKey].content); +} + +// The HTTP gateway sends [JSON metadata][object bytes]. A buffer gives the HTTP transport +// a known length; wrapping the same bytes in an ordinary JS stream does not. +function buildBodyLengthResponse(mode) { + const metadata = new TextEncoder().encode( + JSON.stringify({ ...objResponse, size: String(bodyLengthBytes.byteLength) }) + ); + const payload = new Uint8Array( + metadata.byteLength + bodyLengthBytes.byteLength + ); + payload.set(metadata); + payload.set(bodyLengthBytes, metadata.byteLength); + return new Response( + mode === 'wrapped' + ? new ReadableStream({ + type: 'bytes', + start(controller) { + controller.enqueue(payload); + controller.close(); + }, + }) + : payload, + { headers: { 'cf-r2-metadata-size': String(metadata.byteLength) } } + ); +} + +async function createBodyLengthStream(mode, ctx) { + if (mode === 'native') { + // A response made from a fixed-size buffer already knows how many bytes it contains. + return new Response(bodyLengthBytes).body; + } + if (mode === 'fixed') { + // Explicitly give the stream its byte count using Cloudflare's FixedLengthStream. + const fixed = new FixedLengthStream(bodyLengthBytes.byteLength); + ctx.waitUntil(new Response(bodyLengthBytes).body.pipeTo(fixed.writable)); + return fixed.readable; + } + if (mode === 'expected') { + // Cloudflare's nonstandard expectedLength option also gives a JS stream a byte count. + return new ReadableStream({ + type: 'bytes', + expectedLength: bodyLengthBytes.byteLength, + start(controller) { + controller.enqueue(bodyLengthBytes.slice()); + controller.close(); + }, + }); + } + assert.strictEqual(mode, 'wrapped'); + // Model the gateway's splitMetadataPrefix(): read the JSON, then forward the remaining + // bytes through a new JS stream. Reading size from the JSON does not give that new stream + // a known length. metadataSize counts only the JSON prefix, not the four object bytes. + const response = buildBodyLengthResponse('native'); + const metadataSize = Number(response.headers.get('cf-r2-metadata-size')); + const reader = response.body.getReader({ mode: 'byob' }); + const { value } = await reader.readAtLeast( + metadataSize, + new Uint8Array(metadataSize) + ); + assert.strictEqual( + JSON.parse(new TextDecoder().decode(value)).size, + String(bodyLengthBytes.byteLength) + ); + return new ReadableStream({ + type: 'bytes', + async pull(controller) { + const next = await reader.read(new Uint8Array(64 * 1024)); + if (!next.done) { + controller.enqueue(next.value); + } else { + reader.releaseLock(); + controller.close(); + } + }, + async cancel(reason) { + try { + await reader.cancel(reason); + } finally { + reader.releaseLock(); + } + }, + }); +} + +// A mock gateway serving both HTTP and RPC. Successful uploads must contain exactly "data", +// so the copy tests check the actual bytes, not just whether PUT returns successfully. +export class R2BodyLengthEntrypoint extends WorkerEntrypoint { + releaseBody() { + bodyLengthReleased = true; + } + + waitForCancellation() { + return bodyLengthCancellation; + } + + async fetch(request) { + if (request.method === 'GET') { + const metadata = JSON.parse(request.headers.get('cf-r2-request')); + assert.strictEqual(metadata.method, 'get'); + return buildBodyLengthResponse(metadata.object); + } + assert.strictEqual(request.method, 'PUT'); + const metadataSize = Number(request.headers.get('cf-r2-metadata-size')); + const payload = new Uint8Array(await request.arrayBuffer()); + const metadata = JSON.parse( + new TextDecoder().decode(payload.subarray(0, metadataSize)) + ); + assert.strictEqual(metadata.method, 'put'); + assert.strictEqual(metadata.object, 'destination'); + assert.deepStrictEqual(payload.subarray(metadataSize), bodyLengthBytes); + return Response.json({ + ...objResponse, + name: 'destination', + size: String(bodyLengthBytes.byteLength), + }); + } + + async get(mode, options) { + const object = { ...buildRpcHead(mode), size: bodyLengthBytes.byteLength }; + if (mode === 'missing') { + return null; + } + if (mode === 'metadata') { + return { kind: 'metadata', object }; + } + if (mode === 'deferred') { + bodyLengthReleased = false; + return { + kind: 'body', + object, + body: new ReadableStream({ + async pull(controller) { + // The test gateway waits in its own request context; releaseBody() only flips + // a flag. This timer models delayed upstream I/O, not a wait in the R2 caller. + while (!bodyLengthReleased) { + await scheduler.wait(1); + } + controller.enqueue(bodyLengthBytes.slice()); + controller.close(); + }, + }), + }; + } + if (mode === 'cancel' || mode === 'invalid-size') { + if (mode === 'invalid-size') { + object.size = -1; + } + let cancelled; + bodyLengthCancellation = new Promise((resolve) => { + cancelled = resolve; + }); + return { + kind: 'body', + object, + body: new ReadableStream({ + pull(controller) { + controller.enqueue(bodyLengthBytes.slice()); + }, + cancel() { + cancelled(); + }, + }), + }; + } + if (mode === 'upstream-error') { + return { + kind: 'body', + object, + body: new ReadableStream({ + pull(controller) { + controller.error(new Error('body source failed')); + }, + }), + }; + } + const invalidSizes = { short: 5, long: 3, 'zero-long': 0, unsafe: 2 ** 53 }; + if (Object.hasOwn(invalidSizes, mode)) { + object.size = invalidSizes[mode]; + return { kind: 'body', object, body: new Response(bodyLengthBytes).body }; + } + if (mode === 'unsafe-range') { + object.range = { offset: 0, length: 2 ** 53 }; + return { kind: 'body', object, body: new Response(bodyLengthBytes).body }; + } + if (Object.hasOwn(bodyLengthCopyCases, mode)) { + const fixture = bodyLengthCopyCases[mode]; + assert.deepStrictEqual(options?.range, fixture.options?.range); + object.size = fixture.size; + if (fixture.range !== undefined) { + object.range = fixture.range; + } + const bytes = new TextEncoder().encode(fixture.content); + let offset = 0; + return { + kind: 'body', + object, + body: new ReadableStream({ + type: 'bytes', + pull(controller) { + if (offset < bytes.byteLength) { + controller.enqueue(bytes.slice(offset, ++offset)); + } else { + controller.close(); + } + }, + }), + }; + } + return { + kind: 'body', + object, + body: await createBodyLengthStream(mode, this.ctx), + }; + } + + async put(requestKey, value, options, valueSize) { + const expected = bodyLengthUploadBytes(requestKey); + // workerd supplies this hidden RPC argument from the stream's known length. + assert.strictEqual(valueSize, expected.byteLength); + assert.deepStrictEqual( + new Uint8Array(await new Response(value).arrayBuffer()), + expected + ); + return { + ...buildRpcHead(requestKey), + key: requestKey, + size: expected.byteLength, + }; + } + + async uploadPart( + requestKey, + uploadId, + partNumber, + value, + options, + valueSize + ) { + assert.strictEqual(uploadId, 'body-length-upload'); + const expected = bodyLengthUploadBytes(requestKey); + assert.strictEqual(valueSize, expected.byteLength); + assert.deepStrictEqual( + new Uint8Array(await new Response(value).arrayBuffer()), + expected + ); + return { partNumber, etag: 'body-length-etag' }; + } +} + +async function assertBodyLengthCopy(bucket, source, knownLength, label) { + // knownLength is the expected test outcome, not information supplied to PUT. PUT receives + // only the stream and must discover its remaining length itself; it cannot use object.size. + // Start PUT before any await, so the "immediate" cases cannot accidentally let RPC settle. + const copying = bucket.put('destination', source); + if (knownLength) { + const result = await copying.catch((error) => { + error.message = `${label}: ${error.message}`; + throw error; + }); + assert.strictEqual(result.key, 'destination', label); + assert.strictEqual(result.size, bodyLengthBytes.byteLength, label); + } else { + // Ordinary unsized streams are still invalid PUT inputs. Service RPC can also return + // a sized stream before its receiving endpoint has finished setting up. + await assert.rejects( + copying, + { + name: 'TypeError', + message: + 'Provided readable stream must have a known length (request/response body or readable half of FixedLengthStream)', + }, + label + ); + } +} + +export const r2BodyLengthTests = { + async test(controller, env, ctx) { + // Only the dedicated body-length configuration runs these checks; the main R2 suites + // also import this file but do not provide this binding. + if (env.R2_BODY_LENGTH_TRANSPORT === undefined) { + return; + } + if (env.R2_BODY_LENGTH_TRANSPORT === 'http') { + // HTTP control: native GET -> PUT succeeds because the original stream keeps its + // length tracking after the JSON prefix is read. The wrapped version fails even + // though both results still expose the correct object.size. + for (const mode of ['native', 'wrapped']) { + const result = await env.BUCKET.get(mode); + assert.strictEqual(result.size, bodyLengthBytes.byteLength); + await assertBodyLengthCopy( + env.BUCKET, + result.body, + mode === 'native', + `HTTP ${mode}` + ); + } + return; + } + + // Repeat the same copy attempt with four sources, all containing the four bytes "data": + // - native: a Response body backed by a byte array, with a known length. + // - wrapped: a new JS stream forwarding bytes after the gateway reads the metadata prefix; + // the metadata says size=4, but this forwarding stream has no declared length. + // - fixed: the readable side of a FixedLengthStream(4). + // - expected: a JS byte stream with expectedLength=4. + // + // For each source, first try PUT with a locally created stream. Then get a fresh stream + // through each binding and try PUT either immediately or after one event-loop turn. + // This separates the source's own length from what the receiving RPC stream can report. + for (const mode of ['native', 'wrapped', 'fixed', 'expected']) { + // Create the body in the caller, without receiving it over RPC first. All three + // known-length sources copy successfully; the ordinary wrapped stream is rejected. + await assertBodyLengthCopy( + env.BUCKET, + await createBodyLengthStream(mode, ctx), + mode !== 'wrapped', + `local ${mode}` + ); + // Both bindings call the same fake gateway's get(mode) over RPC. Both receive the + // stream and metadata saying size=4, but they process that result differently: + // - BUCKET runs R2's getRpc(), which uses the metadata to create a local fixed-length + // wrapper. Even a "wrapped" source can therefore be copied immediately. + // - SERVICE returns the RPC result directly. Its stream's length depends on asynchronous + // stream setup; ordinary service RPC does not interpret the sibling object.size field. + // + // Expected PUT outcomes ("sized" means native, fixed, or expected): + // Receiving path / source Immediate PUT PUT after an event-loop turn + // BUCKET / any succeeds succeeds + // SERVICE, legacy / sized length error succeeds + // SERVICE, legacy / wrapped length error length error + // SERVICE, TypeScript / any length error length error + // + // The legacy receiver can discover a declared length once stream setup completes. + // The TypeScript receiver snapshots the initially unknown length, so waiting does not + // help it. The dedicated jsrpc-ts configuration runs this same loop with that receiver. + for (const transport of ['BUCKET', 'SERVICE']) { + for (const timing of ['immediate', 'after-event-loop-turn']) { + // Fetch again for every attempt: a stream consumed or cancelled by one PUT must + // not be reused by the next. Awaiting get() alone need not finish RPC stream setup. + const result = await env[transport].get(mode); + // Metadata is correct in every case, including cases whose stream is still unsized. + assert.strictEqual( + transport === 'BUCKET' ? result.size : result.object.size, + bodyLengthBytes.byteLength + ); + if (timing === 'after-event-loop-turn') { + // Let pending RPC setup run without consuming body bytes. This is a diagnostic + // event-loop yield, not a suggested workaround for application code. + await scheduler.wait(0); + } + await assertBodyLengthCopy( + env.BUCKET, + result.body, + // This boolean selects the expected outcome from the table above. All uploads + // use BUCKET.put(); transport selects only how we obtained the source body. + transport === 'BUCKET' || + (env.R2_BODY_LENGTH_TRANSPORT !== 'jsrpc-ts' && + timing === 'after-event-loop-turn' && + mode !== 'wrapped'), + `${transport} ${mode} ${timing}` + ); + } + } + } + + // The returned range describes the actual body, even when the requested range is longer. + // Empty bodies and multibyte UTF-8 text also need byte counts, not character counts. + for (const [requestKey, fixture] of Object.entries(bodyLengthCopyCases)) { + const result = await env.BUCKET.get(requestKey, fixture.options); + assert.strictEqual(result.size, fixture.size); + assert.strictEqual(result.range?.offset, fixture.range?.offset); + assert.strictEqual(result.range?.length, fixture.range?.length); + const copied = await env.BUCKET.put(requestKey, result.body); + assert.strictEqual( + copied.size, + new TextEncoder().encode(fixture.content).byteLength + ); + } + + // GET must finish before any bytes are available. Releasing the source only afterwards + // catches implementations that wait for the first byte or buffer the whole body. + const deferred = await env.BUCKET.get('deferred'); + assert.strictEqual(deferred.bodyUsed, false); + assert.strictEqual(deferred.body.locked, false); + const copying = env.BUCKET.put('destination', deferred.body); + await env.SERVICE.releaseBody(); + assert.strictEqual((await copying).size, bodyLengthBytes.byteLength); + + // A partial read must reduce the known length. Tee gives us an undisturbed stream for + // RPC transfer; its length must describe only the three unread bytes, not all four. + const partial = await env.BUCKET.get('chunks'); + const reader = partial.body.getReader({ mode: 'byob' }); + const first = await reader.read(new Uint8Array(1)); + assert.deepStrictEqual(first.value, bodyLengthBytes.slice(0, 1)); + reader.releaseLock(); + const [remaining, discarded] = partial.body.tee(); + const discarding = discarded.cancel(); + assert.strictEqual((await env.BUCKET.put('remaining', remaining)).size, 3); + await discarding; + + // Multipart uploads use the same known-length requirement as PUT, including ranged bodies. + for (const requestKey of ['destination', 'bounded']) { + const fixture = bodyLengthCopyCases[requestKey]; + const upload = env.BUCKET.resumeMultipartUpload( + requestKey, + 'body-length-upload' + ); + const source = await env.BUCKET.get( + requestKey === 'destination' ? 'wrapped' : requestKey, + fixture?.options + ); + assert.deepStrictEqual(await upload.uploadPart(1, source.body), { + partNumber: 1, + etag: 'body-length-etag', + }); + } + + // Metadata is a promise about the bytes, not permission to truncate or pad a bad response. + for (const mode of ['short', 'long', 'zero-long']) { + const result = await env.BUCKET.get(mode); + // Read to EOF explicitly: text() can return immediately when the declared length is zero. + const reader = result.body.getReader(); + await assert.rejects( + async () => { + let chunk; + do { + chunk = await reader.read(); + } while (!chunk.done); + }, + { name: 'TypeError' }, + mode + ); + } + await assert.rejects(env.BUCKET.get('unsafe'), { + message: /^internal error; reference = \S+$/, + }); + await assert.rejects(env.BUCKET.get('unsafe-range'), { + message: /^internal error; reference = \S+$/, + }); + + // Wrapping must preserve upstream errors and let consumer cancellation reach the gateway. + await assert.rejects( + async () => (await env.BUCKET.get('upstream-error')).text(), + { + message: 'ReadableStream received over RPC disconnected prematurely.', + } + ); + const cancelled = await env.BUCKET.get('cancel'); + await cancelled.body.cancel('no longer needed'); + await env.SERVICE.waitForCancellation(); + + // A rejected metadata result must cancel its body rather than leave the gateway pumping. + await assert.rejects(env.BUCKET.get('invalid-size'), { + message: /^internal error; reference = \S+$/, + }); + await env.SERVICE.waitForCancellation(); + + // Results without bodies do not need a length wrapper. + assert.strictEqual(await env.BUCKET.get('missing'), null); + assert.strictEqual((await env.BUCKET.get('metadata')).body, undefined); + + // R2 bodies must remain usable with an application's own streaming transforms. + const transformed = (await env.BUCKET.get('wrapped')).body.pipeThrough( + new FixedLengthStream(bodyLengthBytes.byteLength) + ); + assert.strictEqual(await new Response(transformed).text(), 'data'); + }, +}; + +export class R2JsrpcResponseEntrypoint extends WorkerEntrypoint { + head(key) { + if (key === 'rejected') throw new TypeError('gateway failure'); + assert(Object.hasOwn(invalidMetadata, key)); + return { ...object, ...invalidMetadata[key] }; + } + + get(key) { + switch (key) { + case 'unknownKind': + return { kind: 'other', object }; + case 'metadataWithBody': + return { kind: 'metadata', object, body: new ReadableStream() }; + case 'missingBody': + return { kind: 'body', object }; + case 'invalidBody': + return { kind: 'body', object, body: 42 }; + case 'unsafeLength': + return { + kind: 'body', + object: { ...object, size: 2 ** 53 }, + body: new ReadableStream(), + }; + default: + assert.fail('Invalid user input must not reach the gateway'); + } + } + + put() { + return { ...object, ...invalidMetadata.invalidDate }; + } + + list() { + return { + objects: [{ ...object, ...invalidMetadata.invalidDate }], + truncated: false, + delimitedPrefixes: [], + }; + } + + completeMultipartUpload() { + return { ...object, ...invalidMetadata.invalidDate }; + } +} + +export async function testInvalidR2Responses(_controller, env) { + const internalError = { message: /^internal error; reference = \S+$/ }; + for (const key of Object.keys(invalidMetadata)) { + await assert.rejects(env.BUCKET.head(key), internalError, key); + } + for (const key of [ + 'unknownKind', + 'metadataWithBody', + 'missingBody', + 'invalidBody', + 'unsafeLength', + ]) { + await assert.rejects(env.BUCKET.get(key), internalError, key); + } + await assert.rejects(env.BUCKET.put('key', 'body'), internalError); + await assert.rejects(env.BUCKET.list(), internalError); + const upload = env.BUCKET.resumeMultipartUpload('key', 'uploadId'); + await assert.rejects(upload.complete([]), internalError); + + // A rejected gateway call and invalid user input retain their public error messages. + await assert.rejects(env.BUCKET.head('rejected'), { + name: 'TypeError', + message: 'gateway failure', + }); + await assert.rejects( + env.BUCKET.get('invalidInput', { range: { offset: -1 } }), + { + name: 'RangeError', + message: + 'Invalid range. Starting offset (-1) must be greater than or equal to 0.', + } + ); +} + +export default testWorker; diff --git a/src/workerd/api/tests/r2-test.wd-test b/src/workerd/api/tests/r2-test.wd-test index 169c069cefa..916a4d99c46 100644 --- a/src/workerd/api/tests/r2-test.wd-test +++ b/src/workerd/api/tests/r2-test.wd-test @@ -10,6 +10,7 @@ const unitTests :Workerd.Config = ( bindings = [ ( name = "BUCKET", r2Bucket = "r2-test" ), ( name = "SERVICE", service = "r2-test" ), + ( name = "R2_TRANSPORT", text = "http" ), ], compatibilityFlags = ["nodejs_compat", "service_binding_extra_handlers", "streams_enable_constructors", "r2_list_honor_include", "disable_fast_jsg_struct"], streamingTails = ["tail"], @@ -28,5 +29,8 @@ const tailWorker :Workerd.Worker = ( modules = [ (name = "worker", esModule = embed "r2-instrumentation-test.js") ], + bindings = [ + (name = "R2_TRACE_TRANSPORT", text = "http"), + ], compatibilityFlags = ["experimental", "nodejs_compat", "disable_fast_jsg_struct"], ); diff --git a/src/workerd/io/compatibility-date.capnp b/src/workerd/io/compatibility-date.capnp index 81904ebb86a..ede43238976 100644 --- a/src/workerd/io/compatibility-date.capnp +++ b/src/workerd/io/compatibility-date.capnp @@ -1761,4 +1761,11 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef { # A compiled `WebAssembly.Module` passed to the Worker Loader is recompiled when the loaded Worker # or the Worker that compiled it has this flag, so its frames follow the loaded Worker's setting # and module name. + + r2BindingsJsrpc @200 :Bool + $compatEnableFlag("r2_binding_jsrpc") + $experimental; + # When enabled, R2 bindings dispatch to the R2 gateway worker over JSRPC + # instead of synthesising an HTTP request and calling `fetch`. Without this + # flag, R2 bindings continue to use the HTTP transport. } diff --git a/src/workerd/io/worker-interface.capnp b/src/workerd/io/worker-interface.capnp index 659fa0977b2..5df2b633fab 100644 --- a/src/workerd/io/worker-interface.capnp +++ b/src/workerd/io/worker-interface.capnp @@ -553,6 +553,18 @@ enum SerializationTag { # payload (per `serviceStub`) followed by the wrapper module name. See api/wrapped-binding.{h,c++}. durableObjectSnapshot @18; # A Durable Object storage snapshot handle transferred over RPC. + + r2Checksums @19; + # Checksums attached to an R2 object. + + r2Object @20; + # R2 object metadata without a body. + + r2ObjectBody @21; + # R2 object metadata with a transferred body stream. + + r2Error @22; + # A structured error produced by an R2 operation. } enum StreamEncoding { From c44df4a953b21d72cd119ff0e45294e3ad201ea0 Mon Sep 17 00:00:00 2001 From: Carol Xu Date: Mon, 28 Sep 2026 10:40:37 -0500 Subject: [PATCH 02/13] refactor tests to use ObservedR2Binding --- src/workerd/api/tests/BUILD.bazel | 17 +- .../api/tests/r2-instrumentation-test.js | 11 +- .../r2-jsrpc-gateway-contract-test.wd-test | 29 + .../tests/r2-jsrpc-legacy-list-test.wd-test | 28 +- .../api/tests/r2-jsrpc-multipart-test.wd-test | 40 +- .../api/tests/r2-jsrpc-passthrough.capnp | 64 + .../api/tests/r2-jsrpc-range-test.wd-test | 4 +- .../api/tests/r2-jsrpc-serialization-test.js | 62 +- .../tests/r2-jsrpc-serialization-test.wd-test | 33 +- src/workerd/api/tests/r2-jsrpc-test.wd-test | 27 +- src/workerd/api/tests/r2-test.js | 1212 +++++++++-------- src/workerd/api/tests/r2-test.wd-test | 1 + 12 files changed, 899 insertions(+), 629 deletions(-) create mode 100644 src/workerd/api/tests/r2-jsrpc-gateway-contract-test.wd-test create mode 100644 src/workerd/api/tests/r2-jsrpc-passthrough.capnp diff --git a/src/workerd/api/tests/BUILD.bazel b/src/workerd/api/tests/BUILD.bazel index fcbfedfa42f..cca42de3e48 100644 --- a/src/workerd/api/tests/BUILD.bazel +++ b/src/workerd/api/tests/BUILD.bazel @@ -564,7 +564,10 @@ wd_test( wd_test( src = "r2-jsrpc-multipart-test.wd-test", args = ["--experimental"], - data = ["r2-test.js"], + data = [ + "r2-jsrpc-passthrough.capnp", + "r2-test.js", + ], ) wd_test( @@ -594,14 +597,24 @@ wd_test( data = [ "instrumentation-tail-worker.js", "r2-instrumentation-test.js", + "r2-jsrpc-passthrough.capnp", "r2-test.js", ], ) wd_test( - src = "r2-jsrpc-legacy-list-test.wd-test", + src = "r2-jsrpc-gateway-contract-test.wd-test", args = ["--experimental"], data = ["r2-test.js"], +) + +wd_test( + src = "r2-jsrpc-legacy-list-test.wd-test", + args = ["--experimental"], + data = [ + "r2-jsrpc-passthrough.capnp", + "r2-test.js", + ], generate_all_compat_flags_variant = False, ) diff --git a/src/workerd/api/tests/r2-instrumentation-test.js b/src/workerd/api/tests/r2-instrumentation-test.js index 43936219dab..312ae91b579 100644 --- a/src/workerd/api/tests/r2-instrumentation-test.js +++ b/src/workerd/api/tests/r2-instrumentation-test.js @@ -1194,6 +1194,8 @@ export const test = { }, ]; + // The JSRPC transport also records a jsRpcSession span per call and does not set + // cloudflare.r2.response.success, so only the HTTP transport is compared exactly. const projectSharedTags = (span) => Object.fromEntries( Object.entries(span).filter( @@ -1201,15 +1203,16 @@ export const test = { ) ); + // The expected spans come from the default test and deletePerKeyErrorParityTests, which run + // before the other test exports. Spans from those later exports are not checked. const sharedReceived = received .filter((span) => span['cloudflare.binding.type'] === 'r2') - // The JSRPC target runs transport-specific tests after the canonical suite. - .slice(0, expected.length) - .map(projectSharedTags); + .map(projectSharedTags) + .slice(0, expected.length); assert.deepStrictEqual(sharedReceived, expected.map(projectSharedTags)); if (env.R2_TRACE_TRANSPORT === 'http') { - assert.deepStrictEqual(received, expected); + assert.deepStrictEqual(received.slice(0, expected.length), expected); } }, }; diff --git a/src/workerd/api/tests/r2-jsrpc-gateway-contract-test.wd-test b/src/workerd/api/tests/r2-jsrpc-gateway-contract-test.wd-test new file mode 100644 index 00000000000..2a203700055 --- /dev/null +++ b/src/workerd/api/tests/r2-jsrpc-gateway-contract-test.wd-test @@ -0,0 +1,29 @@ +using Workerd = import "/workerd/workerd.capnp"; + +# Checks the R2 binding's JSRPC client against R2BindingEntrypoint, a fake gateway that asserts +# the arguments each method receives and returns results a native R2 binding cannot produce, such +# as malformed objects. r2-jsrpc-test.wd-test runs the canonical and shared suites through a +# passthrough to a native binding. +const unitTests :Workerd.Config = ( + services = [ + ( name = "r2-jsrpc-gateway-contract-test", + worker = ( + modules = [ + ( name = "worker", + esModule = "export { R2BindingEntrypoint, jsrpcGatewayContractTests } from './r2-test.js';" ), + ( name = "r2-test.js", esModule = embed "r2-test.js" ), + ], + bindings = [ + ( name = "BUCKET", + r2Bucket = (name = "r2-jsrpc-gateway-contract-test", + entrypoint = "R2BindingEntrypoint") ), + ( name = "R2_TRANSPORT", text = "jsrpc" ), + ( name = "R2_LIST_HONOR_INCLUDE", text = "true" ), + ], + compatibilityFlags = ["experimental", "nodejs_compat", "service_binding_extra_handlers", + "streams_enable_constructors", "r2_list_honor_include", + "r2_binding_jsrpc", "disable_fast_jsg_struct"], + ) + ), + ], +); diff --git a/src/workerd/api/tests/r2-jsrpc-legacy-list-test.wd-test b/src/workerd/api/tests/r2-jsrpc-legacy-list-test.wd-test index 280225de3a4..5f168e24750 100644 --- a/src/workerd/api/tests/r2-jsrpc-legacy-list-test.wd-test +++ b/src/workerd/api/tests/r2-jsrpc-legacy-list-test.wd-test @@ -1,17 +1,39 @@ using Workerd = import "/workerd/workerd.capnp"; +using Passthrough = import "r2-jsrpc-passthrough.capnp"; # Compatibility-sensitive coverage for list() before r2_list_honor_include. This lives separately # because a future compatibility date necessarily enables the behavior under test. +# +# The canonical and shared suites run through the passthrough in r2-jsrpc-passthrough.capnp, with +# the flag disabled for both the JSRPC client and the native binding. The gateway service checks +# the include list that the JSRPC client sends without the flag. const unitTests :Workerd.Config = ( services = [ - ( name = "r2-jsrpc-legacy-list-test", + ( name = "user", + worker = ( + modules = Passthrough.userModules, + bindings = [ + ( name = "BUCKET", + r2Bucket = (name = "r2-test", entrypoint = "ObservedR2Binding") ), + ( name = "R2_TRANSPORT", text = "jsrpc" ), + ( name = "R2_LIST_HONOR_INCLUDE", text = "false" ), + ], + compatibilityFlags = ["experimental", "nodejs_compat", "service_binding_extra_handlers", + "streams_enable_constructors", "r2_binding_jsrpc", + "disable_fast_jsg_struct"], + ) + ), + ( name = "r2-test", worker = Passthrough.passthroughLegacyListWorker ), + ( name = "gateway", worker = ( modules = [ - ( name = "worker", esModule = embed "r2-test.js" ) + ( name = "worker", + esModule = "export { R2BindingEntrypoint, jsrpcGatewayContractTests } from './r2-test.js';" ), + ( name = "r2-test.js", esModule = embed "r2-test.js" ), ], bindings = [ ( name = "BUCKET", - r2Bucket = (name = "r2-jsrpc-legacy-list-test", entrypoint = "R2BindingEntrypoint") ), + r2Bucket = (name = "gateway", entrypoint = "R2BindingEntrypoint") ), ( name = "R2_TRANSPORT", text = "jsrpc" ), ( name = "R2_LIST_HONOR_INCLUDE", text = "false" ), ], diff --git a/src/workerd/api/tests/r2-jsrpc-multipart-test.wd-test b/src/workerd/api/tests/r2-jsrpc-multipart-test.wd-test index 6849030515d..e4feb6860df 100644 --- a/src/workerd/api/tests/r2-jsrpc-multipart-test.wd-test +++ b/src/workerd/api/tests/r2-jsrpc-multipart-test.wd-test @@ -1,30 +1,49 @@ using Workerd = import "/workerd/workerd.capnp"; +using Passthrough = import "r2-jsrpc-passthrough.capnp"; -# Exercise the bucket multipart RPC contract with both struct-wrapping modes. +# Runs the multipart suites with fast structs; r2-jsrpc-test.wd-test covers slow structs for the +# shared suite. +# +# The fast-structs service calls the passthrough in r2-jsrpc-passthrough.capnp. +# +# The gateway contract suite targets R2BindingEntrypoint in both struct-wrapping modes, because it +# asserts the exact arguments the JSRPC client sends. const unitTests :Workerd.Config = ( services = [ ( name = "fast-structs", worker = ( modules = [ - (name = "worker", esModule = "export { R2BindingEntrypoint, jsrpcMultipartTests } from './r2-test.js';"), + (name = "worker", esModule = "export { r2MultipartApiTests } from './r2-test.js';"), (name = "r2-test.js", esModule = embed "r2-test.js"), ], bindings = [ - (name = "BUCKET", r2Bucket = (name = "fast-structs", entrypoint = "R2BindingEntrypoint")), + (name = "BUCKET", + r2Bucket = (name = "r2-test", entrypoint = "ObservedR2Binding")), (name = "R2_TRANSPORT", text = "jsrpc"), ], compatibilityFlags = ["experimental", "nodejs_compat", "service_binding_extra_handlers", "streams_enable_constructors", "r2_binding_jsrpc", "enable_fast_jsg_struct"], ), ), - ( name = "slow-structs", + ( name = "r2-test", worker = Passthrough.passthroughFastStructsWorker ), + ( name = "gateway-fast-structs", worker = ( - modules = [ - (name = "worker", esModule = "export { R2BindingEntrypoint, jsrpcMultipartTests } from './r2-test.js';"), - (name = "r2-test.js", esModule = embed "r2-test.js"), + modules = .gatewayModules, + bindings = [ + (name = "BUCKET", + r2Bucket = (name = "gateway-fast-structs", entrypoint = "R2BindingEntrypoint")), + (name = "R2_TRANSPORT", text = "jsrpc"), ], + compatibilityFlags = ["experimental", "nodejs_compat", "service_binding_extra_handlers", + "streams_enable_constructors", "r2_binding_jsrpc", "enable_fast_jsg_struct"], + ), + ), + ( name = "gateway-slow-structs", + worker = ( + modules = .gatewayModules, bindings = [ - (name = "BUCKET", r2Bucket = (name = "slow-structs", entrypoint = "R2BindingEntrypoint")), + (name = "BUCKET", + r2Bucket = (name = "gateway-slow-structs", entrypoint = "R2BindingEntrypoint")), (name = "R2_TRANSPORT", text = "jsrpc"), ], compatibilityFlags = ["experimental", "nodejs_compat", "service_binding_extra_handlers", @@ -33,3 +52,8 @@ const unitTests :Workerd.Config = ( ), ], ); + +const gatewayModules :List(Workerd.Worker.Module) = [ + (name = "worker", esModule = "export { R2BindingEntrypoint, jsrpcGatewayMultipartContractTests } from './r2-test.js';"), + (name = "r2-test.js", esModule = embed "r2-test.js"), +]; diff --git a/src/workerd/api/tests/r2-jsrpc-passthrough.capnp b/src/workerd/api/tests/r2-jsrpc-passthrough.capnp new file mode 100644 index 00000000000..c44e1447e96 --- /dev/null +++ b/src/workerd/api/tests/r2-jsrpc-passthrough.capnp @@ -0,0 +1,64 @@ +# Shared pieces of the R2 JSRPC passthrough topology, imported by the R2 JSRPC configs. This file +# is not a test config itself. embed paths resolve relative to this file. +# +# user --JSRPC--> r2-test (ObservedR2Binding) --HTTP via native REAL_BUCKET--> r2-test (fetch) +# +# The r2-test service runs ObservedR2Binding without r2_binding_jsrpc, so its native REAL_BUCKET +# uses the HTTP protocol. REAL_BUCKET points back at the same service, whose default export is +# the HTTP mock from r2-test.js. Importing configs must therefore name the service "r2-test", +# which is also the bucket name that R2 spans record. +# +# Each worker variant matches the mock's R2_LIST_HONOR_INCLUDE expectation to its +# r2_list_honor_include flag. + +@0xe8a01684ab8486bc; + +using Workerd = import "/workerd/workerd.capnp"; + +# Modules for a user Worker that runs the canonical and shared suites. +const userModules :List(Workerd.Worker.Module) = [ + ( name = "worker", + esModule = "export { default, deletePerKeyErrorParityTests, r2ValidationTests, r2BindingApiTests, r2MultipartApiTests } from './r2-test.js';" ), + ( name = "r2-test.js", esModule = embed "r2-test.js" ), +]; + +# Exports ObservedR2Binding and the HTTP mock, but no test handlers, so the test runner does not +# run the tests exported by r2-test.js in this service. +const passthroughModules :List(Workerd.Worker.Module) = [ + ( name = "worker", + esModule = "import mock, { ObservedR2Binding } from './r2-test.js'; export { ObservedR2Binding }; export default { fetch: mock.fetch };" ), + ( name = "r2-test.js", esModule = embed "r2-test.js" ), +]; + +# r2_list_honor_include with slow structs. +const passthroughWorker :Workerd.Worker = ( + modules = .passthroughModules, + bindings = [ + ( name = "REAL_BUCKET", r2Bucket = "r2-test" ), + ( name = "R2_LIST_HONOR_INCLUDE", text = "true" ), + ], + compatibilityFlags = ["nodejs_compat", "streams_enable_constructors", + "r2_list_honor_include", "disable_fast_jsg_struct"], +); + +# No r2_list_honor_include, with slow structs. +const passthroughLegacyListWorker :Workerd.Worker = ( + modules = .passthroughModules, + bindings = [ + ( name = "REAL_BUCKET", r2Bucket = "r2-test" ), + ( name = "R2_LIST_HONOR_INCLUDE", text = "false" ), + ], + compatibilityFlags = ["nodejs_compat", "streams_enable_constructors", + "disable_fast_jsg_struct"], +); + +# No r2_list_honor_include, with fast structs. +const passthroughFastStructsWorker :Workerd.Worker = ( + modules = .passthroughModules, + bindings = [ + ( name = "REAL_BUCKET", r2Bucket = "r2-test" ), + ( name = "R2_LIST_HONOR_INCLUDE", text = "false" ), + ], + compatibilityFlags = ["nodejs_compat", "streams_enable_constructors", + "enable_fast_jsg_struct"], +); diff --git a/src/workerd/api/tests/r2-jsrpc-range-test.wd-test b/src/workerd/api/tests/r2-jsrpc-range-test.wd-test index 9ee470ef40e..1834dbd9ca6 100644 --- a/src/workerd/api/tests/r2-jsrpc-range-test.wd-test +++ b/src/workerd/api/tests/r2-jsrpc-range-test.wd-test @@ -8,7 +8,7 @@ const unitTests :Workerd.Config = ( ( name = "fast-structs", worker = ( modules = [ - (name = "worker", esModule = "export { R2BindingEntrypoint, jsrpcRangeTests } from './r2-test.js';"), + (name = "worker", esModule = "export { R2BindingEntrypoint, jsrpcGatewayRangeContractTests } from './r2-test.js';"), (name = "r2-test.js", esModule = embed "r2-test.js"), ], bindings = [ @@ -22,7 +22,7 @@ const unitTests :Workerd.Config = ( ( name = "slow-structs", worker = ( modules = [ - (name = "worker", esModule = "export { R2BindingEntrypoint, jsrpcRangeTests } from './r2-test.js';"), + (name = "worker", esModule = "export { R2BindingEntrypoint, jsrpcGatewayRangeContractTests } from './r2-test.js';"), (name = "r2-test.js", esModule = embed "r2-test.js"), ], bindings = [ diff --git a/src/workerd/api/tests/r2-jsrpc-serialization-test.js b/src/workerd/api/tests/r2-jsrpc-serialization-test.js index 1cab6500300..8417e346819 100644 --- a/src/workerd/api/tests/r2-jsrpc-serialization-test.js +++ b/src/workerd/api/tests/r2-jsrpc-serialization-test.js @@ -3,75 +3,43 @@ // https://opensource.org/licenses/Apache-2.0 import assert from 'node:assert'; -import { WorkerEntrypoint } from 'cloudflare:workers'; - -export class ObservedR2Binding extends WorkerEntrypoint { - fetch() { - throw new Error('R2 platform binding tests must not use HTTP'); - } - - async head(...args) { - if (args[0] === 'body-as-head') { +import { ObservedR2Binding as PassthroughR2Binding } from './r2-test.js'; + +// Extends the shared passthrough so that some keys return a native R2 result of the wrong type +// for the method, and adds methods that return native results to the test directly, without the +// R2 binding's JSRPC client in between. +export class ObservedR2Binding extends PassthroughR2Binding { + async head(requestKey) { + if (requestKey === 'body-as-head') { return this.env.REAL_BUCKET.get('rpc-json'); } - if (args[0] === 'list-object-as-head') { + if (requestKey === 'list-object-as-head') { const listed = await this.env.REAL_BUCKET.list({ prefix: 'httpMeta', include: [], }); return listed.objects[0]; } - return this.env.REAL_BUCKET.head(...args); + return super.head(requestKey); } - async get(...args) { - if (args[0] === 'list-object-as-get') { + async get(requestKey, options) { + if (requestKey === 'list-object-as-get') { const listed = await this.env.REAL_BUCKET.list({ prefix: 'httpMeta', include: [], }); return listed.objects[0]; } - return this.env.REAL_BUCKET.get(...args); - } - - put(...args) { - return this.env.REAL_BUCKET.put(...args); - } - - list(...args) { - return this.env.REAL_BUCKET.list(...args); + return super.get(requestKey, options); } async checksums() { return (await this.env.REAL_BUCKET.head('multipleChecksums')).checksums; } - getNative(key) { - return this.env.REAL_BUCKET.get(key); - } - - async createMultipartUpload(...args) { - const upload = await this.env.REAL_BUCKET.createMultipartUpload(...args); - return { key: upload.key, uploadId: upload.uploadId }; - } - - uploadPart(key, uploadId, partNumber, value, options) { - return this.env.REAL_BUCKET.resumeMultipartUpload(key, uploadId).uploadPart( - partNumber, - value, - options - ); - } - - abortMultipartUpload(key, uploadId) { - return this.env.REAL_BUCKET.resumeMultipartUpload(key, uploadId).abort(); - } - - completeMultipartUpload(key, uploadId, uploadedParts) { - return this.env.REAL_BUCKET.resumeMultipartUpload(key, uploadId).complete( - uploadedParts - ); + getNative(requestKey) { + return this.env.REAL_BUCKET.get(requestKey); } } diff --git a/src/workerd/api/tests/r2-jsrpc-serialization-test.wd-test b/src/workerd/api/tests/r2-jsrpc-serialization-test.wd-test index 578799f61b0..6300b136397 100644 --- a/src/workerd/api/tests/r2-jsrpc-serialization-test.wd-test +++ b/src/workerd/api/tests/r2-jsrpc-serialization-test.wd-test @@ -1,14 +1,21 @@ using Workerd = import "/workerd/workerd.capnp"; -# The user-facing R2 binding targets a platform Worker. That Worker calls a separate real R2 -# binding whose fetcher targets the mock R2GW entrypoint. Keeping this topology in a dedicated -# config prevents the native-resource round trips from being conflated with the HTTP transport. +# The user-facing R2 binding uses JSRPC to call a platform Worker. That Worker lacks +# r2_binding_jsrpc, so its real R2 binding reaches the HTTP mock in r2-test.js over HTTP, and +# returns the resulting native R2 objects across the JSRPC hop. Keeping this topology in a +# dedicated config prevents the native-resource round trips from being conflated with the HTTP +# transport. +# +# The platform Worker also serves the HTTP mock: its REAL_BUCKET points back at itself, as in +# r2-jsrpc-passthrough.capnp. It uses its own modules because it exports the ObservedR2Binding +# subclass from r2-jsrpc-serialization-test.js, which adds methods for this test. const unitTests :Workerd.Config = ( services = [ (name = "user", worker = ( modules = [ (name = "worker", esModule = "export { nativeR2ResultTests } from './test.js';"), (name = "test.js", esModule = embed "r2-jsrpc-serialization-test.js"), + (name = "r2-test.js", esModule = embed "r2-test.js"), ], bindings = [ (name = "BUCKET", @@ -20,27 +27,19 @@ const unitTests :Workerd.Config = ( "r2_list_honor_include", "r2_binding_jsrpc", "rpc", "fetcher_no_get_put_delete"], )), + # Exports no test handlers, so the test runner does not run the tests in r2-test.js here. (name = "platform", worker = ( modules = [ - (name = "worker", esModule = "export { ObservedR2Binding } from './test.js';"), + (name = "worker", + esModule = "import mock from './r2-test.js'; export { ObservedR2Binding } from './test.js'; export default { fetch: mock.fetch };"), (name = "test.js", esModule = embed "r2-jsrpc-serialization-test.js"), - ], - bindings = [ - (name = "REAL_BUCKET", - r2Bucket = (name = "gateway", entrypoint = "R2BindingEntrypoint")), - ], - compatibilityFlags = ["experimental", "nodejs_compat", "streams_enable_constructors", - "r2_list_honor_include", "r2_binding_jsrpc"], - )), - (name = "gateway", worker = ( - modules = [ - (name = "worker", esModule = "export { R2BindingEntrypoint } from './r2-test.js';"), (name = "r2-test.js", esModule = embed "r2-test.js"), ], bindings = [ - (name = "R2_LIST_HONOR_INCLUDE", text = "true"), + (name = "REAL_BUCKET", r2Bucket = "platform"), ], - compatibilityFlags = ["nodejs_compat", "streams_enable_constructors"], + compatibilityFlags = ["nodejs_compat", "streams_enable_constructors", + "r2_list_honor_include"], )), ], ); diff --git a/src/workerd/api/tests/r2-jsrpc-test.wd-test b/src/workerd/api/tests/r2-jsrpc-test.wd-test index 6b70dc8cca7..c83a57092d2 100644 --- a/src/workerd/api/tests/r2-jsrpc-test.wd-test +++ b/src/workerd/api/tests/r2-jsrpc-test.wd-test @@ -1,25 +1,29 @@ using Workerd = import "/workerd/workerd.capnp"; +using Passthrough = import "r2-jsrpc-passthrough.capnp"; -# Runs the canonical R2 binding suite (r2-test.js) with the JSRPC transport enabled. -# The named entrypoint rejects every HTTP request to detect any transport fallback. +# Runs the canonical R2 binding suite (r2-test.js) and the shared suites with the JSRPC transport +# enabled, through the passthrough in r2-jsrpc-passthrough.capnp. That passthrough forwards each +# call to a native R2 binding that reaches the same HTTP mock as r2-test.wd-test, so the suites +# check that native R2 results and errors survive the JSRPC hop with the same observable behavior +# as the HTTP transport. # # Reaching the JSRPC method table needs two things to line up: # - the r2_binding_jsrpc compatibility flag, which is $experimental and so also needs # "experimental" in the flag list here -# - a binding that targets the named entrypoint the gateway exposes +# - a binding that targets the named entrypoint, which is the production shape; a bare-string +# designator would target the default export instead # -# The `entrypoint` below is the point of this file. A bare-string designator would target the -# default export and would not exercise named-entrypoint dispatch, which is the production shape. +# Tests that assert the gateway's wire shapes or malformed results need a fake gateway rather +# than a passthrough, so they run in the configs that target R2BindingEntrypoint +# (r2-jsrpc-gateway-contract-test.wd-test and the multipart and range configs). const unitTests :Workerd.Config = ( services = [ - ( name = "r2-test", + ( name = "user", worker = ( - modules = [ - ( name = "worker", esModule = embed "r2-test.js" ) - ], + modules = Passthrough.userModules, bindings = [ ( name = "BUCKET", - r2Bucket = (name = "r2-test", entrypoint = "R2BindingEntrypoint") ), + r2Bucket = (name = "r2-test", entrypoint = "ObservedR2Binding") ), ( name = "R2_TRANSPORT", text = "jsrpc" ), ( name = "R2_LIST_HONOR_INCLUDE", text = "true" ), ], @@ -29,7 +33,8 @@ const unitTests :Workerd.Config = ( streamingTails = ["tail"], ) ), - (name = "tail", worker = .tailWorker), + ( name = "r2-test", worker = Passthrough.passthroughWorker ), + ( name = "tail", worker = .tailWorker ), ], extensions = [( modules = [ diff --git a/src/workerd/api/tests/r2-test.js b/src/workerd/api/tests/r2-test.js index 405968c7ddd..b7786c0072a 100644 --- a/src/workerd/api/tests/r2-test.js +++ b/src/workerd/api/tests/r2-test.js @@ -199,7 +199,101 @@ async function assertLargeRpcBody(requestKey, value, valueSize) { } } -function buildGetResponse({ head, body, isList } = {}) { +// The HTTP protocol reports failures with a status code and a CF-R2-Error header, which the +// binding turns into an Error whose message is ": ()". +function r2ErrorHeaders({ v4code, message }) { + return { 'cf-r2-error': JSON.stringify({ version: 1, v4code, message }) }; +} + +function r2ErrorResponse(error) { + return new Response(null, { + status: error.status, + headers: r2ErrorHeaders(error), + }); +} + +const noSuchKeyError = { + status: 404, + v4code: 10007, + message: 'The specified key does not exist.', +}; +const noSuchBucketError = { + status: 404, + v4code: 10006, + message: 'no such bucket', +}; +const preconditionFailedError = { + status: 412, + v4code: 10031, + message: 'At least one of the pre-conditions you specified did not hold.', +}; +const uploadNotFoundError = { + status: 404, + v4code: 10024, + message: 'Upload not found', +}; + +// Multipart uploads for keys starting with 'rpc-multipart-' are tracked so that the mock can +// report uploads that were never created, or were already completed or aborted. The first upload +// for a key gets the ID '-id', and later uploads for the same key get '-id-'. +const activeMultipartUploads = new Map(); +const multipartUploadCounts = new Map(); + +function handleTrackedMultipartRequest(jsonRequest, uploadedBody) { + const requestKey = jsonRequest.object; + const { uploadId } = jsonRequest; + const isActive = activeMultipartUploads.get(uploadId) === requestKey; + switch (jsonRequest.method) { + case 'createMultipartUpload': { + // Metadata is captured when createMultipartUpload() is called, so later changes to the + // caller's objects are not sent. + if (jsonRequest.customFields !== undefined) { + assert.deepStrictEqual(jsonRequest.customFields, [ + { k: 'uploadKey', v: requestKey }, + ]); + assert.deepStrictEqual(jsonRequest.httpFields, httpFields); + } + const count = (multipartUploadCounts.get(requestKey) ?? 0) + 1; + multipartUploadCounts.set(requestKey, count); + const newUploadId = + count === 1 ? `${requestKey}-id` : `${requestKey}-id-${count}`; + activeMultipartUploads.set(newUploadId, requestKey); + return Response.json({ uploadId: newUploadId }); + } + case 'uploadPart': + if (!isActive) { + return r2ErrorResponse(uploadNotFoundError); + } + assert.strictEqual(uploadedBody, body); + return Response.json({ + etag: `${requestKey}/${uploadId}/${jsonRequest.partNumber}`, + }); + case 'completeMultipartUpload': + if (!isActive) { + return r2ErrorResponse(uploadNotFoundError); + } + for (const { part, etag } of jsonRequest.parts ?? []) { + assert.strictEqual(etag, `${requestKey}/${uploadId}/${part}`); + } + activeMultipartUploads.delete(uploadId); + return Response.json({ + ...objResponse, + name: requestKey, + version: uploadId, + }); + case 'abortMultipartUpload': + // Aborting an upload that no longer exists succeeds. + if (isActive) { + activeMultipartUploads.delete(uploadId); + } + return new Response(); + } + throw new Error(`Unexpected method: ${jsonRequest.method}`); +} + +// An R2 error makes this an error response, such as a failed precondition that still carries the +// object's metadata but no body. +function buildGetResponse({ head, body, isList, error } = {}) { const encoder = new TextEncoder(); let meta; if (!isList) { @@ -221,11 +315,16 @@ function buildGetResponse({ head, body, isList } = {}) { }, }) : metadata; + const headers = { + 'cf-r2-metadata-size': metadata.length.toString(), + 'content-length': metadata.length.toString(), + }; + if (error === undefined) { + return new Response(responseBody, { headers }); + } return new Response(responseBody, { - headers: { - 'cf-r2-metadata-size': metadata.length.toString(), - 'content-length': metadata.length.toString(), - }, + status: error.status, + headers: { ...headers, ...r2ErrorHeaders(error) }, }); } @@ -281,18 +380,24 @@ const testWorker = { assert.fail('R2 backend must not be called for invalid input'); } - // The rest of the body is the object payload. - const decoder = new TextDecoder(); - let payload = ''; + // The metadata has already been read from the body, so a Response cannot wrap it. + const chunks = []; for await (const chunk of request.body) { - payload += decoder.decode(chunk, { stream: true }); + chunks.push(chunk); } - payload += decoder.decode(); + const uploadedBody = await new Blob(chunks).text(); // Assert it's the correct version assert.strictEqual(jsonRequest.version, 1); if (jsonRequest.method === 'delete') { + if (jsonRequest.object === 'boom') { + return r2ErrorResponse({ + status: 400, + v4code: 10021, + message: 'bad keys', + }); + } if (jsonRequest.objects) { // A batch delete can report individual key failures in a successful HTTP response. // R2Bucket::delete_() reads this body from the legacy transport but does not expose it. @@ -320,8 +425,18 @@ const testWorker = { return new Response(); } + if (jsonRequest.object.startsWith('rpc-multipart-')) { + return handleTrackedMultipartRequest(jsonRequest, uploadedBody); + } + switch (jsonRequest.object) { case 'basicKey': { + if (jsonRequest.uploadId !== undefined) { + assert( + ['multipartId', 'resumedId'].includes(jsonRequest.uploadId) + ); + } + const resumed = jsonRequest.uploadId === 'resumedId'; switch (jsonRequest.method) { case 'put': { break; @@ -333,18 +448,38 @@ const testWorker = { } case 'uploadPart': { return Response.json({ - etag: 'partEtag', + etag: resumed ? 'resumedRpcPartEtag' : 'partEtag', }); } case 'abortMultipartUpload': { return new Response(); } case 'completeMultipartUpload': { - return Response.json(objResponse); + return Response.json( + resumed + ? { ...objResponse, version: 'resumedRpcObjectVersion' } + : objResponse + ); } } break; } + case 'rpcStream': { + assert.strictEqual(uploadedBody, rpcStreamBody); + if (jsonRequest.method === 'uploadPart') { + assert.strictEqual(jsonRequest.uploadId, 'streamedId'); + return Response.json({ etag: 'partEtag' }); + } + assert.strictEqual(jsonRequest.method, 'put'); + break; + } + case 'rpc-conditional-null': { + assert.strictEqual(jsonRequest.method, 'put'); + assert.deepStrictEqual(jsonRequest.onlyIf, { + etagMatches: [{ value: 'strongEtag', type: 'strong' }], + }); + return r2ErrorResponse(preconditionFailedError); + } case 'onlyIfStrongEtag': { assert.deepStrictEqual(jsonRequest.onlyIf, { etagMatches: [ @@ -550,7 +685,7 @@ const testWorker = { } case 'blobBody': { assert.strictEqual(jsonRequest.method, 'put'); - assert.strictEqual(payload, body); + assert.strictEqual(uploadedBody, body); break; } } @@ -564,71 +699,162 @@ const testWorker = { assert.fail('R2 backend must not be called for invalid input'); } if (jsonRequest.method === 'list') { + // Without r2_list_honor_include, the binding always asks for both kinds of metadata. + // Each config tells the mock which behavior its R2 bindings use. + const honorsIncludes = env.R2_LIST_HONOR_INCLUDE !== 'false'; + const { include = [], ...listRequest } = jsonRequest; + const assertInclude = (requested) => + assert.deepStrictEqual( + include, + honorsIncludes ? requested : [0, 1] + ); + const listResponse = (result) => + buildGetResponse({ head: result, isList: true }); switch (jsonRequest.prefix) { + case undefined: { + assertInclude([]); + assert.deepStrictEqual(listRequest, { + method: 'list', + newRuntime: true, + version: 1, + }); + return listResponse({ objects: [], truncated: false }); + } case 'basic': { - assert.deepEqual(jsonRequest, { + assertInclude([0, 1]); + assert.deepEqual(listRequest, { cursor: 'ai', delimiter: '/', - include: [0, 1], limit: 1, method: 'list', newRuntime: true, prefix: 'basic', version: 1, }); - return buildGetResponse({ - head: { - objects: [objResponse], - truncated: true, - cursor: 'ai', - deliminatedPrefixes: [], - }, - isList: true, + return listResponse({ + objects: [objResponse], + truncated: true, + cursor: 'ai', }); } case 'httpMeta': { - assert.deepEqual(jsonRequest, { - include: [0], + assert.deepEqual(listRequest, { method: 'list', newRuntime: true, prefix: 'httpMeta', version: 1, }); - - return buildGetResponse({ - head: { - objects: [{ ...objResponse, httpFields, customFields: [] }], + // An empty include list returns objects without optional metadata. + if (include.length === 0) { + return listResponse({ + objects: [objResponse], truncated: false, - deliminatedPrefixes: [], - }, - isList: true, + }); + } + assertInclude([0]); + return listResponse({ + objects: [{ ...objResponse, httpFields, customFields: [] }], + truncated: false, }); } case 'customMeta': { - assert.deepEqual(jsonRequest, { - include: [1], + assertInclude([1]); + assert.deepEqual(listRequest, { method: 'list', newRuntime: true, prefix: 'customMeta', version: 1, }); - - return buildGetResponse({ - head: { - objects: [{ ...objResponse, httpFields: {}, customFields }], - truncated: false, - deliminatedPrefixes: [], - }, - isList: true, + return listResponse({ + objects: [{ ...objResponse, httpFields: {}, customFields }], + truncated: false, + }); + } + // Returns only the kinds of metadata that the request includes, which are empty. + case 'rpc-metadata': { + assert.deepStrictEqual(listRequest, { + method: 'list', + newRuntime: true, + prefix: 'rpc-metadata', + version: 1, + }); + return listResponse({ + objects: [ + { + ...objResponse, + ...(include.includes(0) && { httpFields: {} }), + ...(include.includes(1) && { customFields: [] }), + }, + ], + truncated: false, + }); + } + case 'rpc-options': { + assertInclude([]); + assert.deepStrictEqual(listRequest, { + cursor: 'cursor-in', + delimiter: '/', + limit: 2, + method: 'list', + newRuntime: true, + prefix: 'rpc-options', + startAfter: 'after', + version: 1, + }); + return listResponse({ + objects: [], + truncated: true, + cursor: 'cursor-out', + delimitedPrefixes: ['rpc-options/'], }); } + case 'rpc-boom': + return r2ErrorResponse(noSuchBucketError); } + throw new Error(`Unexpected list prefix: ${jsonRequest.prefix}`); } assert(['get', 'head'].includes(jsonRequest.method)); switch (jsonRequest.object) { - case 'basicKey': { + case 'basicKey': + case 'rpc-lazy-body': + case 'rpc-bytes': + case 'rpc-array-buffer': + case 'rpc-blob': { return buildGetResponse({ body }); } + case 'missing': + return r2ErrorResponse(noSuchKeyError); + case 'boom': + case 'rpc-get-boom': + return r2ErrorResponse(noSuchBucketError); + // The binding converts a non-string key to a string. + case '12345': { + assert.strictEqual(jsonRequest.method, 'head'); + return buildGetResponse(); + } + case 'ranged': { + assert.strictEqual(jsonRequest.method, 'head'); + return buildGetResponse({ + head: { range: { offset: '10', length: '20' } }, + }); + } + case 'rpc-json': { + assert.strictEqual(jsonRequest.method, 'get'); + const jsonBody = JSON.stringify({ ok: true }); + return buildGetResponse({ + head: { size: String(jsonBody.length) }, + body: jsonBody, + }); + } + case 'rpc-conditional-metadata': { + assert.strictEqual(jsonRequest.method, 'get'); + if (jsonRequest.onlyIf !== undefined) { + assert.deepStrictEqual(jsonRequest.onlyIf, { + etagMatches: [{ value: 'objectEtag', type: 'strong' }], + }); + } + return buildGetResponse({ error: preconditionFailedError }); + } case 'rangeOffLen': { assert.deepEqual(jsonRequest.range, { offset: '1', @@ -981,16 +1207,9 @@ const testWorker = { }); list.objects[0] = { ...list.objects[0] }; list.objects[0].checksums = { ...list.objects[0].checksums }; - const expected = { ...HeadObject, ...head }; - if ( - env.R2_TRANSPORT === 'jsrpc' && - env.R2_LIST_HONOR_INCLUDE === 'true' - ) { - expected.customMetadata = undefined; - } assert.deepEqual(list, { delimitedPrefixes: [], - objects: [expected], + objects: [{ ...HeadObject, ...head }], truncated: false, }); } @@ -1032,16 +1251,9 @@ const testWorker = { }); list.objects[0] = { ...list.objects[0] }; list.objects[0].checksums = { ...list.objects[0].checksums }; - const expected = { ...HeadObject, ...head }; - if ( - env.R2_TRANSPORT === 'jsrpc' && - env.R2_LIST_HONOR_INCLUDE === 'true' - ) { - expected.httpMetadata = undefined; - } assert.deepEqual(list, { delimitedPrefixes: [], - objects: [expected], + objects: [{ ...HeadObject, ...head }], truncated: false, }); } @@ -1175,66 +1387,104 @@ const testWorker = { }, }; -const finishedRpcUploads = new Set(); +// JSRPC bodies can arrive as a stream whose length is known only from the separate size argument. +// The native binding requires a known-length stream, so restore it from that argument. +function withKnownLength(value, valueSize) { + if (!(value instanceof ReadableStream)) { + return value; + } + const { readable, writable } = new FixedLengthStream(valueSize); + value.pipeTo(writable).catch(() => {}); + return readable; +} -// Checks that the key and upload ids match test expectations -function assertMultipartIdentity(requestKey, uploadId, action) { - assert.strictEqual(typeof requestKey, 'string'); - assert.strictEqual(typeof uploadId, 'string'); - if (requestKey.startsWith('rpc-multipart-')) { - assert( - [`${requestKey}-id`, 'resumed-a', 'resumed-b', 'missing'].includes( - uploadId - ) +// Implements the R2 binding's JSRPC methods by forwarding each call to a native R2 binding, in +// the way a customer Worker can stand in for R2. The JSRPC tests target this entrypoint so the +// shared suite exercises the serialization of native R2 results and errors, while REAL_BUCKET +// reaches the same HTTP mock as the HTTP tests. HTTP requests are rejected to detect any +// transport fallback on the JSRPC hop. +export class ObservedR2Binding extends WorkerEntrypoint { + fetch() { + throw new Error('R2 JSRPC tests must not use HTTP'); + } + + head(requestKey) { + return this.env.REAL_BUCKET.head(requestKey); + } + + get(requestKey, options) { + return this.env.REAL_BUCKET.get(requestKey, options); + } + + put(requestKey, value, options, valueSize) { + return this.env.REAL_BUCKET.put( + requestKey, + withKnownLength(value, valueSize), + options ); - const missing = - uploadId === 'missing' || - finishedRpcUploads.has(JSON.stringify([requestKey, uploadId])); - if (missing && action !== 'abortMultipartUpload') { - throw new Error(`${action}: Upload not found (10024)`); - } - return; } - const expected = { - 'must-not-call': 'invalidPartUploadId', - rpcStream: 'streamedId', - largeBuffer: 'largeUploadId', - 'rpc-wrong-part-number': 'wrongPartUploadId', - }[requestKey]; - if (expected !== undefined) { - assert.strictEqual(uploadId, expected); - } else { - assert( - uploadId === 'multipartId' || - (requestKey === key && uploadId === 'resumedId') + + delete(keys) { + return this.env.REAL_BUCKET.delete(keys); + } + + list(options) { + return this.env.REAL_BUCKET.list(options); + } + + async createMultipartUpload(requestKey, options) { + const upload = await this.env.REAL_BUCKET.createMultipartUpload( + requestKey, + options ); + return { key: upload.key, uploadId: upload.uploadId }; + } + + uploadPart(requestKey, uploadId, partNumber, value, options, valueSize) { + return this.env.REAL_BUCKET.resumeMultipartUpload( + requestKey, + uploadId + ).uploadPart(partNumber, withKnownLength(value, valueSize), options); + } + + abortMultipartUpload(requestKey, uploadId) { + return this.env.REAL_BUCKET.resumeMultipartUpload( + requestKey, + uploadId + ).abort(); + } + + completeMultipartUpload(requestKey, uploadId, uploadedParts) { + return this.env.REAL_BUCKET.resumeMultipartUpload( + requestKey, + uploadId + ).complete(uploadedParts); } } -// This entrypoint rejects HTTP requests so the JSRPC tests detect any transport fallback. -// The HTTP tests use the default export's backend. +// A fake R2 gateway for the gateway contract tests (jsrpcGateway*ContractTests). Each method +// asserts the exact arguments the JSRPC client sends for the keys those tests use, and some keys +// return results that a native R2 binding cannot produce. It rejects HTTP requests to detect any +// transport fallback. export class R2BindingEntrypoint extends WorkerEntrypoint { fetch() { throw new Error('R2 JSRPC tests must not use HTTP'); } head(requestKey) { - if (requestKey === 'missing') { - return null; - } - if (requestKey === 'boom') { - throw new Error('head: no such bucket (10006)'); - } - const result = buildRpcHead(requestKey); - if (requestKey === 'rpc-malformed-size') { - result.size = -1; - } - if (requestKey === 'rpc-malformed-range') { - result.range = { offset: -1, length: 1 }; - } - if (requestKey === 'rpc-malformed-date') { - result.uploaded = new Date(NaN); + switch (requestKey) { + case 'rpc-malformed-size': + result.size = -1; + break; + case 'rpc-malformed-range': + result.range = { offset: -1, length: 1 }; + break; + case 'rpc-malformed-date': + result.uploaded = new Date(NaN); + break; + default: + assert.fail(`unexpected head key: ${requestKey}`); } return result; } @@ -1259,178 +1509,87 @@ export class R2BindingEntrypoint extends WorkerEntrypoint { } return null; } - if (requestKey === 'missing') { - return null; - } - if (requestKey === 'rpc-get-boom') { - throw new Error('get: no such bucket (10006)'); - } - if (requestKey === 'rpc-malformed-kind') { - return { kind: 'other', object: buildRpcHead(requestKey) }; - } - if (requestKey === 'rpc-malformed-metadata-body') { - return { - kind: 'metadata', - object: buildRpcHead(requestKey), - body: new ReadableStream(), - }; - } - if (requestKey === 'rpc-malformed-missing-body') { - return { kind: 'body', object: buildRpcHead(requestKey) }; - } - if (requestKey === 'rpc-options') { - assert.deepStrictEqual(options, { - onlyIf: { - etagMatches: 'strongEtag', - uploadedBefore: new Date(0), - secondsGranularity: false, - }, - range: { offset: 1, length: 3 }, - ssecKey: hexKey, - }); - } - if (requestKey === 'rpc-header-options') { - assert(options.onlyIf instanceof Headers); - assert.strictEqual( - options.onlyIf.get('if-match'), - '"strongEtag", W/"weakEtag"' - ); - assert.strictEqual( - options.onlyIf.get('if-none-match'), - '"firstEtag", "secondEtag"' - ); - assert.strictEqual( - options.onlyIf.get('if-modified-since'), - new Date(0).toUTCString() - ); - assert(options.range instanceof Headers); - assert.strictEqual(options.range.get('range'), 'bytes=1-3'); - } - if (requestKey === 'onlyIfMultipleEtags') { - assert(options.onlyIf instanceof Headers); - assert.strictEqual( - options.onlyIf.get('if-match'), - '"strongEtag", W/"weakEtag"' - ); - assert.strictEqual( - options.onlyIf.get('if-none-match'), - '"firstEtag", "secondEtag"' - ); - } - if (requestKey === 'must-not-call') { - throw new Error('get RPC method must not be called'); - } - - const object = buildRpcHead(requestKey); - if (requestKey === 'rangeOffLen') { - assert.deepStrictEqual(options, { range: { offset: 1, length: 3 } }); - object.range = { offset: 1, length: 3 }; - } else if (requestKey === 'rangeSuff') { - assert.deepStrictEqual(options, { range: { suffix: 2 } }); - object.range = { offset: 5, length: 2 }; - } - - if (requestKey === 'rpc-conditional-metadata') { - return { kind: 'metadata', object }; - } - - const responseBody = - requestKey === 'rangeOffLen' - ? 'ont' - : requestKey === 'rangeSuff' - ? 'nt' - : requestKey === 'rpc-json' - ? JSON.stringify({ ok: true }) - : body; - if (object.range === undefined) { - object.size = new TextEncoder().encode(responseBody).byteLength; - } - let sent = false; - return { - kind: 'body', - object, - body: new ReadableStream({ - pull(controller) { - assert.strictEqual(sent, false); - sent = true; - controller.enqueue(new TextEncoder().encode(responseBody)); - controller.close(); - }, - }), - }; - } - - delete(keys) { - if (keys === 'boom') { - throw new Error('delete: bad keys (10021)'); - } - if (Array.isArray(keys)) { - if (keys[0] === perKeyErrorDeleteKeys[0]) { - // R2BindingEntrypoint has already discarded the successful HTTP batch response body before - // deleteRpc() receives this void RPC result. - assert.deepEqual(keys, perKeyErrorDeleteKeys); - } else { - assert.deepEqual(keys, [key, key + '2']); - } - } else { - assert.strictEqual(keys, key); + switch (requestKey) { + case 'rpc-malformed-kind': + return { kind: 'other', object: buildRpcHead(requestKey) }; + case 'rpc-malformed-metadata-body': + return { + kind: 'metadata', + object: buildRpcHead(requestKey), + body: new ReadableStream(), + }; + case 'rpc-malformed-missing-body': + return { kind: 'body', object: buildRpcHead(requestKey) }; + case 'rpc-options': + assert.deepStrictEqual(options, { + onlyIf: { + etagMatches: 'strongEtag', + uploadedBefore: new Date(0), + secondsGranularity: false, + }, + range: { offset: 1, length: 3 }, + ssecKey: hexKey, + }); + return null; + case 'rpc-header-options': + assert(options.onlyIf instanceof Headers); + assert.strictEqual( + options.onlyIf.get('if-match'), + '"strongEtag", W/"weakEtag"' + ); + assert.strictEqual( + options.onlyIf.get('if-none-match'), + '"firstEtag", "secondEtag"' + ); + assert.strictEqual( + options.onlyIf.get('if-modified-since'), + new Date(0).toUTCString() + ); + assert(options.range instanceof Headers); + assert.strictEqual(options.range.get('range'), 'bytes=1-3'); + return null; } - return rpcSuccess(null); + assert.fail(`unexpected get key: ${requestKey}`); } async put(requestKey, value, options, valueSize) { - if (requestKey === 'must-not-call') { - throw new Error('put RPC method must not be called'); - } - if (requestKey === 'rpc-null-value') { - assert.strictEqual(value, null); - assert.strictEqual(valueSize, 0); - return buildRpcHead(requestKey); - } - if (requestKey === 'rpc-conditional-null') { - assert.deepStrictEqual(options.onlyIf, { - etagMatches: 'strongEtag', - secondsGranularity: false, - }); - return null; - } - if (requestKey === 'rpc-put-options') { - assert.deepStrictEqual(options.onlyIf, { - etagMatches: 'strongEtag', - uploadedBefore: new Date(0), - secondsGranularity: false, - }); - assert.deepStrictEqual(options.httpMetadata, httpMetaObj); - assert.deepStrictEqual(options.customMetadata, customMetadata); - assert.deepStrictEqual(new Uint8Array(options.md5), md5Buffer); - assert.strictEqual(options.storageClass, 'InfrequentAccess'); - assert.strictEqual(options.ssecKey, hexKey); - } - if (requestKey === 'rpc-put-header-options') { - assert(options.onlyIf instanceof Headers); - assert.strictEqual( - options.onlyIf.get('if-match'), - '"strongEtag", W/"weakEtag"' - ); - assert.strictEqual( - options.onlyIf.get('if-none-match'), - '"firstEtag", "secondEtag"' - ); - } - if (requestKey === 'onlyIfMultipleEtags') { - assert(options.onlyIf instanceof Headers); - assert.strictEqual( - options.onlyIf.get('if-match'), - '"strongEtag", W/"weakEtag"' - ); - assert.strictEqual( - options.onlyIf.get('if-none-match'), - '"firstEtag", "secondEtag"' - ); - } - if (requestKey.startsWith('large')) { - await assertLargeRpcBody(requestKey, value, valueSize); - return buildRpcHead(requestKey, options); + switch (requestKey) { + case 'rpc-null-value': + assert.strictEqual(value, null); + assert.strictEqual(valueSize, 0); + return buildRpcHead(requestKey); + case 'rpc-put-options': + assert.deepStrictEqual(options.onlyIf, { + etagMatches: 'strongEtag', + uploadedBefore: new Date(0), + secondsGranularity: false, + }); + assert.deepStrictEqual(options.httpMetadata, httpMetaObj); + assert.deepStrictEqual(options.customMetadata, customMetadata); + assert.deepStrictEqual(new Uint8Array(options.md5), md5Buffer); + assert.strictEqual(options.storageClass, 'InfrequentAccess'); + assert.strictEqual(options.ssecKey, hexKey); + break; + case 'rpc-put-header-options': + assert(options.onlyIf instanceof Headers); + assert.strictEqual( + options.onlyIf.get('if-match'), + '"strongEtag", W/"weakEtag"' + ); + assert.strictEqual( + options.onlyIf.get('if-none-match'), + '"firstEtag", "secondEtag"' + ); + break; + case 'largeBuffer': + case 'largeString': + case 'largeBlob': + await assertLargeRpcBody(requestKey, value, valueSize); + return buildRpcHead(requestKey, options); + case 'rpcStream': + break; + default: + assert.fail(`unexpected put key: ${requestKey}`); } const uploaded = await new Response(value).text(); const expected = requestKey === 'rpcStream' ? rpcStreamBody : body; @@ -1448,9 +1607,6 @@ export class R2BindingEntrypoint extends WorkerEntrypoint { ? (options?.include ?? []) : ['httpMetadata', 'customMetadata']; - if (options?.prefix === 'rpc-boom') { - throw new Error('list: no such bucket (10006)'); - } if (options?.prefix === 'rpc-malformed') { return { objects: [{}], truncated: false }; } @@ -1471,39 +1627,6 @@ export class R2BindingEntrypoint extends WorkerEntrypoint { }; } - if (options?.prefix === 'basic') { - assert.deepStrictEqual(options, { - limit: 1, - prefix: 'basic', - cursor: 'ai', - delimiter: '/', - include: effectiveIncludes, - }); - return { - objects: [listRpcHead('basic', effectiveIncludes)], - truncated: true, - cursor: 'ai', - }; - } - if (options?.prefix === 'httpMeta') { - return { - objects: [listRpcHead('httpMetadata', effectiveIncludes)], - truncated: false, - }; - } - if (options?.prefix === 'customMeta') { - return { - objects: [listRpcHead('customMetadata', effectiveIncludes)], - truncated: false, - }; - } - if (options?.prefix === 'rpc-metadata') { - return { - objects: [listRpcHead('basic', effectiveIncludes)], - truncated: false, - }; - } - if (honorsIncludes) { assert.strictEqual(options, undefined); } else { @@ -1515,19 +1638,12 @@ export class R2BindingEntrypoint extends WorkerEntrypoint { } createMultipartUpload(requestKey, options) { - if (requestKey === 'must-not-call') { - throw new Error('createMultipartUpload RPC method must not be called'); - } - if (requestKey === 'rpc-create-options') { - assert.deepStrictEqual(options.httpMetadata, httpMetaObj); - assert.deepStrictEqual(options.customMetadata, customMetadata); - assert.strictEqual(options.storageClass, 'InfrequentAccess'); - assert.strictEqual(options.ssecKey, hexKey); - } - assert.strictEqual(typeof requestKey, 'string'); - return requestKey.startsWith('rpc-multipart-') - ? `${requestKey}-id` - : 'multipartId'; + assert.strictEqual(requestKey, 'rpc-create-options'); + assert.deepStrictEqual(options.httpMetadata, httpMetaObj); + assert.deepStrictEqual(options.customMetadata, customMetadata); + assert.strictEqual(options.storageClass, 'InfrequentAccess'); + assert.strictEqual(options.ssecKey, hexKey); + return 'multipartId'; } async uploadPart( @@ -1538,98 +1654,39 @@ export class R2BindingEntrypoint extends WorkerEntrypoint { options, valueSize ) { - assertMultipartIdentity(requestKey, uploadId, 'uploadPart'); - assert(partNumber >= 1 && partNumber <= 10000); - if (requestKey === 'must-not-call') { - throw new Error('uploadPart RPC method must not be called'); - } + const expectedUploadId = { + rpcStream: 'streamedId', + largeBuffer: 'largeUploadId', + 'rpc-wrong-part-number': 'wrongPartUploadId', + }[requestKey]; + assert.strictEqual(uploadId, expectedUploadId); if (requestKey === 'largeBuffer') { await assertLargeRpcBody(requestKey, value, valueSize); return { partNumber, etag: 'partEtag' }; } const uploaded = await new Response(value).text(); - const expected = - requestKey === 'ssecMultipart' - ? 'hey' - : requestKey === 'rpcStream' - ? rpcStreamBody - : body; + const expected = requestKey === 'rpcStream' ? rpcStreamBody : body; assert.strictEqual(uploaded, expected); assert.strictEqual( valueSize, new TextEncoder().encode(expected).byteLength ); - if (requestKey === 'ssecMultipart') { - assert.strictEqual(options?.ssecKey, hexKey); - } return { partNumber: requestKey === 'rpc-wrong-part-number' ? 9999 : partNumber, - etag: requestKey.startsWith('rpc-multipart-') - ? `${requestKey}/${uploadId}/${partNumber}` - : uploadId === 'resumedId' - ? 'resumedRpcPartEtag' - : 'partEtag', + etag: 'partEtag', }; } +} - abortMultipartUpload(requestKey, uploadId) { - assertMultipartIdentity(requestKey, uploadId, 'abortMultipartUpload'); - if (requestKey.startsWith('rpc-multipart-')) { - finishedRpcUploads.add(JSON.stringify([requestKey, uploadId])); - } - } - - completeMultipartUpload(requestKey, uploadId, uploadedParts) { - assertMultipartIdentity(requestKey, uploadId, 'completeMultipartUpload'); - if (requestKey === 'must-not-call') { - throw new Error('completeMultipartUpload RPC method must not be called'); - } - for (const part of uploadedParts) { - assert(part.partNumber >= 1 && part.partNumber <= 10000); - assert.strictEqual(typeof part.etag, 'string'); - if (requestKey.startsWith('rpc-multipart-')) { - assert.strictEqual( - part.etag, - `${requestKey}/${uploadId}/${part.partNumber}` - ); - } - } - const result = buildRpcHead(requestKey); - // The backend completion response omits create-time HTTP and custom metadata. - result.httpMetadata = {}; - result.customMetadata = {}; - if (requestKey.startsWith('rpc-multipart-')) { - finishedRpcUploads.add(JSON.stringify([requestKey, uploadId])); - result.key = requestKey; - result.version = uploadId; - } - if (uploadId === 'resumedId') { - result.version = 'resumedRpcObjectVersion'; - } - return result; - } -} - -function listRpcHead(requestKey, includes) { - const result = buildRpcHead(requestKey); - if (!includes.includes('httpMetadata')) { - delete result.httpMetadata; - } - if (!includes.includes('customMetadata')) { - delete result.customMetadata; - } - return result; -} - -// Test that documents the behaviour of multi-key deletion where one fails and the other succeeds -export const deletePerKeyErrorParityTests = { - async test(_ctrl, env) { - if (env.R2_TRANSPORT !== 'http' && env.R2_TRANSPORT !== 'jsrpc') { - return; +// Test that documents the behaviour of multi-key deletion where one fails and the other succeeds +export const deletePerKeyErrorParityTests = { + async test(_ctrl, env) { + if (env.R2_TRANSPORT !== 'http' && env.R2_TRANSPORT !== 'jsrpc') { + return; } // Both transports implement Promise: the legacy path discards the successful HTTP body, - // while the RPC path receives the corresponding void result from R2BindingEntrypoint. + // while the RPC path receives the corresponding void result from its backend entrypoint. const deletion = env.BUCKET.delete(perKeyErrorDeleteKeys); assert(deletion instanceof Promise); assert.strictEqual(await deletion, undefined); @@ -1733,43 +1790,33 @@ export const r2ValidationTests = { }, }; -// These cases cover RPC boundary behavior that the HTTP-oriented fake cannot observe directly. -// The canonical API suite remains identical for both transport configurations. -export const jsrpcTransportTests = { - async test(ctrl, env, ctx) { - if (env.R2_TRANSPORT !== 'jsrpc') { - return; - } - +// Public R2 binding behavior that both transports implement the same way. The JSRPC configs run +// these through ObservedR2Binding, so results and errors come from the same HTTP mock as the HTTP +// config. +export const r2BindingApiTests = { + async test(ctrl, env) { assert.strictEqual(await env.BUCKET.head('missing'), null); + assert.strictEqual(await env.BUCKET.get('missing'), null); const ranged = await env.BUCKET.head('ranged'); assert.deepStrictEqual(ranged.range, { offset: 10, length: 20 }); + const coerced = await env.BUCKET.head(12345); + assert.strictEqual(coerced.key, key); + + // Backend errors surface as plain Errors, without an R2Error code property. await assert.rejects(env.BUCKET.head('boom'), (err) => { assert.strictEqual(err.message, 'head: no such bucket (10006)'); assert.strictEqual(err.code, undefined); return true; }); - await assert.rejects(env.BUCKET.head('rpc-malformed-size'), { - message: /^internal error; reference = \S+$/, - }); - await assert.rejects(env.BUCKET.head('rpc-malformed-range'), { - message: /^internal error; reference = \S+$/, - }); - await assert.rejects(env.BUCKET.head('rpc-malformed-date'), { - message: /^internal error; reference = \S+$/, + await assert.rejects(env.BUCKET.get('rpc-get-boom'), { + message: 'get: no such bucket (10006)', }); - await assert.rejects(env.BUCKET.delete('boom'), { message: 'delete: bad keys (10021)', }); - const coerced = await env.BUCKET.head(12345); - assert.strictEqual(coerced.key, key); - - assert.strictEqual(await env.BUCKET.get('missing'), null); - const conditional = await env.BUCKET.get('rpc-conditional-metadata', { onlyIf: { etagMatches: 'objectEtag' }, }); @@ -1800,37 +1847,6 @@ export const jsrpcTransportTests = { const blobResult = await env.BUCKET.get('rpc-blob'); assert.strictEqual(await (await blobResult.blob()).text(), body); - await env.BUCKET.get('rpc-options', { - onlyIf: { - etagMatches: 'strongEtag', - uploadedBefore: new Date(0), - }, - range: { offset: 1, length: 3 }, - ssecKey: bufferKey, - }); - await env.BUCKET.get('rpc-header-options', { - onlyIf: new Headers({ - 'if-match': '"strongEtag", W/"weakEtag"', - 'if-none-match': '"firstEtag", "secondEtag"', - 'if-modified-since': new Date(0).toUTCString(), - }), - range: new Headers({ range: 'bytes=1-3' }), - }); - - await assert.rejects(env.BUCKET.get('rpc-malformed-kind'), { - message: /^internal error; reference = \S+$/, - }); - await assert.rejects(env.BUCKET.get('rpc-malformed-metadata-body'), { - message: /^internal error; reference = \S+$/, - }); - // The missing-body assertion logs the details and exposes only an internal error reference. - await assert.rejects(env.BUCKET.get('rpc-malformed-missing-body'), { - message: /^internal error; reference = \S+$/, - }); - await assert.rejects(env.BUCKET.get('rpc-get-boom'), { - message: 'get: no such bucket (10006)', - }); - const emptyList = await env.BUCKET.list(); assert.deepStrictEqual(emptyList, { objects: [], @@ -1887,35 +1903,13 @@ export const jsrpcTransportTests = { await assert.rejects(env.BUCKET.list({ prefix: 'rpc-boom' }), { message: 'list: no such bucket (10006)', }); - await assert.rejects(env.BUCKET.list({ prefix: 'rpc-malformed' }), { - message: /^internal error; reference = \S+$/, - }); - const nullPut = await env.BUCKET.put('rpc-null-value', null); - assert.strictEqual(nullPut.key, key); assert.strictEqual( await env.BUCKET.put('rpc-conditional-null', body, { onlyIf: { etagMatches: 'strongEtag' }, }), null ); - await env.BUCKET.put('rpc-put-options', body, { - onlyIf: { - etagMatches: 'strongEtag', - uploadedBefore: new Date(0), - }, - httpMetadata: httpMetaHeaders, - customMetadata, - md5: md5Buffer, - storageClass: 'InfrequentAccess', - ssecKey: bufferKey, - }); - await env.BUCKET.put('rpc-put-header-options', body, { - onlyIf: new Headers({ - 'if-match': '"strongEtag", W/"weakEtag"', - 'if-none-match': '"firstEtag", "secondEtag"', - }), - }); const encodedStreamBody = new TextEncoder().encode(rpcStreamBody); { @@ -1946,53 +1940,21 @@ export const jsrpcTransportTests = { 'Provided readable stream must have a known length (request/response body or readable half of FixedLengthStream)', } ); - assert.strictEqual( - unknownLengthCancelReason.message, - 'Stream cancelled because the associated put operation encountered an error.' - ); - - const largeBufferBacking = new Uint8Array(largeRpcBodySize + 2); - largeBufferBacking.fill(0x41); - const largeBuffer = largeBufferBacking.subarray(1, -1); - largeBuffer[0] = 0x11; - largeBuffer[largeBuffer.length - 1] = 0x22; - await env.BUCKET.put('largeBuffer', largeBuffer); - - const largeString = 'é'.repeat(largeRpcBodySize / 2); - await env.BUCKET.put('largeString', largeString); - - const largeBlobBytes = new Uint8Array(largeRpcBodySize); - largeBlobBytes.fill(0x5a); - await env.BUCKET.put('largeBlob', new Blob([largeBlobBytes])); + // The JSRPC client cancels a stream it rejects; the HTTP client leaves it to the caller. + if (env.R2_TRANSPORT === 'jsrpc') { + assert.strictEqual( + unknownLengthCancelReason.message, + 'Stream cancelled because the associated put operation encountered an error.' + ); + } else { + assert.strictEqual(unknownLengthCancelReason, undefined); + } }, }; -export const jsrpcMultipartTests = { +// Multipart behavior that both transports implement the same way. See r2BindingApiTests. +export const r2MultipartApiTests = { async test(ctrl, env) { - if (env.R2_TRANSPORT !== 'jsrpc') { - return; - } - - const createOptionsUpload = await env.BUCKET.createMultipartUpload( - 'rpc-create-options', - { - httpMetadata: httpMetaHeaders, - customMetadata, - storageClass: 'InfrequentAccess', - ssecKey: bufferKey, - } - ); - assert.strictEqual(createOptionsUpload.uploadId, 'multipartId'); - - const encodedStreamBody = new TextEncoder().encode(rpcStreamBody); - // Use a view with a non-zero byte offset to verify that the RPC upload streams only the view's - // bytes, rather than the entire underlying ArrayBuffer. - const largeBufferBacking = new Uint8Array(largeRpcBodySize + 2); - largeBufferBacking.fill(0x41); - const largeBuffer = largeBufferBacking.subarray(1, -1); - largeBuffer[0] = 0x11; - largeBuffer[largeBuffer.length - 1] = 0x22; - const resumed = env.BUCKET.resumeMultipartUpload(key, 'resumedId'); assert.strictEqual(resumed.key, key); assert.strictEqual(resumed.uploadId, 'resumedId'); @@ -2015,6 +1977,7 @@ export const jsrpcMultipartTests = { assert.strictEqual(typeof resumedObject.checksums.toJSON, 'function'); assert.strictEqual(JSON.stringify(resumedObject.checksums), '{}'); + const encodedStreamBody = new TextEncoder().encode(rpcStreamBody); const streamedUpload = env.BUCKET.resumeMultipartUpload( 'rpcStream', 'streamedId' @@ -2031,24 +1994,6 @@ export const jsrpcMultipartTests = { etag: 'partEtag', }); - const largeUpload = env.BUCKET.resumeMultipartUpload( - 'largeBuffer', - 'largeUploadId' - ); - assert.deepStrictEqual(await largeUpload.uploadPart(1, largeBuffer), { - partNumber: 1, - etag: 'partEtag', - }); - - const wrongPartUpload = env.BUCKET.resumeMultipartUpload( - 'rpc-wrong-part-number', - 'wrongPartUploadId' - ); - assert.deepStrictEqual(await wrongPartUpload.uploadPart(3, body), { - partNumber: 3, - etag: 'partEtag', - }); - // Each operation carries its key and upload ID, including interleaved calls on the same key. const uploads = []; for (const uploadKey of ['rpc-multipart-first', 'rpc-multipart-second']) { @@ -2065,7 +2010,15 @@ export const jsrpcMultipartTests = { assert.strictEqual(upload.uploadId, `${uploadKey}-id`); uploads.push(upload); } - for (const uploadId of ['resumed-a', 'resumed-b']) { + // More uploads for the same key, addressed through wrappers resumed from their IDs. + for (const uploadId of [ + 'rpc-multipart-first-id-2', + 'rpc-multipart-first-id-3', + ]) { + const created = await env.BUCKET.createMultipartUpload( + 'rpc-multipart-first' + ); + assert.strictEqual(created.uploadId, uploadId); const upload = env.BUCKET.resumeMultipartUpload( 'rpc-multipart-first', uploadId @@ -2098,7 +2051,7 @@ export const jsrpcMultipartTests = { const result = await upload.complete(uploadedParts[i]); assert.strictEqual(result.key, upload.key); assert.strictEqual(result.version, upload.uploadId); - // Created and resumed uploads both use the metadata returned by the gateway. + // Created and resumed uploads both use the metadata returned by the backend. assert.deepStrictEqual(result.customMetadata, {}); const resultHeaders = new Headers(); result.writeHttpMetadata(resultHeaders); @@ -2131,7 +2084,196 @@ export const jsrpcMultipartTests = { }, }; -export const jsrpcRangeTests = { +// Gateway contract tests: they target R2BindingEntrypoint, a fake gateway that asserts the exact +// arguments the JSRPC client sends and returns results that a native R2 binding cannot produce. +// A passthrough to a native binding cannot observe or produce either. +export const jsrpcGatewayContractTests = { + async test(ctrl, env) { + if (env.R2_TRANSPORT !== 'jsrpc') { + return; + } + + await assert.rejects(env.BUCKET.head('rpc-malformed-size'), { + message: /^internal error; reference = \S+$/, + }); + await assert.rejects(env.BUCKET.head('rpc-malformed-range'), { + message: /^internal error; reference = \S+$/, + }); + await assert.rejects(env.BUCKET.head('rpc-malformed-date'), { + message: /^internal error; reference = \S+$/, + }); + + await env.BUCKET.get('rpc-options', { + onlyIf: { + etagMatches: 'strongEtag', + uploadedBefore: new Date(0), + }, + range: { offset: 1, length: 3 }, + ssecKey: bufferKey, + }); + await env.BUCKET.get('rpc-header-options', { + onlyIf: new Headers({ + 'if-match': '"strongEtag", W/"weakEtag"', + 'if-none-match': '"firstEtag", "secondEtag"', + 'if-modified-since': new Date(0).toUTCString(), + }), + range: new Headers({ range: 'bytes=1-3' }), + }); + + await assert.rejects(env.BUCKET.get('rpc-malformed-kind'), { + message: /^internal error; reference = \S+$/, + }); + await assert.rejects(env.BUCKET.get('rpc-malformed-metadata-body'), { + message: /^internal error; reference = \S+$/, + }); + // The missing-body assertion logs the details and exposes only an internal error reference. + await assert.rejects(env.BUCKET.get('rpc-malformed-missing-body'), { + message: /^internal error; reference = \S+$/, + }); + + // The gateway checks the list options, including the include list the compatibility flag + // selects. + assert.deepStrictEqual(await env.BUCKET.list(), { + objects: [], + truncated: false, + delimitedPrefixes: [], + }); + assert.deepStrictEqual( + await env.BUCKET.list({ + limit: 2, + prefix: 'rpc-options', + cursor: 'cursor-in', + delimiter: '/', + startAfter: 'after', + include: [], + }), + { + objects: [], + truncated: true, + cursor: 'cursor-out', + delimitedPrefixes: ['rpc-options/'], + } + ); + await assert.rejects(env.BUCKET.list({ prefix: 'rpc-malformed' }), { + message: /^internal error; reference = \S+$/, + }); + + const nullPut = await env.BUCKET.put('rpc-null-value', null); + assert.strictEqual(nullPut.key, key); + await env.BUCKET.put('rpc-put-options', body, { + onlyIf: { + etagMatches: 'strongEtag', + uploadedBefore: new Date(0), + }, + httpMetadata: httpMetaHeaders, + customMetadata, + md5: md5Buffer, + storageClass: 'InfrequentAccess', + ssecKey: bufferKey, + }); + await env.BUCKET.put('rpc-put-header-options', body, { + onlyIf: new Headers({ + 'if-match': '"strongEtag", W/"weakEtag"', + 'if-none-match': '"firstEtag", "secondEtag"', + }), + }); + + // The gateway checks the streamed body and its separately sent size. + const encodedStreamBody = new TextEncoder().encode(rpcStreamBody); + { + const { readable, writable } = new FixedLengthStream( + encodedStreamBody.byteLength + ); + const writer = writable.getWriter(); + const writing = writer + .write(encodedStreamBody) + .then(() => writer.close()); + await env.BUCKET.put('rpcStream', readable); + await writing; + } + + // Bodies above the inline limit are sent as streams. + const largeBufferBacking = new Uint8Array(largeRpcBodySize + 2); + largeBufferBacking.fill(0x41); + const largeBuffer = largeBufferBacking.subarray(1, -1); + largeBuffer[0] = 0x11; + largeBuffer[largeBuffer.length - 1] = 0x22; + await env.BUCKET.put('largeBuffer', largeBuffer); + + const largeString = 'é'.repeat(largeRpcBodySize / 2); + await env.BUCKET.put('largeString', largeString); + + const largeBlobBytes = new Uint8Array(largeRpcBodySize); + largeBlobBytes.fill(0x5a); + await env.BUCKET.put('largeBlob', new Blob([largeBlobBytes])); + }, +}; + +// Multipart gateway contract tests. See jsrpcGatewayContractTests. +export const jsrpcGatewayMultipartContractTests = { + async test(ctrl, env) { + if (env.R2_TRANSPORT !== 'jsrpc') { + return; + } + + const createOptionsUpload = await env.BUCKET.createMultipartUpload( + 'rpc-create-options', + { + httpMetadata: httpMetaHeaders, + customMetadata, + storageClass: 'InfrequentAccess', + ssecKey: bufferKey, + } + ); + assert.strictEqual(createOptionsUpload.uploadId, 'multipartId'); + + // The gateway checks the streamed body and its separately sent size. + const encodedStreamBody = new TextEncoder().encode(rpcStreamBody); + const streamedUpload = env.BUCKET.resumeMultipartUpload( + 'rpcStream', + 'streamedId' + ); + const { readable, writable } = new FixedLengthStream( + encodedStreamBody.byteLength + ); + const writer = writable.getWriter(); + const writing = writer.write(encodedStreamBody).then(() => writer.close()); + assert.deepStrictEqual(await streamedUpload.uploadPart(2, readable), { + partNumber: 2, + etag: 'partEtag', + }); + await writing; + + // Use a view with a non-zero byte offset to verify that the RPC upload streams only the view's + // bytes, rather than the entire underlying ArrayBuffer. + const largeBufferBacking = new Uint8Array(largeRpcBodySize + 2); + largeBufferBacking.fill(0x41); + const largeBuffer = largeBufferBacking.subarray(1, -1); + largeBuffer[0] = 0x11; + largeBuffer[largeBuffer.length - 1] = 0x22; + const largeUpload = env.BUCKET.resumeMultipartUpload( + 'largeBuffer', + 'largeUploadId' + ); + assert.deepStrictEqual(await largeUpload.uploadPart(1, largeBuffer), { + partNumber: 1, + etag: 'partEtag', + }); + + // The gateway reports a different part number; the result uses the requested one. + const wrongPartUpload = env.BUCKET.resumeMultipartUpload( + 'rpc-wrong-part-number', + 'wrongPartUploadId' + ); + assert.deepStrictEqual(await wrongPartUpload.uploadPart(3, body), { + partNumber: 3, + etag: 'partEtag', + }); + }, +}; + +// Range gateway contract tests. See jsrpcGatewayContractTests. +export const jsrpcGatewayRangeContractTests = { async test(controller, env) { if (env.R2_TRANSPORT !== 'jsrpc') { return; diff --git a/src/workerd/api/tests/r2-test.wd-test b/src/workerd/api/tests/r2-test.wd-test index 916a4d99c46..e76c75fb8b9 100644 --- a/src/workerd/api/tests/r2-test.wd-test +++ b/src/workerd/api/tests/r2-test.wd-test @@ -11,6 +11,7 @@ const unitTests :Workerd.Config = ( ( name = "BUCKET", r2Bucket = "r2-test" ), ( name = "SERVICE", service = "r2-test" ), ( name = "R2_TRANSPORT", text = "http" ), + ( name = "R2_LIST_HONOR_INCLUDE", text = "true" ), ], compatibilityFlags = ["nodejs_compat", "service_binding_extra_handlers", "streams_enable_constructors", "r2_list_honor_include", "disable_fast_jsg_struct"], streamingTails = ["tail"], From 9c69cac11dc7b18de0ca2d4ee39170c17ff937a5 Mon Sep 17 00:00:00 2001 From: Carol Xu Date: Mon, 28 Sep 2026 17:30:23 -0500 Subject: [PATCH 03/13] remove unneeded fast struct tests, group them under general compat flag test --- src/workerd/api/tests/BUILD.bazel | 5 +- .../tests/r2-jsrpc-legacy-list-test.wd-test | 3 +- .../api/tests/r2-jsrpc-multipart-test.wd-test | 50 +++-------------- .../api/tests/r2-jsrpc-passthrough.capnp | 23 ++------ src/workerd/api/tests/r2-jsrpc-test.wd-test | 4 +- src/workerd/api/tests/r2-test.js | 55 ++++++++++++++----- src/workerd/api/tests/r2-test.wd-test | 4 +- 7 files changed, 60 insertions(+), 84 deletions(-) diff --git a/src/workerd/api/tests/BUILD.bazel b/src/workerd/api/tests/BUILD.bazel index cca42de3e48..5f058c44aec 100644 --- a/src/workerd/api/tests/BUILD.bazel +++ b/src/workerd/api/tests/BUILD.bazel @@ -564,10 +564,7 @@ wd_test( wd_test( src = "r2-jsrpc-multipart-test.wd-test", args = ["--experimental"], - data = [ - "r2-jsrpc-passthrough.capnp", - "r2-test.js", - ], + data = ["r2-test.js"], ) wd_test( diff --git a/src/workerd/api/tests/r2-jsrpc-legacy-list-test.wd-test b/src/workerd/api/tests/r2-jsrpc-legacy-list-test.wd-test index 5f168e24750..9c7af705795 100644 --- a/src/workerd/api/tests/r2-jsrpc-legacy-list-test.wd-test +++ b/src/workerd/api/tests/r2-jsrpc-legacy-list-test.wd-test @@ -19,8 +19,7 @@ const unitTests :Workerd.Config = ( ( name = "R2_LIST_HONOR_INCLUDE", text = "false" ), ], compatibilityFlags = ["experimental", "nodejs_compat", "service_binding_extra_handlers", - "streams_enable_constructors", "r2_binding_jsrpc", - "disable_fast_jsg_struct"], + "streams_enable_constructors", "r2_binding_jsrpc"], ) ), ( name = "r2-test", worker = Passthrough.passthroughLegacyListWorker ), diff --git a/src/workerd/api/tests/r2-jsrpc-multipart-test.wd-test b/src/workerd/api/tests/r2-jsrpc-multipart-test.wd-test index e4feb6860df..8dba6134bb0 100644 --- a/src/workerd/api/tests/r2-jsrpc-multipart-test.wd-test +++ b/src/workerd/api/tests/r2-jsrpc-multipart-test.wd-test @@ -1,49 +1,20 @@ using Workerd = import "/workerd/workerd.capnp"; -using Passthrough = import "r2-jsrpc-passthrough.capnp"; -# Runs the multipart suites with fast structs; r2-jsrpc-test.wd-test covers slow structs for the -# shared suite. -# -# The fast-structs service calls the passthrough in r2-jsrpc-passthrough.capnp. -# -# The gateway contract suite targets R2BindingEntrypoint in both struct-wrapping modes, because it -# asserts the exact arguments the JSRPC client sends. +# Checks the multipart JSRPC client against R2BindingEntrypoint, a fake gateway that asserts the +# exact arguments the client sends. It uses the same struct mode as the other R2 JSRPC configs; +# the multipart wire inputs do not depend on property presence, so one mode is enough. +# r2-jsrpc-test.wd-test and r2-jsrpc-legacy-list-test.wd-test run r2MultipartApiTests through +# the passthrough to a native binding. const unitTests :Workerd.Config = ( services = [ - ( name = "fast-structs", + ( name = "gateway", worker = ( modules = [ - (name = "worker", esModule = "export { r2MultipartApiTests } from './r2-test.js';"), + (name = "worker", esModule = "export { R2BindingEntrypoint, jsrpcGatewayMultipartContractTests } from './r2-test.js';"), (name = "r2-test.js", esModule = embed "r2-test.js"), ], bindings = [ - (name = "BUCKET", - r2Bucket = (name = "r2-test", entrypoint = "ObservedR2Binding")), - (name = "R2_TRANSPORT", text = "jsrpc"), - ], - compatibilityFlags = ["experimental", "nodejs_compat", "service_binding_extra_handlers", - "streams_enable_constructors", "r2_binding_jsrpc", "enable_fast_jsg_struct"], - ), - ), - ( name = "r2-test", worker = Passthrough.passthroughFastStructsWorker ), - ( name = "gateway-fast-structs", - worker = ( - modules = .gatewayModules, - bindings = [ - (name = "BUCKET", - r2Bucket = (name = "gateway-fast-structs", entrypoint = "R2BindingEntrypoint")), - (name = "R2_TRANSPORT", text = "jsrpc"), - ], - compatibilityFlags = ["experimental", "nodejs_compat", "service_binding_extra_handlers", - "streams_enable_constructors", "r2_binding_jsrpc", "enable_fast_jsg_struct"], - ), - ), - ( name = "gateway-slow-structs", - worker = ( - modules = .gatewayModules, - bindings = [ - (name = "BUCKET", - r2Bucket = (name = "gateway-slow-structs", entrypoint = "R2BindingEntrypoint")), + (name = "BUCKET", r2Bucket = (name = "gateway", entrypoint = "R2BindingEntrypoint")), (name = "R2_TRANSPORT", text = "jsrpc"), ], compatibilityFlags = ["experimental", "nodejs_compat", "service_binding_extra_handlers", @@ -52,8 +23,3 @@ const unitTests :Workerd.Config = ( ), ], ); - -const gatewayModules :List(Workerd.Worker.Module) = [ - (name = "worker", esModule = "export { R2BindingEntrypoint, jsrpcGatewayMultipartContractTests } from './r2-test.js';"), - (name = "r2-test.js", esModule = embed "r2-test.js"), -]; diff --git a/src/workerd/api/tests/r2-jsrpc-passthrough.capnp b/src/workerd/api/tests/r2-jsrpc-passthrough.capnp index c44e1447e96..233ea5c183d 100644 --- a/src/workerd/api/tests/r2-jsrpc-passthrough.capnp +++ b/src/workerd/api/tests/r2-jsrpc-passthrough.capnp @@ -9,7 +9,8 @@ # which is also the bucket name that R2 spans record. # # Each worker variant matches the mock's R2_LIST_HONOR_INCLUDE expectation to its -# r2_list_honor_include flag. +# r2_list_honor_include flag. The workers do not set a JSG struct mode, so the compatibility date +# of each test variant selects it. @0xe8a01684ab8486bc; @@ -30,7 +31,7 @@ const passthroughModules :List(Workerd.Worker.Module) = [ ( name = "r2-test.js", esModule = embed "r2-test.js" ), ]; -# r2_list_honor_include with slow structs. +# r2_list_honor_include. const passthroughWorker :Workerd.Worker = ( modules = .passthroughModules, bindings = [ @@ -38,27 +39,15 @@ const passthroughWorker :Workerd.Worker = ( ( name = "R2_LIST_HONOR_INCLUDE", text = "true" ), ], compatibilityFlags = ["nodejs_compat", "streams_enable_constructors", - "r2_list_honor_include", "disable_fast_jsg_struct"], + "r2_list_honor_include"], ); -# No r2_list_honor_include, with slow structs. +# No r2_list_honor_include. const passthroughLegacyListWorker :Workerd.Worker = ( modules = .passthroughModules, bindings = [ ( name = "REAL_BUCKET", r2Bucket = "r2-test" ), ( name = "R2_LIST_HONOR_INCLUDE", text = "false" ), ], - compatibilityFlags = ["nodejs_compat", "streams_enable_constructors", - "disable_fast_jsg_struct"], -); - -# No r2_list_honor_include, with fast structs. -const passthroughFastStructsWorker :Workerd.Worker = ( - modules = .passthroughModules, - bindings = [ - ( name = "REAL_BUCKET", r2Bucket = "r2-test" ), - ( name = "R2_LIST_HONOR_INCLUDE", text = "false" ), - ], - compatibilityFlags = ["nodejs_compat", "streams_enable_constructors", - "enable_fast_jsg_struct"], + compatibilityFlags = ["nodejs_compat", "streams_enable_constructors"], ); diff --git a/src/workerd/api/tests/r2-jsrpc-test.wd-test b/src/workerd/api/tests/r2-jsrpc-test.wd-test index c83a57092d2..6ffcafdff6b 100644 --- a/src/workerd/api/tests/r2-jsrpc-test.wd-test +++ b/src/workerd/api/tests/r2-jsrpc-test.wd-test @@ -29,7 +29,7 @@ const unitTests :Workerd.Config = ( ], compatibilityFlags = ["experimental", "nodejs_compat", "service_binding_extra_handlers", "streams_enable_constructors", "r2_list_honor_include", - "r2_binding_jsrpc", "disable_fast_jsg_struct"], + "r2_binding_jsrpc"], streamingTails = ["tail"], ) ), @@ -50,5 +50,5 @@ const tailWorker :Workerd.Worker = ( bindings = [ (name = "R2_TRACE_TRANSPORT", text = "jsrpc"), ], - compatibilityFlags = ["experimental", "nodejs_compat", "disable_fast_jsg_struct"], + compatibilityFlags = ["experimental", "nodejs_compat"], ); diff --git a/src/workerd/api/tests/r2-test.js b/src/workerd/api/tests/r2-test.js index b7786c0072a..7d9b6d49b12 100644 --- a/src/workerd/api/tests/r2-test.js +++ b/src/workerd/api/tests/r2-test.js @@ -328,10 +328,38 @@ function buildGetResponse({ head, body, isList, error } = {}) { }); } +// With the fastJsgStruct compatibility flag, a JSG struct such as R2HTTPMetadata or R2Range has +// every optional field as an own property, set to undefined when the field is unset. Without the +// flag, unset fields are absent. Dropping unset fields lets an assertion check the same shape in +// both modes. +function withoutUnsetFields(struct) { + return Object.fromEntries( + Object.entries(struct).filter(([, value]) => value !== undefined) + ); +} + +// Copies a list() result into plain objects that compare the same in both JSG struct modes. +function plainListResult(list) { + return { + ...withoutUnsetFields(list), + objects: list.objects.map((object) => { + const plain = { ...object, checksums: { ...object.checksums } }; + if (plain.httpMetadata !== undefined) { + plain.httpMetadata = withoutUnsetFields(plain.httpMetadata); + } + return plain; + }), + }; +} + async function compareResponse(res, { head, body } = {}, bytes) { // Destructuring syntax looks ugly, but gets around needing to construct HeadResponse objects(somehow?) const { ...obj } = await res; obj.checksums = { ...obj.checksums }; + obj.httpMetadata = withoutUnsetFields(obj.httpMetadata); + if (obj.range !== undefined) { + obj.range = withoutUnsetFields(obj.range); + } assert.deepEqual(obj, { ...HeadObject, ...head, @@ -1060,9 +1088,7 @@ const testWorker = { delimiter: '/', include: ['httpMetadata', 'customMetadata'], }); - list.objects[0] = { ...list.objects[0] }; - list.objects[0].checksums = { ...list.objects[0].checksums }; - assert.deepEqual(list, { + assert.deepEqual(plainListResult(list), { objects: [HeadObject], truncated: true, cursor: 'ai', @@ -1205,9 +1231,7 @@ const testWorker = { prefix: 'httpMeta', include: ['httpMetadata'], }); - list.objects[0] = { ...list.objects[0] }; - list.objects[0].checksums = { ...list.objects[0].checksums }; - assert.deepEqual(list, { + assert.deepEqual(plainListResult(list), { delimitedPrefixes: [], objects: [{ ...HeadObject, ...head }], truncated: false, @@ -1249,9 +1273,7 @@ const testWorker = { prefix: 'customMeta', include: ['customMetadata'], }); - list.objects[0] = { ...list.objects[0] }; - list.objects[0].checksums = { ...list.objects[0].checksums }; - assert.deepEqual(list, { + assert.deepEqual(plainListResult(list), { delimitedPrefixes: [], objects: [{ ...HeadObject, ...head }], truncated: false, @@ -1377,7 +1399,7 @@ const testWorker = { new Uint8Array(headResp.checksums.sha256), sha256Buffer ); - assert.deepStrictEqual(headResp.checksums.toJSON(), { + assert.deepStrictEqual(withoutUnsetFields(headResp.checksums.toJSON()), { md5: '9a0364b9e99bb480dd25e1f0284c8555', sha1: '2a0364b9e99bb480dd25e1f0284c855511223344', sha256: @@ -1799,7 +1821,10 @@ export const r2BindingApiTests = { assert.strictEqual(await env.BUCKET.get('missing'), null); const ranged = await env.BUCKET.head('ranged'); - assert.deepStrictEqual(ranged.range, { offset: 10, length: 20 }); + assert.deepStrictEqual(withoutUnsetFields(ranged.range), { + offset: 10, + length: 20, + }); const coerced = await env.BUCKET.head(12345); assert.strictEqual(coerced.key, key); @@ -1821,7 +1846,7 @@ export const r2BindingApiTests = { onlyIf: { etagMatches: 'objectEtag' }, }); assert.strictEqual(conditional.body, undefined); - assert.deepStrictEqual(conditional.httpMetadata, {}); + assert.deepStrictEqual(withoutUnsetFields(conditional.httpMetadata), {}); assert.deepStrictEqual(conditional.customMetadata, {}); assert.strictEqual(typeof conditional.writeHttpMetadata, 'function'); @@ -1848,7 +1873,7 @@ export const r2BindingApiTests = { assert.strictEqual(await (await blobResult.blob()).text(), body); const emptyList = await env.BUCKET.list(); - assert.deepStrictEqual(emptyList, { + assert.deepStrictEqual(plainListResult(emptyList), { objects: [], truncated: false, delimitedPrefixes: [], @@ -1862,7 +1887,7 @@ export const r2BindingApiTests = { startAfter: 'after', include: [], }); - assert.deepStrictEqual(optionList, { + assert.deepStrictEqual(plainListResult(optionList), { objects: [], truncated: true, cursor: 'cursor-out', @@ -1892,7 +1917,7 @@ export const r2BindingApiTests = { !include.includes('customMetadata') ); if (include.includes('httpMetadata')) { - assert.deepStrictEqual(object.httpMetadata, {}); + assert.deepStrictEqual(withoutUnsetFields(object.httpMetadata), {}); } if (include.includes('customMetadata')) { assert.deepStrictEqual(object.customMetadata, {}); diff --git a/src/workerd/api/tests/r2-test.wd-test b/src/workerd/api/tests/r2-test.wd-test index e76c75fb8b9..75e77e46e92 100644 --- a/src/workerd/api/tests/r2-test.wd-test +++ b/src/workerd/api/tests/r2-test.wd-test @@ -13,7 +13,7 @@ const unitTests :Workerd.Config = ( ( name = "R2_TRANSPORT", text = "http" ), ( name = "R2_LIST_HONOR_INCLUDE", text = "true" ), ], - compatibilityFlags = ["nodejs_compat", "service_binding_extra_handlers", "streams_enable_constructors", "r2_list_honor_include", "disable_fast_jsg_struct"], + compatibilityFlags = ["nodejs_compat", "service_binding_extra_handlers", "streams_enable_constructors", "r2_list_honor_include"], streamingTails = ["tail"], ) ), @@ -33,5 +33,5 @@ const tailWorker :Workerd.Worker = ( bindings = [ (name = "R2_TRACE_TRANSPORT", text = "http"), ], - compatibilityFlags = ["experimental", "nodejs_compat", "disable_fast_jsg_struct"], + compatibilityFlags = ["experimental", "nodejs_compat"], ); From cec2d551d35b2e249d6ae12ec35edbc9aea173a0 Mon Sep 17 00:00:00 2001 From: Carol Xu Date: Tue, 29 Sep 2026 09:58:30 -0500 Subject: [PATCH 04/13] Remove R2BindingEntrypoint mock and related tests --- src/workerd/api/tests/BUILD.bazel | 18 - .../r2-jsrpc-gateway-contract-test.wd-test | 29 - .../tests/r2-jsrpc-legacy-list-test.wd-test | 21 +- .../api/tests/r2-jsrpc-multipart-test.wd-test | 25 - .../api/tests/r2-jsrpc-range-test.wd-test | 37 -- src/workerd/api/tests/r2-jsrpc-test.wd-test | 4 - src/workerd/api/tests/r2-test.js | 536 +----------------- 7 files changed, 8 insertions(+), 662 deletions(-) delete mode 100644 src/workerd/api/tests/r2-jsrpc-gateway-contract-test.wd-test delete mode 100644 src/workerd/api/tests/r2-jsrpc-multipart-test.wd-test delete mode 100644 src/workerd/api/tests/r2-jsrpc-range-test.wd-test diff --git a/src/workerd/api/tests/BUILD.bazel b/src/workerd/api/tests/BUILD.bazel index 5f058c44aec..4e2d7e81721 100644 --- a/src/workerd/api/tests/BUILD.bazel +++ b/src/workerd/api/tests/BUILD.bazel @@ -561,18 +561,6 @@ wd_test( lint = False, ) -wd_test( - src = "r2-jsrpc-multipart-test.wd-test", - args = ["--experimental"], - data = ["r2-test.js"], -) - -wd_test( - src = "r2-jsrpc-range-test.wd-test", - args = ["--experimental"], - data = ["r2-test.js"], -) - wd_test( src = "r2-jsrpc-response-test.wd-test", args = ["--experimental"], @@ -599,12 +587,6 @@ wd_test( ], ) -wd_test( - src = "r2-jsrpc-gateway-contract-test.wd-test", - args = ["--experimental"], - data = ["r2-test.js"], -) - wd_test( src = "r2-jsrpc-legacy-list-test.wd-test", args = ["--experimental"], diff --git a/src/workerd/api/tests/r2-jsrpc-gateway-contract-test.wd-test b/src/workerd/api/tests/r2-jsrpc-gateway-contract-test.wd-test deleted file mode 100644 index 2a203700055..00000000000 --- a/src/workerd/api/tests/r2-jsrpc-gateway-contract-test.wd-test +++ /dev/null @@ -1,29 +0,0 @@ -using Workerd = import "/workerd/workerd.capnp"; - -# Checks the R2 binding's JSRPC client against R2BindingEntrypoint, a fake gateway that asserts -# the arguments each method receives and returns results a native R2 binding cannot produce, such -# as malformed objects. r2-jsrpc-test.wd-test runs the canonical and shared suites through a -# passthrough to a native binding. -const unitTests :Workerd.Config = ( - services = [ - ( name = "r2-jsrpc-gateway-contract-test", - worker = ( - modules = [ - ( name = "worker", - esModule = "export { R2BindingEntrypoint, jsrpcGatewayContractTests } from './r2-test.js';" ), - ( name = "r2-test.js", esModule = embed "r2-test.js" ), - ], - bindings = [ - ( name = "BUCKET", - r2Bucket = (name = "r2-jsrpc-gateway-contract-test", - entrypoint = "R2BindingEntrypoint") ), - ( name = "R2_TRANSPORT", text = "jsrpc" ), - ( name = "R2_LIST_HONOR_INCLUDE", text = "true" ), - ], - compatibilityFlags = ["experimental", "nodejs_compat", "service_binding_extra_handlers", - "streams_enable_constructors", "r2_list_honor_include", - "r2_binding_jsrpc", "disable_fast_jsg_struct"], - ) - ), - ], -); diff --git a/src/workerd/api/tests/r2-jsrpc-legacy-list-test.wd-test b/src/workerd/api/tests/r2-jsrpc-legacy-list-test.wd-test index 9c7af705795..12c8fab1ca1 100644 --- a/src/workerd/api/tests/r2-jsrpc-legacy-list-test.wd-test +++ b/src/workerd/api/tests/r2-jsrpc-legacy-list-test.wd-test @@ -5,8 +5,7 @@ using Passthrough = import "r2-jsrpc-passthrough.capnp"; # because a future compatibility date necessarily enables the behavior under test. # # The canonical and shared suites run through the passthrough in r2-jsrpc-passthrough.capnp, with -# the flag disabled for both the JSRPC client and the native binding. The gateway service checks -# the include list that the JSRPC client sends without the flag. +# the flag disabled for both the JSRPC client and the native binding. const unitTests :Workerd.Config = ( services = [ ( name = "user", @@ -23,23 +22,5 @@ const unitTests :Workerd.Config = ( ) ), ( name = "r2-test", worker = Passthrough.passthroughLegacyListWorker ), - ( name = "gateway", - worker = ( - modules = [ - ( name = "worker", - esModule = "export { R2BindingEntrypoint, jsrpcGatewayContractTests } from './r2-test.js';" ), - ( name = "r2-test.js", esModule = embed "r2-test.js" ), - ], - bindings = [ - ( name = "BUCKET", - r2Bucket = (name = "gateway", entrypoint = "R2BindingEntrypoint") ), - ( name = "R2_TRANSPORT", text = "jsrpc" ), - ( name = "R2_LIST_HONOR_INCLUDE", text = "false" ), - ], - compatibilityFlags = ["experimental", "nodejs_compat", "service_binding_extra_handlers", - "streams_enable_constructors", "r2_binding_jsrpc", - "disable_fast_jsg_struct"], - ) - ), ], ); diff --git a/src/workerd/api/tests/r2-jsrpc-multipart-test.wd-test b/src/workerd/api/tests/r2-jsrpc-multipart-test.wd-test deleted file mode 100644 index 8dba6134bb0..00000000000 --- a/src/workerd/api/tests/r2-jsrpc-multipart-test.wd-test +++ /dev/null @@ -1,25 +0,0 @@ -using Workerd = import "/workerd/workerd.capnp"; - -# Checks the multipart JSRPC client against R2BindingEntrypoint, a fake gateway that asserts the -# exact arguments the client sends. It uses the same struct mode as the other R2 JSRPC configs; -# the multipart wire inputs do not depend on property presence, so one mode is enough. -# r2-jsrpc-test.wd-test and r2-jsrpc-legacy-list-test.wd-test run r2MultipartApiTests through -# the passthrough to a native binding. -const unitTests :Workerd.Config = ( - services = [ - ( name = "gateway", - worker = ( - modules = [ - (name = "worker", esModule = "export { R2BindingEntrypoint, jsrpcGatewayMultipartContractTests } from './r2-test.js';"), - (name = "r2-test.js", esModule = embed "r2-test.js"), - ], - bindings = [ - (name = "BUCKET", r2Bucket = (name = "gateway", entrypoint = "R2BindingEntrypoint")), - (name = "R2_TRANSPORT", text = "jsrpc"), - ], - compatibilityFlags = ["experimental", "nodejs_compat", "service_binding_extra_handlers", - "streams_enable_constructors", "r2_binding_jsrpc", "disable_fast_jsg_struct"], - ), - ), - ], -); diff --git a/src/workerd/api/tests/r2-jsrpc-range-test.wd-test b/src/workerd/api/tests/r2-jsrpc-range-test.wd-test deleted file mode 100644 index 1834dbd9ca6..00000000000 --- a/src/workerd/api/tests/r2-jsrpc-range-test.wd-test +++ /dev/null @@ -1,37 +0,0 @@ -using Workerd = import "/workerd/workerd.capnp"; - -# The main R2 JSRPC suite disables fast structs. Exercise both modes here because fast structs -# materialize absent optional fields as undefined, while the gateway identifies suffix ranges -# by property presence. Offset/length ranges must reach the gateway without a suffix property. -const unitTests :Workerd.Config = ( - services = [ - ( name = "fast-structs", - worker = ( - modules = [ - (name = "worker", esModule = "export { R2BindingEntrypoint, jsrpcGatewayRangeContractTests } from './r2-test.js';"), - (name = "r2-test.js", esModule = embed "r2-test.js"), - ], - bindings = [ - (name = "BUCKET", r2Bucket = (name = "fast-structs", entrypoint = "R2BindingEntrypoint")), - (name = "R2_TRANSPORT", text = "jsrpc"), - ], - compatibilityFlags = ["experimental", "nodejs_compat", "service_binding_extra_handlers", - "r2_binding_jsrpc", "enable_fast_jsg_struct"], - ), - ), - ( name = "slow-structs", - worker = ( - modules = [ - (name = "worker", esModule = "export { R2BindingEntrypoint, jsrpcGatewayRangeContractTests } from './r2-test.js';"), - (name = "r2-test.js", esModule = embed "r2-test.js"), - ], - bindings = [ - (name = "BUCKET", r2Bucket = (name = "slow-structs", entrypoint = "R2BindingEntrypoint")), - (name = "R2_TRANSPORT", text = "jsrpc"), - ], - compatibilityFlags = ["experimental", "nodejs_compat", "service_binding_extra_handlers", - "r2_binding_jsrpc", "disable_fast_jsg_struct"], - ), - ), - ], -); diff --git a/src/workerd/api/tests/r2-jsrpc-test.wd-test b/src/workerd/api/tests/r2-jsrpc-test.wd-test index 6ffcafdff6b..8158b1aaa11 100644 --- a/src/workerd/api/tests/r2-jsrpc-test.wd-test +++ b/src/workerd/api/tests/r2-jsrpc-test.wd-test @@ -12,10 +12,6 @@ using Passthrough = import "r2-jsrpc-passthrough.capnp"; # "experimental" in the flag list here # - a binding that targets the named entrypoint, which is the production shape; a bare-string # designator would target the default export instead -# -# Tests that assert the gateway's wire shapes or malformed results need a fake gateway rather -# than a passthrough, so they run in the configs that target R2BindingEntrypoint -# (r2-jsrpc-gateway-contract-test.wd-test and the multipart and range configs). const unitTests :Workerd.Config = ( services = [ ( name = "user", diff --git a/src/workerd/api/tests/r2-test.js b/src/workerd/api/tests/r2-test.js index 7d9b6d49b12..c476fa999d0 100644 --- a/src/workerd/api/tests/r2-test.js +++ b/src/workerd/api/tests/r2-test.js @@ -8,16 +8,7 @@ import { WorkerEntrypoint } from 'cloudflare:workers'; const key = 'basicKey'; const body = 'content'; const rpcStreamBody = 'café'; -const rpcInlineBodyLimit = 16 << 20; -const largeRpcBodySize = rpcInlineBodyLimit + 2; const perKeyErrorDeleteKeys = ['existing', 'missing']; -const rpcRanges = { - offset: { offset: 1 }, - length: { length: 3 }, - bounded: { offset: 0, length: 3 }, - suffix: { suffix: 2 }, - empty: {}, -}; const httpMetaObj = { contentType: 'text/plain', contentLanguage: 'en-US', @@ -122,81 +113,19 @@ const invalidMetadata = { invalidChecksum: { checksums: { md5: {} } }, }; -function buildRpcHead(requestKey, multipartOptions) { - const result = { +// Object metadata in the shape a JSRPC R2 gateway returns. +function buildRpcHead() { + return { key, version: objResponse.version, size: Number(objResponse.size), etag: objResponse.etag, uploaded: new Date(Number(objResponse.uploaded)), - storageClass: multipartOptions?.storageClass ?? objResponse.storageClass, + storageClass: objResponse.storageClass, checksums: {}, httpMetadata: {}, - customMetadata: multipartOptions?.customMetadata ?? {}, + customMetadata: {}, }; - - if (multipartOptions?.httpMetadata !== undefined) { - result.httpMetadata = - multipartOptions.httpMetadata instanceof Headers - ? httpMetaObj - : multipartOptions.httpMetadata; - } - - switch (requestKey) { - case 'httpMetadata': - result.httpMetadata = httpMetaObj; - break; - case 'customMetadata': - result.customMetadata = customMetadata; - break; - case 'classInfrequentAccess': - result.storageClass = 'InfrequentAccess'; - break; - case 'ssec': - case 'ssecMultipart': - result.ssecKeyMd5 = keyMd5; - break; - case 'multipleChecksums': - result.checksums = { - md5: md5Buffer.buffer, - sha1: sha1Buffer.buffer, - sha256: sha256Buffer.buffer, - }; - break; - case 'ranged': - result.range = { offset: 10, length: 20 }; - break; - } - - return result; -} - -async function assertLargeRpcBody(requestKey, value, valueSize) { - assert(value instanceof ReadableStream); - assert.strictEqual(valueSize, largeRpcBodySize); - const uploaded = new Uint8Array(await new Response(value).arrayBuffer()); - assert.strictEqual(uploaded.byteLength, largeRpcBodySize); - - switch (requestKey) { - case 'largeBuffer': - assert.strictEqual(uploaded[0], 0x11); - assert.strictEqual(uploaded[1], 0x41); - assert.strictEqual(uploaded.at(-2), 0x41); - assert.strictEqual(uploaded.at(-1), 0x22); - break; - case 'largeString': - assert.strictEqual(uploaded[0], 0xc3); - assert.strictEqual(uploaded[1], 0xa9); - assert.strictEqual(uploaded.at(-2), 0xc3); - assert.strictEqual(uploaded.at(-1), 0xa9); - break; - case 'largeBlob': - assert.strictEqual(uploaded[0], 0x5a); - assert.strictEqual(uploaded.at(-1), 0x5a); - break; - default: - assert.fail(`unexpected large RPC body key: ${requestKey}`); - } } // The HTTP protocol reports failures with a status code and a CF-R2-Error header, which the @@ -1484,222 +1413,6 @@ export class ObservedR2Binding extends WorkerEntrypoint { } } -// A fake R2 gateway for the gateway contract tests (jsrpcGateway*ContractTests). Each method -// asserts the exact arguments the JSRPC client sends for the keys those tests use, and some keys -// return results that a native R2 binding cannot produce. It rejects HTTP requests to detect any -// transport fallback. -export class R2BindingEntrypoint extends WorkerEntrypoint { - fetch() { - throw new Error('R2 JSRPC tests must not use HTTP'); - } - - head(requestKey) { - const result = buildRpcHead(requestKey); - switch (requestKey) { - case 'rpc-malformed-size': - result.size = -1; - break; - case 'rpc-malformed-range': - result.range = { offset: -1, length: 1 }; - break; - case 'rpc-malformed-date': - result.uploaded = new Date(NaN); - break; - default: - assert.fail(`unexpected head key: ${requestKey}`); - } - return result; - } - - get(requestKey, options) { - if (requestKey.startsWith('rpc-range-')) { - const range = options?.range; - const rangeKey = requestKey.slice('rpc-range-'.length); - if (rangeKey === 'headers') { - assert(range instanceof Headers); - assert.strictEqual(range.get('range'), 'bytes=1-3'); - assert.strictEqual(range.get('x-test'), 'preserved'); - } else if (rangeKey === 'no-range') { - assert.strictEqual(range, undefined); - } else if (Object.hasOwn(rpcRanges, rangeKey)) { - assert.deepStrictEqual(range, rpcRanges[rangeKey]); - assert.strictEqual('suffix' in range, rangeKey === 'suffix'); - } else { - assert.fail( - 'Invalid ranges must be rejected before calling the gateway' - ); - } - return null; - } - switch (requestKey) { - case 'rpc-malformed-kind': - return { kind: 'other', object: buildRpcHead(requestKey) }; - case 'rpc-malformed-metadata-body': - return { - kind: 'metadata', - object: buildRpcHead(requestKey), - body: new ReadableStream(), - }; - case 'rpc-malformed-missing-body': - return { kind: 'body', object: buildRpcHead(requestKey) }; - case 'rpc-options': - assert.deepStrictEqual(options, { - onlyIf: { - etagMatches: 'strongEtag', - uploadedBefore: new Date(0), - secondsGranularity: false, - }, - range: { offset: 1, length: 3 }, - ssecKey: hexKey, - }); - return null; - case 'rpc-header-options': - assert(options.onlyIf instanceof Headers); - assert.strictEqual( - options.onlyIf.get('if-match'), - '"strongEtag", W/"weakEtag"' - ); - assert.strictEqual( - options.onlyIf.get('if-none-match'), - '"firstEtag", "secondEtag"' - ); - assert.strictEqual( - options.onlyIf.get('if-modified-since'), - new Date(0).toUTCString() - ); - assert(options.range instanceof Headers); - assert.strictEqual(options.range.get('range'), 'bytes=1-3'); - return null; - } - assert.fail(`unexpected get key: ${requestKey}`); - } - - async put(requestKey, value, options, valueSize) { - switch (requestKey) { - case 'rpc-null-value': - assert.strictEqual(value, null); - assert.strictEqual(valueSize, 0); - return buildRpcHead(requestKey); - case 'rpc-put-options': - assert.deepStrictEqual(options.onlyIf, { - etagMatches: 'strongEtag', - uploadedBefore: new Date(0), - secondsGranularity: false, - }); - assert.deepStrictEqual(options.httpMetadata, httpMetaObj); - assert.deepStrictEqual(options.customMetadata, customMetadata); - assert.deepStrictEqual(new Uint8Array(options.md5), md5Buffer); - assert.strictEqual(options.storageClass, 'InfrequentAccess'); - assert.strictEqual(options.ssecKey, hexKey); - break; - case 'rpc-put-header-options': - assert(options.onlyIf instanceof Headers); - assert.strictEqual( - options.onlyIf.get('if-match'), - '"strongEtag", W/"weakEtag"' - ); - assert.strictEqual( - options.onlyIf.get('if-none-match'), - '"firstEtag", "secondEtag"' - ); - break; - case 'largeBuffer': - case 'largeString': - case 'largeBlob': - await assertLargeRpcBody(requestKey, value, valueSize); - return buildRpcHead(requestKey, options); - case 'rpcStream': - break; - default: - assert.fail(`unexpected put key: ${requestKey}`); - } - const uploaded = await new Response(value).text(); - const expected = requestKey === 'rpcStream' ? rpcStreamBody : body; - assert.strictEqual(uploaded, expected); - assert.strictEqual( - valueSize, - new TextEncoder().encode(expected).byteLength - ); - return buildRpcHead(requestKey, options); - } - - list(options) { - const honorsIncludes = this.env.R2_LIST_HONOR_INCLUDE === 'true'; - const effectiveIncludes = honorsIncludes - ? (options?.include ?? []) - : ['httpMetadata', 'customMetadata']; - - if (options?.prefix === 'rpc-malformed') { - return { objects: [{}], truncated: false }; - } - if (options?.prefix === 'rpc-options') { - assert.deepStrictEqual(options, { - limit: 2, - prefix: 'rpc-options', - cursor: 'cursor-in', - delimiter: '/', - startAfter: 'after', - include: effectiveIncludes, - }); - return { - objects: [], - truncated: true, - cursor: 'cursor-out', - delimitedPrefixes: ['rpc-options/'], - }; - } - - if (honorsIncludes) { - assert.strictEqual(options, undefined); - } else { - assert.deepStrictEqual(options, { - include: ['httpMetadata', 'customMetadata'], - }); - } - return { objects: [], truncated: false }; - } - - createMultipartUpload(requestKey, options) { - assert.strictEqual(requestKey, 'rpc-create-options'); - assert.deepStrictEqual(options.httpMetadata, httpMetaObj); - assert.deepStrictEqual(options.customMetadata, customMetadata); - assert.strictEqual(options.storageClass, 'InfrequentAccess'); - assert.strictEqual(options.ssecKey, hexKey); - return 'multipartId'; - } - - async uploadPart( - requestKey, - uploadId, - partNumber, - value, - options, - valueSize - ) { - const expectedUploadId = { - rpcStream: 'streamedId', - largeBuffer: 'largeUploadId', - 'rpc-wrong-part-number': 'wrongPartUploadId', - }[requestKey]; - assert.strictEqual(uploadId, expectedUploadId); - if (requestKey === 'largeBuffer') { - await assertLargeRpcBody(requestKey, value, valueSize); - return { partNumber, etag: 'partEtag' }; - } - const uploaded = await new Response(value).text(); - const expected = requestKey === 'rpcStream' ? rpcStreamBody : body; - assert.strictEqual(uploaded, expected); - assert.strictEqual( - valueSize, - new TextEncoder().encode(expected).byteLength - ); - return { - partNumber: requestKey === 'rpc-wrong-part-number' ? 9999 : partNumber, - etag: 'partEtag', - }; - } -} - // Test that documents the behaviour of multi-key deletion where one fails and the other succeeds export const deletePerKeyErrorParityTests = { async test(_ctrl, env) { @@ -2109,241 +1822,6 @@ export const r2MultipartApiTests = { }, }; -// Gateway contract tests: they target R2BindingEntrypoint, a fake gateway that asserts the exact -// arguments the JSRPC client sends and returns results that a native R2 binding cannot produce. -// A passthrough to a native binding cannot observe or produce either. -export const jsrpcGatewayContractTests = { - async test(ctrl, env) { - if (env.R2_TRANSPORT !== 'jsrpc') { - return; - } - - await assert.rejects(env.BUCKET.head('rpc-malformed-size'), { - message: /^internal error; reference = \S+$/, - }); - await assert.rejects(env.BUCKET.head('rpc-malformed-range'), { - message: /^internal error; reference = \S+$/, - }); - await assert.rejects(env.BUCKET.head('rpc-malformed-date'), { - message: /^internal error; reference = \S+$/, - }); - - await env.BUCKET.get('rpc-options', { - onlyIf: { - etagMatches: 'strongEtag', - uploadedBefore: new Date(0), - }, - range: { offset: 1, length: 3 }, - ssecKey: bufferKey, - }); - await env.BUCKET.get('rpc-header-options', { - onlyIf: new Headers({ - 'if-match': '"strongEtag", W/"weakEtag"', - 'if-none-match': '"firstEtag", "secondEtag"', - 'if-modified-since': new Date(0).toUTCString(), - }), - range: new Headers({ range: 'bytes=1-3' }), - }); - - await assert.rejects(env.BUCKET.get('rpc-malformed-kind'), { - message: /^internal error; reference = \S+$/, - }); - await assert.rejects(env.BUCKET.get('rpc-malformed-metadata-body'), { - message: /^internal error; reference = \S+$/, - }); - // The missing-body assertion logs the details and exposes only an internal error reference. - await assert.rejects(env.BUCKET.get('rpc-malformed-missing-body'), { - message: /^internal error; reference = \S+$/, - }); - - // The gateway checks the list options, including the include list the compatibility flag - // selects. - assert.deepStrictEqual(await env.BUCKET.list(), { - objects: [], - truncated: false, - delimitedPrefixes: [], - }); - assert.deepStrictEqual( - await env.BUCKET.list({ - limit: 2, - prefix: 'rpc-options', - cursor: 'cursor-in', - delimiter: '/', - startAfter: 'after', - include: [], - }), - { - objects: [], - truncated: true, - cursor: 'cursor-out', - delimitedPrefixes: ['rpc-options/'], - } - ); - await assert.rejects(env.BUCKET.list({ prefix: 'rpc-malformed' }), { - message: /^internal error; reference = \S+$/, - }); - - const nullPut = await env.BUCKET.put('rpc-null-value', null); - assert.strictEqual(nullPut.key, key); - await env.BUCKET.put('rpc-put-options', body, { - onlyIf: { - etagMatches: 'strongEtag', - uploadedBefore: new Date(0), - }, - httpMetadata: httpMetaHeaders, - customMetadata, - md5: md5Buffer, - storageClass: 'InfrequentAccess', - ssecKey: bufferKey, - }); - await env.BUCKET.put('rpc-put-header-options', body, { - onlyIf: new Headers({ - 'if-match': '"strongEtag", W/"weakEtag"', - 'if-none-match': '"firstEtag", "secondEtag"', - }), - }); - - // The gateway checks the streamed body and its separately sent size. - const encodedStreamBody = new TextEncoder().encode(rpcStreamBody); - { - const { readable, writable } = new FixedLengthStream( - encodedStreamBody.byteLength - ); - const writer = writable.getWriter(); - const writing = writer - .write(encodedStreamBody) - .then(() => writer.close()); - await env.BUCKET.put('rpcStream', readable); - await writing; - } - - // Bodies above the inline limit are sent as streams. - const largeBufferBacking = new Uint8Array(largeRpcBodySize + 2); - largeBufferBacking.fill(0x41); - const largeBuffer = largeBufferBacking.subarray(1, -1); - largeBuffer[0] = 0x11; - largeBuffer[largeBuffer.length - 1] = 0x22; - await env.BUCKET.put('largeBuffer', largeBuffer); - - const largeString = 'é'.repeat(largeRpcBodySize / 2); - await env.BUCKET.put('largeString', largeString); - - const largeBlobBytes = new Uint8Array(largeRpcBodySize); - largeBlobBytes.fill(0x5a); - await env.BUCKET.put('largeBlob', new Blob([largeBlobBytes])); - }, -}; - -// Multipart gateway contract tests. See jsrpcGatewayContractTests. -export const jsrpcGatewayMultipartContractTests = { - async test(ctrl, env) { - if (env.R2_TRANSPORT !== 'jsrpc') { - return; - } - - const createOptionsUpload = await env.BUCKET.createMultipartUpload( - 'rpc-create-options', - { - httpMetadata: httpMetaHeaders, - customMetadata, - storageClass: 'InfrequentAccess', - ssecKey: bufferKey, - } - ); - assert.strictEqual(createOptionsUpload.uploadId, 'multipartId'); - - // The gateway checks the streamed body and its separately sent size. - const encodedStreamBody = new TextEncoder().encode(rpcStreamBody); - const streamedUpload = env.BUCKET.resumeMultipartUpload( - 'rpcStream', - 'streamedId' - ); - const { readable, writable } = new FixedLengthStream( - encodedStreamBody.byteLength - ); - const writer = writable.getWriter(); - const writing = writer.write(encodedStreamBody).then(() => writer.close()); - assert.deepStrictEqual(await streamedUpload.uploadPart(2, readable), { - partNumber: 2, - etag: 'partEtag', - }); - await writing; - - // Use a view with a non-zero byte offset to verify that the RPC upload streams only the view's - // bytes, rather than the entire underlying ArrayBuffer. - const largeBufferBacking = new Uint8Array(largeRpcBodySize + 2); - largeBufferBacking.fill(0x41); - const largeBuffer = largeBufferBacking.subarray(1, -1); - largeBuffer[0] = 0x11; - largeBuffer[largeBuffer.length - 1] = 0x22; - const largeUpload = env.BUCKET.resumeMultipartUpload( - 'largeBuffer', - 'largeUploadId' - ); - assert.deepStrictEqual(await largeUpload.uploadPart(1, largeBuffer), { - partNumber: 1, - etag: 'partEtag', - }); - - // The gateway reports a different part number; the result uses the requested one. - const wrongPartUpload = env.BUCKET.resumeMultipartUpload( - 'rpc-wrong-part-number', - 'wrongPartUploadId' - ); - assert.deepStrictEqual(await wrongPartUpload.uploadPart(3, body), { - partNumber: 3, - etag: 'partEtag', - }); - }, -}; - -// Range gateway contract tests. See jsrpcGatewayContractTests. -export const jsrpcGatewayRangeContractTests = { - async test(controller, env) { - if (env.R2_TRANSPORT !== 'jsrpc') { - return; - } - - for (const [rangeKey, range] of Object.entries(rpcRanges)) { - assert.strictEqual( - await env.BUCKET.get(`rpc-range-${rangeKey}`, { range }), - null - ); - } - assert.strictEqual( - await env.BUCKET.get('rpc-range-offset', { - range: { offset: 1, length: undefined, suffix: undefined }, - }), - null - ); - assert.strictEqual( - await env.BUCKET.get('rpc-range-suffix', { - range: { offset: undefined, length: undefined, suffix: 2 }, - }), - null - ); - assert.strictEqual( - await env.BUCKET.get('rpc-range-empty', { - range: { offset: undefined, length: undefined, suffix: undefined }, - }), - null - ); - - assert.strictEqual( - await env.BUCKET.get('rpc-range-headers', { - range: new Headers({ range: 'bytes=1-3', 'x-test': 'preserved' }), - }), - null - ); - assert.strictEqual(await env.BUCKET.get('rpc-range-no-range'), null); - assert.strictEqual(await env.BUCKET.get('rpc-range-no-range', {}), null); - assert.strictEqual( - await env.BUCKET.get('rpc-range-no-range', { range: new Headers() }), - null - ); - }, -}; - // The source-kind matrix uses the same four bytes and reports object.size = 4. It checks whether // the stream itself knows its byte count: PUT receives only the body, not object.size. const bodyLengthBytes = new TextEncoder().encode('data'); @@ -2503,7 +1981,7 @@ export class R2BodyLengthEntrypoint extends WorkerEntrypoint { } async get(mode, options) { - const object = { ...buildRpcHead(mode), size: bodyLengthBytes.byteLength }; + const object = { ...buildRpcHead(), size: bodyLengthBytes.byteLength }; if (mode === 'missing') { return null; } @@ -2609,7 +2087,7 @@ export class R2BodyLengthEntrypoint extends WorkerEntrypoint { expected ); return { - ...buildRpcHead(requestKey), + ...buildRpcHead(), key: requestKey, size: expected.byteLength, }; From df90b32d60422e5d4938cd8d75ba45a505de8bd1 Mon Sep 17 00:00:00 2001 From: Carol Xu Date: Thu, 1 Oct 2026 01:52:34 -0500 Subject: [PATCH 05/13] fixing various edge cases --- src/workerd/api/r2-bucket.c++ | 64 ++++++++---- src/workerd/api/r2-bucket.h | 12 ++- src/workerd/api/r2-rpc.c++ | 9 +- src/workerd/api/tests/BUILD.bazel | 21 ++++ .../api/tests/r2-jsrpc-serialization-test.js | 37 +++++++ .../tests/r2-jsrpc-serialization-test.wd-test | 42 ++++---- .../r2-jsrpc-serialization-ts-test.wd-test | 26 +++++ .../api/tests/r2-string-upload-test.wd-test | 32 ++++++ src/workerd/api/tests/r2-test.js | 98 +++++++++++++++++++ 9 files changed, 297 insertions(+), 44 deletions(-) create mode 100644 src/workerd/api/tests/r2-jsrpc-serialization-ts-test.wd-test create mode 100644 src/workerd/api/tests/r2-string-upload-test.wd-test diff --git a/src/workerd/api/r2-bucket.c++ b/src/workerd/api/r2-bucket.c++ index 1af64e88af1..ff5c22e42a2 100644 --- a/src/workerd/api/r2-bucket.c++ +++ b/src/workerd/api/r2-bucket.c++ @@ -714,6 +714,17 @@ jsg::Ref headResultFromSerializable( kj::mv(rpc.storageClass), kj::mv(rpc.ssecKeyMd5)); } +static uint64_t getResultBodyLength(R2Bucket::HeadResult& object) { + // Retrieve body length from range/size metadata to avoid depending on the stream to report it. + auto bodyLength = object.getSize(); + KJ_IF_SOME(range, object.getRange()) { + bodyLength = range.length.orDefault(bodyLength); + } + KJ_ASSERT(bodyLength <= 9007199254740991ull, + "Malformed R2 get RPC result: body length must be a safe integer."); + return static_cast(bodyLength); +} + static jsg::Ref getResultFromSerializable( jsg::Lock& js, R2Bucket::SerializableGetResult rpc) { KJ_ASSERT(rpc.kind == "body", "Malformed R2 get RPC result: expected a body result."); @@ -723,15 +734,8 @@ static jsg::Ref getResultFromSerializable( [&] { body.forceCancel(js, js.error("Malformed R2 get RPC result.")).markAsHandled(js); }); auto object = headResultFromSerializable(js, kj::mv(rpc.object), MissingMetadataPolicy::EMPTY); - // Retrieve body length from range/size metadata to avoid depending on the stream to report it. - auto bodyLength = object->getSize(); - KJ_IF_SOME(range, object->getRange()) { - bodyLength = range.length.orDefault(bodyLength); - } - KJ_ASSERT(bodyLength <= 9007199254740991ull, - "Malformed R2 get RPC result: body length must be a safe integer."); auto& context = IoContext::current(); - auto pipe = newIdentityPipe(static_cast(bodyLength)); + auto pipe = newIdentityPipe(getResultBodyLength(*object)); auto pump = context.waitForDeferredProxy(body.pumpTo(js, kj::mv(pipe.out), EndStream::YES)); body = JsReadableStream::create( js, context, kj::heap(kj::mv(pipe.in), kj::mv(pump))); @@ -2120,6 +2124,7 @@ void R2Bucket::HeadResult::writeHttpMetadata(jsg::Lock& js, Headers& headers) { jsg::Promise> R2Bucket::GetResult::arrayBuffer(jsg::Lock& js) { return js.evalNow([&] { + ensureBodyLength(js); auto& context = IoContext::current(); return body.arrayBuffer(js, context.getLimitEnforcer().getBufferingLimit()); }); @@ -2127,6 +2132,7 @@ jsg::Promise> R2Bucket::GetResult::arrayBuffer(js jsg::Promise> R2Bucket::GetResult::bytes(jsg::Lock& js) { return js.evalNow([&] { + ensureBodyLength(js); auto& context = IoContext::current(); return body.bytes(js, context.getLimitEnforcer().getBufferingLimit()); }); @@ -2135,6 +2141,7 @@ jsg::Promise> R2Bucket::GetResult::bytes(jsg::Lock jsg::Promise R2Bucket::GetResult::text(jsg::Lock& js) { // Copy-pasted from http.c++ return js.evalNow([&] { + ensureBodyLength(js); // Check for a disturbed body before emitting the non-text warning below. (body.text() // performs the same check with the same error message; this one just runs first.) JSG_REQUIRE(!body.isDisturbed(js), TypeError, @@ -2196,9 +2203,8 @@ jsg::Ref R2Bucket::Checksums::deserialize(jsg::Lock& js, jsg::Deserializer& deserializer, const jsg::TypeHandler& checksumsHandler) { requireR2RpcDeserializer(deserializer); - auto checksums = KJ_UNWRAP_OR(checksumsHandler.tryUnwrap(js, deserializer.readValue(js)), { - JSG_FAIL_REQUIRE(DOMDataCloneError, "Deserialization failed: invalid R2 checksums payload"); - }); + auto checksums = KJ_ASSERT_NONNULL(checksumsHandler.tryUnwrap(js, deserializer.readValue(js)), + "Deserialization failed: invalid R2 checksums payload"); return js.alloc(kj::mv(checksums.md5), kj::mv(checksums.sha1), kj::mv(checksums.sha256), kj::mv(checksums.sha384), kj::mv(checksums.sha512)); } @@ -2250,16 +2256,27 @@ jsg::Ref R2Bucket::HeadResult::deserialize(jsg::Lock& js, jsg::Deserializer& deserializer, const jsg::TypeHandler& headResultHandler) { requireR2RpcDeserializer(deserializer); - auto headResult = KJ_UNWRAP_OR(headResultHandler.tryUnwrap(js, deserializer.readValue(js)), { - JSG_FAIL_REQUIRE(DOMDataCloneError, "Deserialization failed: invalid R2 object payload"); - }); + auto headResult = KJ_ASSERT_NONNULL(headResultHandler.tryUnwrap(js, deserializer.readValue(js)), + "Deserialization failed: invalid R2 object payload"); return headResultFromSerializable(js, kj::mv(headResult), MissingMetadataPolicy::ABSENT); } +void R2Bucket::GetResult::ensureBodyLength(jsg::Lock& js) { + KJ_IF_SOME(length, expectedBodyLength) { + auto& context = IoContext::current(); + auto pipe = newIdentityPipe(length); + auto pump = context.waitForDeferredProxy(body.pumpTo(js, kj::mv(pipe.out), EndStream::YES)); + body = JsReadableStream::create( + js, context, kj::heap(kj::mv(pipe.in), kj::mv(pump))); + expectedBodyLength = kj::none; + } +} + void R2Bucket::GetResult::serialize(jsg::Lock& js, jsg::Serializer& serializer, const jsg::TypeHandler& getResultHandler) { requireR2RpcSerializer(serializer); + ensureBodyLength(js); serializer.write(js, jsg::JsValue(getResultHandler.wrap(js, SerializableGetResult{ @@ -2274,14 +2291,21 @@ jsg::Ref R2Bucket::GetResult::deserialize(jsg::Lock& js, jsg::Deserializer& deserializer, const jsg::TypeHandler& getResultHandler) { requireR2RpcDeserializer(deserializer); - auto getResult = KJ_UNWRAP_OR(getResultHandler.tryUnwrap(js, deserializer.readValue(js)), { - JSG_FAIL_REQUIRE(DOMDataCloneError, "Deserialization failed: invalid R2 object body payload"); - }); - JSG_REQUIRE(getResult.kind == "body", DOMDataCloneError, + auto getResult = KJ_ASSERT_NONNULL(getResultHandler.tryUnwrap(js, deserializer.readValue(js)), + "Deserialization failed: invalid R2 object body payload"); + KJ_ASSERT(getResult.kind == "body", "Deserialization failed: R2 object body payload has an invalid kind"); - JSG_REQUIRE(getResult.body != kj::none, DOMDataCloneError, + KJ_ASSERT(getResult.body != kj::none, "Deserialization failed: R2 object body payload is missing its body"); - return getResultFromSerializable(js, kj::mv(getResult)); + auto object = + headResultFromSerializable(js, kj::mv(getResult.object), MissingMetadataPolicy::EMPTY); + auto bodyLength = getResultBodyLength(*object); + auto customMetadata = object->getCustomMetadata(); + return js.alloc(kj::str(object->getName()), kj::str(object->getVersion()), + object->getSize(), kj::str(object->getEtag()), object->getChecksums(), object->getUploaded(), + object->getHttpMetadata(), kj::mv(customMetadata), object->getRange(), + kj::str(object->getStorageClass()), mapCopyString(object->getSSECKeyMd5()), + KJ_ASSERT_NONNULL(kj::mv(getResult.body)), bodyLength); } R2Bucket::StringChecksums R2Bucket::Checksums::toJSON() { diff --git a/src/workerd/api/r2-bucket.h b/src/workerd/api/r2-bucket.h index e2f728f6a52..e67093f8ad5 100644 --- a/src/workerd/api/r2-bucket.h +++ b/src/workerd/api/r2-bucket.h @@ -540,7 +540,8 @@ class R2Bucket: public jsg::Object { jsg::Optional range, kj::String storageClass, jsg::Optional ssecKeyMd5, - JsReadableStream body) + JsReadableStream body, + kj::Maybe expectedBodyLength = kj::none) : HeadResult(kj::mv(name), kj::mv(version), size, @@ -552,9 +553,11 @@ class R2Bucket: public jsg::Object { range, kj::mv(storageClass), kj::mv(ssecKeyMd5)), - body(kj::mv(body)) {} + body(kj::mv(body)), + expectedBodyLength(expectedBodyLength) {} JsReadableStream getBody(jsg::Lock& js) { + ensureBodyLength(js); return body.addRef(js); } @@ -600,6 +603,11 @@ class R2Bucket: public jsg::Object { private: JsReadableStream body; + // RPC deserialization cannot run the JavaScript needed to wrap a TypeScript stream. + // Its length is restored on first use, after the deserializer finishes reading the graph. + kj::Maybe expectedBodyLength; + + void ensureBodyLength(jsg::Lock& js); void visitForGc(jsg::GcVisitor& visitor) { visitor.visit(body); diff --git a/src/workerd/api/r2-rpc.c++ b/src/workerd/api/r2-rpc.c++ index 4189034b047..49f56d4f1da 100644 --- a/src/workerd/api/r2-rpc.c++ +++ b/src/workerd/api/r2-rpc.c++ @@ -21,6 +21,9 @@ namespace workerd::api { namespace { constexpr size_t R2_RPC_INLINE_BODY_LIMIT = 16u << 20; +// V8's UTF-16 string serialization can use twice the UTF-8 byte count. Keep strings within +// the same inline budget as binary values, leaving room for the other RPC arguments. +constexpr size_t R2_RPC_INLINE_STRING_LIMIT = R2_RPC_INLINE_BODY_LIMIT / 2; JsReadableStream makeR2RpcMemoryStream( jsg::Lock& js, kj::ArrayPtr bytes, kj::Maybe> backing = kj::none) { @@ -76,7 +79,7 @@ PreparedR2RpcBody prepareR2RpcBody(jsg::Lock& js, R2PutValue& value) { } KJ_CASE_ONEOF(text, jsg::NonCoercible) { auto size = text.value.size(); - if (size > R2_RPC_INLINE_BODY_LIMIT) { + if (size > R2_RPC_INLINE_STRING_LIMIT) { auto bytes = text.value.asBytes(); return {.value = makeR2RpcMemoryStream(js, bytes, kj::heap(kj::mv(text.value))), .size = static_cast(size)}; @@ -128,8 +131,8 @@ jsg::Ref R2Error::deserialize(jsg::Lock& js, jsg::Deserializer& deserializer, const jsg::TypeHandler& payloadHandler) { requireR2RpcDeserializer(deserializer); - auto payload = KJ_UNWRAP_OR(payloadHandler.tryUnwrap(js, deserializer.readValue(js)), - { JSG_FAIL_REQUIRE(DOMDataCloneError, "Deserialization failed: invalid R2 error payload"); }); + auto payload = KJ_ASSERT_NONNULL(payloadHandler.tryUnwrap(js, deserializer.readValue(js)), + "Deserialization failed: invalid R2 error payload"); auto result = js.alloc(payload.code, kj::mv(payload.message)); result->action = kj::mv(payload.action); result->errorForStack = v8::Global( diff --git a/src/workerd/api/tests/BUILD.bazel b/src/workerd/api/tests/BUILD.bazel index 4e2d7e81721..1024088fca5 100644 --- a/src/workerd/api/tests/BUILD.bazel +++ b/src/workerd/api/tests/BUILD.bazel @@ -576,6 +576,17 @@ wd_test( ], ) +wd_test( + src = "r2-jsrpc-serialization-ts-test.wd-test", + args = ["--experimental"], + data = [ + "r2-jsrpc-serialization-test.js", + "r2-jsrpc-serialization-test.wd-test", + "r2-test.js", + ], + lint = False, +) + wd_test( src = "r2-jsrpc-test.wd-test", args = ["--experimental"], @@ -597,6 +608,16 @@ wd_test( generate_all_compat_flags_variant = False, ) +wd_test( + size = "large", + src = "r2-string-upload-test.wd-test", + args = ["--experimental"], + data = [ + "r2-jsrpc-passthrough.capnp", + "r2-test.js", + ], +) + wd_test( src = "r2-test.wd-test", args = ["--experimental"], diff --git a/src/workerd/api/tests/r2-jsrpc-serialization-test.js b/src/workerd/api/tests/r2-jsrpc-serialization-test.js index 8417e346819..e76a132fc91 100644 --- a/src/workerd/api/tests/r2-jsrpc-serialization-test.js +++ b/src/workerd/api/tests/r2-jsrpc-serialization-test.js @@ -41,6 +41,10 @@ export class ObservedR2Binding extends PassthroughR2Binding { getNative(requestKey) { return this.env.REAL_BUCKET.get(requestKey); } + + echoNative(object) { + return object; + } } export const nativeR2ResultTests = { @@ -85,10 +89,43 @@ export const nativeR2ResultTests = { const directGet = await env.PLATFORM.getNative('rpc-json'); assert.throws(() => structuredClone(directGet), { name: 'DataCloneError' }); assert.deepStrictEqual(await directGet.json(), { ok: true }); + assert.strictEqual(directGet.bodyUsed, true); const object = await env.BUCKET.get('rpc-json'); assert.strictEqual(typeof object.text, 'function'); + assert.strictEqual(object.bodyUsed, false); + const objectBody = object.body; + assert.strictEqual(object.body, objectBody); + assert.strictEqual(object.bodyUsed, false); + assert.strictEqual(objectBody.locked, false); assert.deepStrictEqual(await object.json(), { ok: true }); + assert.strictEqual(object.body, objectBody); + assert.strictEqual(object.bodyUsed, true); + + // Each consumption method must work without first accessing the body getter. + for (const method of ['arrayBuffer', 'bytes', 'text', 'blob']) { + const result = await env.PLATFORM.getNative('rpc-lazy-body'); + const consumed = await result[method](); + assert.strictEqual(await new Response(consumed).text(), 'content'); + assert.strictEqual(result.bodyUsed, true); + } + + // A deserialized native result must be serializable again before its body is accessed. + const echoed = await env.PLATFORM.echoNative( + await env.PLATFORM.getNative('rpc-json') + ); + assert.deepStrictEqual(await echoed.json(), { ok: true }); + + // Both service RPC and an R2 interceptor preserve the length for immediate streaming copies. + for (const get of [ + () => env.PLATFORM.getNative('rpc-lazy-body'), + () => env.BUCKET.get('rpc-lazy-body'), + ]) { + const source = await get(); + const copied = await env.BUCKET.put('basicKey', source.body); + assert.strictEqual(copied.size, 7); + assert.strictEqual(source.bodyUsed, true); + } const metadataOnly = await env.BUCKET.get('rpc-conditional-metadata'); assert.strictEqual(metadataOnly.key, 'basicKey'); diff --git a/src/workerd/api/tests/r2-jsrpc-serialization-test.wd-test b/src/workerd/api/tests/r2-jsrpc-serialization-test.wd-test index 6300b136397..f4a22070cc6 100644 --- a/src/workerd/api/tests/r2-jsrpc-serialization-test.wd-test +++ b/src/workerd/api/tests/r2-jsrpc-serialization-test.wd-test @@ -12,11 +12,7 @@ using Workerd = import "/workerd/workerd.capnp"; const unitTests :Workerd.Config = ( services = [ (name = "user", worker = ( - modules = [ - (name = "worker", esModule = "export { nativeR2ResultTests } from './test.js';"), - (name = "test.js", esModule = embed "r2-jsrpc-serialization-test.js"), - (name = "r2-test.js", esModule = embed "r2-test.js"), - ], + modules = .testModules, bindings = [ (name = "BUCKET", r2Bucket = (name = "platform", entrypoint = "ObservedR2Binding")), @@ -27,19 +23,27 @@ const unitTests :Workerd.Config = ( "r2_list_honor_include", "r2_binding_jsrpc", "rpc", "fetcher_no_get_put_delete"], )), - # Exports no test handlers, so the test runner does not run the tests in r2-test.js here. - (name = "platform", worker = ( - modules = [ - (name = "worker", - esModule = "import mock from './r2-test.js'; export { ObservedR2Binding } from './test.js'; export default { fetch: mock.fetch };"), - (name = "test.js", esModule = embed "r2-jsrpc-serialization-test.js"), - (name = "r2-test.js", esModule = embed "r2-test.js"), - ], - bindings = [ - (name = "REAL_BUCKET", r2Bucket = "platform"), - ], - compatibilityFlags = ["nodejs_compat", "streams_enable_constructors", - "r2_list_honor_include"], - )), + (name = "platform", worker = .platformWorker), + ], +); + +const testModules :List(Workerd.Worker.Module) = [ + (name = "worker", esModule = "export { nativeR2ResultTests } from './test.js';"), + (name = "test.js", esModule = embed "r2-jsrpc-serialization-test.js"), + (name = "r2-test.js", esModule = embed "r2-test.js"), +]; + +# Exports no test handlers, so the test runner does not run the tests in r2-test.js here. +const platformWorker :Workerd.Worker = ( + modules = [ + (name = "worker", + esModule = "import mock from './r2-test.js'; export { ObservedR2Binding } from './test.js'; export default { fetch: mock.fetch };"), + (name = "test.js", esModule = embed "r2-jsrpc-serialization-test.js"), + (name = "r2-test.js", esModule = embed "r2-test.js"), + ], + bindings = [ + (name = "REAL_BUCKET", r2Bucket = "platform"), ], + compatibilityFlags = ["nodejs_compat", "streams_enable_constructors", + "r2_list_honor_include"], ); diff --git a/src/workerd/api/tests/r2-jsrpc-serialization-ts-test.wd-test b/src/workerd/api/tests/r2-jsrpc-serialization-ts-test.wd-test new file mode 100644 index 00000000000..e0d84622ed4 --- /dev/null +++ b/src/workerd/api/tests/r2-jsrpc-serialization-ts-test.wd-test @@ -0,0 +1,26 @@ +using Workerd = import "/workerd/workerd.capnp"; +using NativeR2 = import "r2-jsrpc-serialization-test.wd-test"; + +# Native R2 object bodies cross the RPC deserializer's no-JavaScript graph-read scope. +# The TypeScript stream receiver requires its streams to be hydrated before that scope. +const unitTests :Workerd.Config = ( + services = [ + (name = "user", worker = ( + modules = NativeR2.testModules, + bindings = [ + (name = "BUCKET", + r2Bucket = (name = "platform", entrypoint = "ObservedR2Binding")), + (name = "PLATFORM", + service = (name = "platform", entrypoint = "ObservedR2Binding")), + ], + compatibilityFlags = ["experimental", "nodejs_compat", "streams_enable_constructors", + "r2_list_honor_include", "r2_binding_jsrpc", "rpc", + "fetcher_no_get_put_delete", "typescript_implemented_streams"], + )), + (name = "platform", worker = NativeR2.platformWorker), + ], + autogates = [ + "workerd-autogate-per-isolate-javascript-bootstrap", + "workerd-autogate-rpc-externals-hydration", + ], +); diff --git a/src/workerd/api/tests/r2-string-upload-test.wd-test b/src/workerd/api/tests/r2-string-upload-test.wd-test new file mode 100644 index 00000000000..faab23807bb --- /dev/null +++ b/src/workerd/api/tests/r2-string-upload-test.wd-test @@ -0,0 +1,32 @@ +using Workerd = import "/workerd/workerd.capnp"; +using Passthrough = import "r2-jsrpc-passthrough.capnp"; + +# Both transports upload strings to the same HTTP mock, which verifies the UTF-8 bytes and +# declared length. The JSRPC route forwards through the native R2 interceptor. +const unitTests :Workerd.Config = ( + services = [ + (name = "http", worker = ( + modules = .testModules, + bindings = [ + (name = "BUCKET", r2Bucket = "r2-test"), + (name = "R2_STRING_UPLOAD_TEST", text = "true"), + ], + compatibilityFlags = ["nodejs_compat", "streams_enable_constructors"], + )), + (name = "jsrpc", worker = ( + modules = .testModules, + bindings = [ + (name = "BUCKET", r2Bucket = (name = "r2-test", entrypoint = "ObservedR2Binding")), + (name = "R2_STRING_UPLOAD_TEST", text = "true"), + ], + compatibilityFlags = ["experimental", "nodejs_compat", "streams_enable_constructors", + "r2_binding_jsrpc"], + )), + (name = "r2-test", worker = Passthrough.passthroughWorker), + ], +); + +const testModules :List(Workerd.Worker.Module) = [ + (name = "worker", esModule = "export { r2StringPutTests, r2StringUploadPartTests } from './r2-test.js';"), + (name = "r2-test.js", esModule = embed "r2-test.js"), +]; diff --git a/src/workerd/api/tests/r2-test.js b/src/workerd/api/tests/r2-test.js index c476fa999d0..9f2c5387a06 100644 --- a/src/workerd/api/tests/r2-test.js +++ b/src/workerd/api/tests/r2-test.js @@ -3,11 +3,19 @@ // https://opensource.org/licenses/Apache-2.0 import assert from 'node:assert'; +import { Buffer } from 'node:buffer'; import { WorkerEntrypoint } from 'cloudflare:workers'; const key = 'basicKey'; const body = 'content'; const rpcStreamBody = 'café'; +const stringUploadCases = { + 'string-upload-below-limit': { size: (8 << 20) - 1, suffix: 'Ā' }, + 'string-upload-at-limit': { size: 8 << 20, suffix: 'Ā' }, + 'string-upload-above-limit': { size: (8 << 20) + 1, suffix: 'Ā' }, + 'string-upload-rpc-limit': { size: 16 << 20, suffix: 'Ā' }, + 'string-upload-ascii': { size: 16 << 20, suffix: '' }, +}; const perKeyErrorDeleteKeys = ['existing', 'missing']; const httpMetaObj = { contentType: 'text/plain', @@ -220,6 +228,62 @@ function handleTrackedMultipartRequest(jsonRequest, uploadedBody) { throw new Error(`Unexpected method: ${jsonRequest.method}`); } +async function handleStringUploadRequest(request, jsonRequest, metadataSize) { + const expected = stringUploadCases[jsonRequest.object]; + const suffix = new TextEncoder().encode(expected.suffix); + const asciiLength = expected.size - suffix.byteLength; + assert.strictEqual( + request.headers.get('content-length'), + String(metadataSize + expected.size) + ); + + // Check every byte without retaining another full copy of the large upload. + const ascii = new Uint8Array(1 << 20).fill(0x61); + const reader = request.body.getReader({ mode: 'byob' }); + let received = 0; + try { + for (;;) { + const { value: chunk, done } = await reader.readAtLeast( + ascii.length, + new Uint8Array(ascii.length) + ); + if (done) break; + const asciiEnd = Math.min( + chunk.length, + Math.max(asciiLength - received, 0) + ); + assert.strictEqual( + Buffer.compare( + chunk.subarray(0, asciiEnd), + ascii.subarray(0, asciiEnd) + ), + 0 + ); + const suffixStart = Math.max(received - asciiLength, 0); + assert.deepStrictEqual( + chunk.subarray(asciiEnd), + suffix.subarray(suffixStart, suffixStart + chunk.length - asciiEnd) + ); + received += chunk.length; + } + } finally { + reader.releaseLock(); + } + assert.strictEqual(received, expected.size); + + if (jsonRequest.method === 'put') { + return Response.json({ + ...objResponse, + name: jsonRequest.object, + size: String(received), + }); + } + assert.strictEqual(jsonRequest.method, 'uploadPart'); + assert.strictEqual(jsonRequest.uploadId, 'string-upload-id'); + assert.strictEqual(jsonRequest.partNumber, 1); + return Response.json({ etag: `string-upload-${received}` }); +} + // An R2 error makes this an error response, such as a failed precondition that still carries the // object's metadata but no body. function buildGetResponse({ head, body, isList, error } = {}) { @@ -336,6 +400,9 @@ const testWorker = { if (jsonRequest.object === 'must-not-call') { assert.fail('R2 backend must not be called for invalid input'); } + if (Object.hasOwn(stringUploadCases, jsonRequest.object)) { + return handleStringUploadRequest(request, jsonRequest, metadataSize); + } // The metadata has already been read from the body, so a Response cannot wrap it. const chunks = []; @@ -1822,6 +1889,37 @@ export const r2MultipartApiTests = { }, }; +function stringUploadValue({ size, suffix }) { + return ( + 'a'.repeat(size - new TextEncoder().encode(suffix).byteLength) + suffix + ); +} + +export const r2StringPutTests = { + async test(_ctrl, env) { + if (env.R2_STRING_UPLOAD_TEST !== 'true') return; + for (const [key, fixture] of Object.entries(stringUploadCases)) { + const result = await env.BUCKET.put(key, stringUploadValue(fixture)); + assert.strictEqual(result.key, key); + assert.strictEqual(result.size, fixture.size); + } + }, +}; + +export const r2StringUploadPartTests = { + async test(_ctrl, env) { + if (env.R2_STRING_UPLOAD_TEST !== 'true') return; + for (const [key, fixture] of Object.entries(stringUploadCases)) { + const upload = env.BUCKET.resumeMultipartUpload(key, 'string-upload-id'); + const result = await upload.uploadPart(1, stringUploadValue(fixture)); + assert.deepStrictEqual(result, { + partNumber: 1, + etag: `string-upload-${fixture.size}`, + }); + } + }, +}; + // The source-kind matrix uses the same four bytes and reports object.size = 4. It checks whether // the stream itself knows its byte count: PUT receives only the body, not object.size. const bodyLengthBytes = new TextEncoder().encode('data'); From b6f4efa9b6940749115adcdaa3e9d7bddbbd0953 Mon Sep 17 00:00:00 2001 From: Carol Xu Date: Mon, 5 Oct 2026 14:06:31 -0500 Subject: [PATCH 06/13] Remove trace spans from R2 JSRPC methods The JSRPC methods on R2Bucket and R2MultipartUpload no longer create R2 user trace spans or set request/response tags. The HTTP-backed methods keep their spans unchanged. The JSRPC spans could not report cloudflare.r2.response.success or the R2 error tags, because the gateway's RPC results do not carry the backend status. Rather than emit spans that diverge from the HTTP transport, drop them until the RPC protocol can support parity. Drop addHttpMetadataRequestSpanTags and the TraceContext parameter of makeSerializableConditional, which only the JSRPC methods used. r2-jsrpc-test no longer attaches the instrumentation tail worker, and r2-instrumentation-test compares the HTTP transport's spans exactly. --- src/workerd/api/r2-bucket.c++ | 171 ++++-------------- src/workerd/api/r2-multipart.c++ | 37 +--- src/workerd/api/tests/BUILD.bazel | 2 - .../api/tests/r2-instrumentation-test.js | 21 +-- src/workerd/api/tests/r2-jsrpc-test.wd-test | 17 -- src/workerd/api/tests/r2-test.wd-test | 3 - 6 files changed, 38 insertions(+), 213 deletions(-) diff --git a/src/workerd/api/r2-bucket.c++ b/src/workerd/api/r2-bucket.c++ index ff5c22e42a2..c87d957b127 100644 --- a/src/workerd/api/r2-bucket.c++ +++ b/src/workerd/api/r2-bucket.c++ @@ -333,25 +333,19 @@ kj::String buildEtagsString(kj::ArrayPtr etagArray) { } R2Bucket::SerializableConditional makeSerializableConditional( - jsg::Lock& js, R2Bucket::UnwrappedConditional& conditional, TraceContext& traceContext) { + R2Bucket::UnwrappedConditional& conditional) { R2Bucket::SerializableConditional rpcConditional{ .secondsGranularity = conditional.secondsGranularity}; KJ_IF_SOME(etags, conditional.etagMatches) { - auto value = buildEtagsString(etags); - traceContext.setTag("cloudflare.r2.request.only_if.etag_matches"_kjc, value.asPtr()); - rpcConditional.etagMatches = kj::mv(value); + rpcConditional.etagMatches = buildEtagsString(etags); } KJ_IF_SOME(etags, conditional.etagDoesNotMatch) { - auto value = buildEtagsString(etags); - traceContext.setTag("cloudflare.r2.request.only_if.etag_does_not_match"_kjc, value.asPtr()); - rpcConditional.etagDoesNotMatch = kj::mv(value); + rpcConditional.etagDoesNotMatch = buildEtagsString(etags); } KJ_IF_SOME(date, conditional.uploadedBefore) { - traceContext.setTag("cloudflare.r2.request.only_if.uploaded_before"_kjc, toISOString(js, date)); rpcConditional.uploadedBefore = date; } KJ_IF_SOME(date, conditional.uploadedAfter) { - traceContext.setTag("cloudflare.r2.request.only_if.uploaded_after"_kjc, toISOString(js, date)); rpcConditional.uploadedAfter = date; } return rpcConditional; @@ -440,35 +434,11 @@ static R2Bucket::HttpMetadata normalizeHttpMetadata( /***************************** * * Helper functions to set various tags on traceContext. - * - addHttpMetadataRequestSpanTags * - addR2ResponseSpanTags * - addHeadResultSpanTags + * - addListResultSpanTags * ******************************/ -static void addHttpMetadataRequestSpanTags( - TraceContext& traceContext, const R2Bucket::HttpMetadata& metadata) { - KJ_IF_SOME(value, metadata.contentType) { - traceContext.setTag("cloudflare.r2.request.http_metadata.content_type"_kjc, value.asPtr()); - } - KJ_IF_SOME(value, metadata.contentEncoding) { - traceContext.setTag("cloudflare.r2.request.http_metadata.content_encoding"_kjc, value.asPtr()); - } - KJ_IF_SOME(value, metadata.contentDisposition) { - traceContext.setTag( - "cloudflare.r2.request.http_metadata.content_disposition"_kjc, value.asPtr()); - } - KJ_IF_SOME(value, metadata.contentLanguage) { - traceContext.setTag("cloudflare.r2.request.http_metadata.content_language"_kjc, value.asPtr()); - } - KJ_IF_SOME(value, metadata.cacheControl) { - traceContext.setTag("cloudflare.r2.request.http_metadata.cache_control"_kjc, value.asPtr()); - } - KJ_IF_SOME(value, metadata.cacheExpiry) { - traceContext.setTag("cloudflare.r2.request.http_metadata.cache_expiry"_kjc, - (value - kj::UNIX_EPOCH) / kj::MILLISECONDS); - } -} - static void addR2ResponseSpanTags(TraceContext& traceContext, R2Result& r2Result) { traceContext.setTag("cloudflare.r2.response.success"_kjc, r2Result.success()); KJ_IF_SOME(e, r2Result.getR2ErrorMessage()) { @@ -782,19 +752,15 @@ jsg::Promise>> R2Bucket::headRpc(jsg::L const jsg::TypeHandler>& headFnHandler, const jsg::TypeHandler>>& headResultHandler) { return js.evalNow([&] { - TraceContext traceContext = makeR2TraceContext("r2_head"_kjc, "HeadObject"_kjc, key.asPtr()); - return callR2RpcMethod>(js, getRpcMethod(js, "head"_kj), rpcPropHandler, headFnHandler, headResultHandler, kj::mv(key)) .then(js, - [traceContext = kj::mv(traceContext)](jsg::Lock& js, - kj::Maybe parsed) mutable -> kj::Maybe> { + [](jsg::Lock& js, + kj::Maybe parsed) -> kj::Maybe> { // A missing object is null, not an error: the gateway maps the 404 that // R2Result::objectNotFound() used to represent onto a null return. KJ_IF_SOME(backend, parsed) { - auto result = headResultFromBackend(js, kj::mv(backend)); - addHeadResultSpanTags(js, traceContext, *result.get()); - return kj::mv(result); + return headResultFromBackend(js, kj::mv(backend)); } return kj::none; }); @@ -810,8 +776,6 @@ R2Bucket::getRpc(jsg::Lock& js, jsg::Function)>>& getFnHandler, const jsg::TypeHandler>>& getResultHandler) { return js.evalNow([&] { - TraceContext traceContext = makeR2TraceContext("r2_get"_kjc, "GetObject"_kjc, key.asPtr()); - jsg::Optional rpcOptions; KJ_IF_SOME(o, options) { SerializableGetOptions normalized; @@ -829,7 +793,7 @@ R2Bucket::getRpc(jsg::Lock& js, KJ_UNREACHABLE; }(); - auto rpcConditional = makeSerializableConditional(js, conditional, traceContext); + auto rpcConditional = makeSerializableConditional(conditional); KJ_SWITCH_ONEOF(i) { KJ_CASE_ONEOF(conditional, Conditional) { @@ -850,8 +814,6 @@ R2Bucket::getRpc(jsg::Lock& js, ") must be greater than or equal to 0."); JSG_REQUIRE(isWholeNumber(offset), RangeError, "Invalid range. Starting offset (", offset, ") must be an integer, not floating point."); - traceContext.setTag( - "cloudflare.r2.request.range.offset"_kjc, static_cast(offset)); fields.add(jsg::Dict::Field{.name = kj::str("offset"), .value = offset}); } KJ_IF_SOME(length, value.length) { @@ -859,8 +821,6 @@ R2Bucket::getRpc(jsg::Lock& js, ") must be greater than or equal to 0."); JSG_REQUIRE(isWholeNumber(length), RangeError, "Invalid range. Length (", length, ") must be an integer, not floating point."); - traceContext.setTag( - "cloudflare.r2.request.range.length"_kjc, static_cast(length)); fields.add(jsg::Dict::Field{.name = kj::str("length"), .value = length}); } KJ_IF_SOME(suffix, value.suffix) { @@ -872,15 +832,12 @@ R2Bucket::getRpc(jsg::Lock& js, ") must be greater than or equal to 0."); JSG_REQUIRE(isWholeNumber(suffix), RangeError, "Invalid range. Suffix (", suffix, ") must be an integer, not floating point."); - traceContext.setTag( - "cloudflare.r2.request.range.suffix"_kjc, static_cast(suffix)); fields.add(jsg::Dict::Field{.name = kj::str("suffix"), .value = suffix}); } normalized.range = jsg::Dict{.fields = fields.releaseAsArray()}; } KJ_CASE_ONEOF(headers, jsg::Ref) { - KJ_IF_SOME(value, headers->getCommon(js, capnp::CommonHeaderName::RANGE)) { - traceContext.setTag("cloudflare.r2.request.range"_kjc, value.asPtr()); + if (headers->getCommon(js, capnp::CommonHeaderName::RANGE) != kj::none) { normalized.range = kj::mv(headers); } } @@ -888,7 +845,6 @@ R2Bucket::getRpc(jsg::Lock& js, } KJ_IF_SOME(ssecKey, buildSsecKey(kj::mv(o.ssecKey))) { - traceContext.setTag("cloudflare.r2.request.ssec_key"_kjc, true); normalized.ssecKey = kj::mv(ssecKey); } rpcOptions = kj::mv(normalized); @@ -897,20 +853,16 @@ R2Bucket::getRpc(jsg::Lock& js, return callR2RpcMethod>(js, getRpcMethod(js, "get"_kj), rpcPropHandler, getFnHandler, getResultHandler, kj::mv(key), kj::mv(rpcOptions)) .then(js, - [traceContext = kj::mv(traceContext)]( - jsg::Lock& js, kj::Maybe parsed) mutable - -> kj::OneOf>, jsg::Ref> { + [](jsg::Lock& js, kj::Maybe parsed) + -> kj::OneOf>, jsg::Ref> { KJ_IF_SOME(backend, parsed) { KJ_SWITCH_ONEOF(backend) { KJ_CASE_ONEOF(native, jsg::Ref) { - addHeadResultSpanTags(js, traceContext, *native.get()); return kj::Maybe>(kj::mv(native)); } KJ_CASE_ONEOF(native, jsg::Ref) { - auto result = headResultFromBackend( + return headResultFromBackend( js, HeadBackendResult(kj::mv(native)), MissingMetadataPolicy::EMPTY); - addHeadResultSpanTags(js, traceContext, *result.get()); - return kj::mv(result); } KJ_CASE_ONEOF(rpc, SerializableGetResult) { if (rpc.kind == "metadata") { @@ -919,15 +871,11 @@ R2Bucket::getRpc(jsg::Lock& js, js, js.error("Malformed R2 get RPC result: metadata result had a body.")); KJ_FAIL_ASSERT("Malformed R2 get RPC result: metadata result had a body."); } - auto result = - headResultFromSerializable(js, kj::mv(rpc.object), MissingMetadataPolicy::EMPTY); - addHeadResultSpanTags(js, traceContext, *result.get()); - return kj::mv(result); + return headResultFromSerializable( + js, kj::mv(rpc.object), MissingMetadataPolicy::EMPTY); } if (rpc.kind == "body") { - auto result = getResultFromSerializable(js, kj::mv(rpc)); - addHeadResultSpanTags(js, traceContext, *result.get()); - return kj::Maybe>(kj::mv(result)); + return kj::Maybe>(getResultFromSerializable(js, kj::mv(rpc))); } KJ_IF_SOME(body, rpc.body) { @@ -949,15 +897,10 @@ jsg::Promise R2Bucket::deleteRpc(jsg::Lock& js, deleteFnHandler, const jsg::TypeHandler>& deleteResultHandler) { return js.evalNow([&] { - auto& context = IoContext::current(); - TraceContext traceContext = makeR2TraceContext("r2_delete"_kjc, "DeleteObject"_kjc); - traceContext.setTag("cloudflare.r2.request.keys"_kjc, kj::str(keys)); - // The result is discarded, matching delete_: a missing key is success, and per-key failures in // a batch delete are reported in a body the binding has never read. - auto promise = callR2RpcMethod(js, getRpcMethod(js, "delete"_kj), rpcPropHandler, - deleteFnHandler, deleteResultHandler, kj::mv(keys)); - return context.attachSpans(js, kj::mv(promise), kj::mv(traceContext)); + return callR2RpcMethod(js, getRpcMethod(js, "delete"_kj), rpcPropHandler, deleteFnHandler, + deleteResultHandler, kj::mv(keys)); }); } @@ -985,11 +928,9 @@ jsg::Promise>> R2Bucket::putRpc(jsg::Lo } }); - TraceContext traceContext = makeR2TraceContext("r2_put"_kjc, "PutObject"_kjc, key.asPtr()); - bool hashAlreadySpecified = false; const auto prepareChecksum = - [&](auto checksum, kj::StringPtr algorithm, kj::StringPtr tag, size_t byteLength, + [&](auto checksum, kj::StringPtr algorithm, size_t byteLength, size_t hexLength) -> jsg::Optional, kj::String>> { KJ_IF_SOME(c, checksum) { JSG_REQUIRE( @@ -1000,11 +941,7 @@ jsg::Promise>> R2Bucket::putRpc(jsg::Lo auto bytes = buffer.getHandle(js); JSG_REQUIRE(bytes.size() == byteLength, TypeError, algorithm, " is ", byteLength, " bytes, not ", bytes.size()); - auto value = kj::heapArray(bytes.asArrayPtr()); - traceContext.setTag("cloudflare.r2.request.checksum.type"_kjc, tag); - traceContext.setTag( - "cloudflare.r2.request.checksum.value"_kjc, kj::encodeHex(value.asPtr())); - return kj::mv(value); + return kj::heapArray(bytes.asArrayPtr()); } KJ_CASE_ONEOF(text, jsg::NonCoercible) { JSG_REQUIRE(text.value.size() == hexLength, TypeError, algorithm, " is ", hexLength, @@ -1012,8 +949,6 @@ jsg::Promise>> R2Bucket::putRpc(jsg::Lo auto decoded = kj::decodeHex(text.value); JSG_REQUIRE(!decoded.hadErrors, TypeError, "Provided ", algorithm, " wasn't a valid hex string"); - traceContext.setTag("cloudflare.r2.request.checksum.type"_kjc, tag); - traceContext.setTag("cloudflare.r2.request.checksum.value"_kjc, text.value.asPtr()); return kj::heapArray(decoded); } } @@ -1037,7 +972,7 @@ jsg::Promise>> R2Bucket::putRpc(jsg::Lo KJ_UNREACHABLE; }(); - auto rpcConditional = makeSerializableConditional(js, conditional, traceContext); + auto rpcConditional = makeSerializableConditional(conditional); KJ_SWITCH_ONEOF(condition) { KJ_CASE_ONEOF(conditional, Conditional) { @@ -1051,19 +986,10 @@ jsg::Promise>> R2Bucket::putRpc(jsg::Lo jsg::Optional>> httpMetadata; KJ_IF_SOME(metadata, o.httpMetadata) { - auto normalized = normalizeHttpMetadata(js, kj::mv(metadata)); - addHttpMetadataRequestSpanTags(traceContext, normalized); - httpMetadata = kj::mv(normalized); - } - KJ_IF_SOME(_, o.customMetadata) { - traceContext.setTag("cloudflare.r2.request.custom_metadata"_kjc, true); - } - KJ_IF_SOME(storageClass, o.storageClass) { - traceContext.setTag("cloudflare.r2.request.storage_class"_kjc, storageClass.asPtr()); + httpMetadata = normalizeHttpMetadata(js, kj::mv(metadata)); } jsg::Optional, kj::String>> ssecKey; KJ_IF_SOME(key, buildSsecKey(kj::mv(o.ssecKey))) { - traceContext.setTag("cloudflare.r2.request.ssec_key"_kjc, true); ssecKey = kj::mv(key); } @@ -1071,11 +997,11 @@ jsg::Promise>> R2Bucket::putRpc(jsg::Lo .onlyIf = kj::mv(onlyIf), .httpMetadata = kj::mv(httpMetadata), .customMetadata = kj::mv(o.customMetadata), - .md5 = prepareChecksum(kj::mv(o.md5), "MD5"_kj, "md5"_kj, 16, 32), - .sha1 = prepareChecksum(kj::mv(o.sha1), "SHA-1"_kj, "sha1"_kj, 20, 40), - .sha256 = prepareChecksum(kj::mv(o.sha256), "SHA-256"_kj, "sha256"_kj, 32, 64), - .sha384 = prepareChecksum(kj::mv(o.sha384), "SHA-384"_kj, "sha384"_kj, 48, 96), - .sha512 = prepareChecksum(kj::mv(o.sha512), "SHA-512"_kj, "sha512"_kj, 64, 128), + .md5 = prepareChecksum(kj::mv(o.md5), "MD5"_kj, 16, 32), + .sha1 = prepareChecksum(kj::mv(o.sha1), "SHA-1"_kj, 20, 40), + .sha256 = prepareChecksum(kj::mv(o.sha256), "SHA-256"_kj, 32, 64), + .sha384 = prepareChecksum(kj::mv(o.sha384), "SHA-384"_kj, 48, 96), + .sha512 = prepareChecksum(kj::mv(o.sha512), "SHA-512"_kj, 64, 128), .storageClass = kj::mv(o.storageClass), .ssecKey = kj::mv(ssecKey), }; @@ -1091,19 +1017,15 @@ jsg::Promise>> R2Bucket::putRpc(jsg::Lo rpcValue = kj::mv(prepared.value); valueSize = prepared.size; } - traceContext.setTag("cloudflare.r2.request.size"_kjc, static_cast(valueSize)); cancelReader.cancel(); auto rpcPromise = putFn(js, kj::mv(key), kj::mv(rpcValue), kj::mv(rpcOptions), valueSize); auto promise = unwrapR2RpcPromise>(js, kj::mv(rpcPromise), putResultHandler); return promise.then(js, - [traceContext = kj::mv(traceContext)](jsg::Lock& js, - kj::Maybe parsed) mutable -> kj::Maybe> { + [](jsg::Lock& js, kj::Maybe parsed) -> kj::Maybe> { KJ_IF_SOME(backend, parsed) { - auto result = headResultFromBackend(js, kj::mv(backend)); - addHeadResultSpanTags(js, traceContext, *result.get()); - return kj::mv(result); + return headResultFromBackend(js, kj::mv(backend)); } return kj::none; }); @@ -1118,24 +1040,13 @@ jsg::Promise> R2Bucket::createMultipartUploadRpc(jsg createFnHandler, const jsg::TypeHandler>& uploadResultHandler) { return js.evalNow([&] { - TraceContext traceContext = makeR2TraceContext( - "r2_createMultipartUpload"_kjc, "CreateMultipartUpload"_kjc, key.asPtr()); - R2MultipartUpload::Metadata metadata; KJ_IF_SOME(o, options) { KJ_IF_SOME(metadata, o.httpMetadata) { auto normalized = normalizeHttpMetadata(js, kj::mv(metadata)); - addHttpMetadataRequestSpanTags(traceContext, normalized); o.httpMetadata = kj::mv(normalized); } - KJ_IF_SOME(_, o.customMetadata) { - traceContext.setTag("cloudflare.r2.request.custom_metadata"_kjc, true); - } - KJ_IF_SOME(storageClass, o.storageClass) { - traceContext.setTag("cloudflare.r2.request.storage_class"_kjc, storageClass.asPtr()); - } KJ_IF_SOME(ssecKey, buildSsecKey(kj::mv(o.ssecKey))) { - traceContext.setTag("cloudflare.r2.request.ssec_key"_kjc, true); o.ssecKey = kj::mv(ssecKey); } } @@ -1157,8 +1068,7 @@ jsg::Promise> R2Bucket::createMultipartUploadRpc(jsg uploadResultHandler, kj::str(key), kj::mv(options)); return uploadPromise.then(js, - [bucket = JSG_THIS, key = kj::mv(key), metadata = kj::mv(metadata), - traceContext = kj::mv(traceContext)]( + [bucket = JSG_THIS, key = kj::mv(key), metadata = kj::mv(metadata)]( jsg::Lock& js, MultipartUploadBackendResult backend) mutable { kj::String uploadId; KJ_SWITCH_ONEOF(backend) { @@ -1171,7 +1081,6 @@ jsg::Promise> R2Bucket::createMultipartUploadRpc(jsg uploadId = kj::mv(handle.uploadId); } } - traceContext.setTag("cloudflare.r2.response.upload_id"_kjc, uploadId.asPtr()); return js.alloc( kj::mv(key), kj::mv(uploadId), kj::mv(bucket), kj::mv(metadata)); }); @@ -1725,8 +1634,6 @@ jsg::Promise R2Bucket::listRpc(jsg::Lock& js, const jsg::TypeHandler>& listResultHandler, CompatibilityFlags::Reader flags) { return js.evalNow([&] { - TraceContext traceContext = makeR2TraceContext("r2_list"_kjc, "ListObjects"_kjc); - const bool honorsIncludes = flags.getR2ListHonorIncludeFields(); jsg::Optional rpcOptions; if (options != kj::none || !honorsIncludes) { @@ -1736,33 +1643,23 @@ jsg::Promise R2Bucket::listRpc(jsg::Lock& js, KJ_IF_SOME(o, options) { KJ_IF_SOME(limit, o.limit) { normalized.limit = limit; - traceContext.setTag("cloudflare.r2.request.limit"_kjc, static_cast(limit)); } KJ_IF_SOME(prefix, o.prefix) { - traceContext.setTag("cloudflare.r2.request.prefix"_kjc, prefix.value.asPtr()); normalized.prefix = kj::mv(prefix.value); } KJ_IF_SOME(cursor, o.cursor) { - traceContext.setTag("cloudflare.r2.request.cursor"_kjc, cursor.value.asPtr()); normalized.cursor = kj::mv(cursor.value); } KJ_IF_SOME(delimiter, o.delimiter) { - traceContext.setTag("cloudflare.r2.request.delimiter"_kjc, delimiter.value.asPtr()); normalized.delimiter = kj::mv(delimiter.value); } KJ_IF_SOME(startAfter, o.startAfter) { - traceContext.setTag("cloudflare.r2.request.start_after"_kjc, startAfter.value.asPtr()); normalized.startAfter = kj::mv(startAfter.value); } KJ_IF_SOME(requestedIncludes, o.include) { for (auto& requested: requestedIncludes) { - if (requested.value == "httpMetadata") { - traceContext.setTag("cloudflare.r2.request.include.http_metadata"_kjc, true); - } else if (requested.value == "customMetadata") { - traceContext.setTag("cloudflare.r2.request.include.custom_metadata"_kjc, true); - } else { - JSG_FAIL_REQUIRE(RangeError, "Unsupported include value ", requested.value); - } + JSG_REQUIRE(requested.value == "httpMetadata" || requested.value == "customMetadata", + RangeError, "Unsupported include value ", requested.value); if (honorsIncludes) { includes.add(kj::mv(requested.value)); } @@ -1780,9 +1677,7 @@ jsg::Promise R2Bucket::listRpc(jsg::Lock& js, return callR2RpcMethod(js, getRpcMethod(js, "list"_kj), rpcPropHandler, listFnHandler, listResultHandler, kj::mv(rpcOptions)) - .then(js, - [traceContext = kj::mv(traceContext)]( - jsg::Lock& js, ListBackendResult backend) mutable -> ListResult { + .then(js, [](jsg::Lock& js, ListBackendResult backend) -> ListResult { ListResult result; result.objects = KJ_MAP(object, backend.objects) { return headResultFromBackend(js, kj::mv(object), MissingMetadataPolicy::ABSENT); @@ -1793,8 +1688,6 @@ jsg::Promise R2Bucket::listRpc(jsg::Lock& js, } result.delimitedPrefixes = kj::mv(backend.delimitedPrefixes).orDefault(kj::heapArray(0)); - - addListResultSpanTags(traceContext, result); return kj::mv(result); }); }); diff --git a/src/workerd/api/r2-multipart.c++ b/src/workerd/api/r2-multipart.c++ index e28d04ced88..801627d47b7 100644 --- a/src/workerd/api/r2-multipart.c++ +++ b/src/workerd/api/r2-multipart.c++ @@ -152,28 +152,18 @@ jsg::Promise R2MultipartUpload::uploadPartRpc(j JSG_REQUIRE(partNumber >= 1 && partNumber <= 10000, TypeError, "Part number must be between 1 and 10000 (inclusive). Actual value was: ", partNumber); - TraceContext traceContext = bucket->makeR2TraceContext("r2_uploadPart"_kjc, "UploadPart"_kjc); - traceContext.setTag("cloudflare.r2.request.upload_id"_kjc, uploadId.asPtr()); - traceContext.setTag("cloudflare.r2.request.part_number"_kjc, static_cast(partNumber)); - traceContext.setTag("cloudflare.r2.request.key"_kjc, key.asPtr()); - KJ_IF_SOME(o, options) { KJ_IF_SOME(ssecKey, buildSsecKey(kj::mv(o.ssecKey))) { - traceContext.setTag("cloudflare.r2.request.ssec_key"_kjc, true); o.ssecKey = kj::mv(ssecKey); } } auto prepared = prepareR2RpcBody(js, value); - traceContext.setTag("cloudflare.r2.request.size"_kjc, static_cast(prepared.size)); auto promise = callR2RpcMethod(js, bucket->getRpcMethod(js, "uploadPart"_kj), rpcPropHandler, uploadPartFnHandler, uploadPartResultHandler, kj::str(key), kj::str(uploadId), partNumber, kj::mv(prepared.value), kj::mv(options), prepared.size); - return promise.then(js, - [partNumber, traceContext = kj::mv(traceContext)]( - jsg::Lock& js, UploadedPart uploadedPart) mutable { - traceContext.setTag("cloudflare.r2.response.etag"_kjc, uploadedPart.etag.asPtr()); + return promise.then(js, [partNumber](jsg::Lock& js, UploadedPart uploadedPart) { return UploadedPart{.partNumber = partNumber, .etag = kj::mv(uploadedPart.etag)}; }); }); @@ -247,14 +237,6 @@ jsg::Promise> R2MultipartUpload::completeRpc(jsg: kj::String, kj::String, kj::Array)>>& completeFnHandler, const jsg::TypeHandler>& completeResultHandler) { return js.evalNow([&] { - TraceContext traceContext = - bucket->makeR2TraceContext("r2_completeMultipartUpload"_kjc, "CompleteMultipartUpload"_kjc); - traceContext.setTag("cloudflare.r2.request.upload_id"_kjc, uploadId.asPtr()); - traceContext.setTag("cloudflare.r2.request.key"_kjc, key.asPtr()); - kj::String partIds = - kj::strArray(KJ_MAP(part, uploadedParts) { return kj::str(part.partNumber); }, ", "); - traceContext.setTag("cloudflare.r2.request.uploaded_parts"_kjc, kj::mv(partIds)); - for (auto& part: uploadedParts) { JSG_REQUIRE(part.partNumber >= 1 && part.partNumber <= 10000, TypeError, "Part number must be between 1 and 10000 (inclusive). Actual value was: ", @@ -264,12 +246,8 @@ jsg::Promise> R2MultipartUpload::completeRpc(jsg: auto promise = callR2RpcMethod(js, bucket->getRpcMethod(js, "completeMultipartUpload"_kj), rpcPropHandler, completeFnHandler, completeResultHandler, kj::str(key), kj::str(uploadId), kj::mv(uploadedParts)); - return promise.then(js, - [traceContext = kj::mv(traceContext)]( - jsg::Lock& js, R2Bucket::HeadBackendResult backend) mutable { - auto result = headResultFromBackend(js, kj::mv(backend)); - addHeadResultSpanTags(js, traceContext, *result.get()); - return result; + return promise.then(js, [](jsg::Lock& js, R2Bucket::HeadBackendResult backend) { + return headResultFromBackend(js, kj::mv(backend)); }); }); } @@ -320,15 +298,8 @@ jsg::Promise R2MultipartUpload::abortRpc(jsg::Lock& js, const jsg::TypeHandler>& abortFnHandler, const jsg::TypeHandler>& abortResultHandler) { return js.evalNow([&] { - auto& context = IoContext::current(); - TraceContext traceContext = - bucket->makeR2TraceContext("r2_abortMultipartUpload"_kjc, "AbortMultipartUpload"_kjc); - traceContext.setTag("cloudflare.r2.request.upload_id"_kjc, uploadId.asPtr()); - traceContext.setTag("cloudflare.r2.request.key"_kjc, key.asPtr()); - - auto promise = callR2RpcMethod(js, bucket->getRpcMethod(js, "abortMultipartUpload"_kj), + return callR2RpcMethod(js, bucket->getRpcMethod(js, "abortMultipartUpload"_kj), rpcPropHandler, abortFnHandler, abortResultHandler, kj::str(key), kj::str(uploadId)); - return context.attachSpans(js, kj::mv(promise), kj::mv(traceContext)); }); } } // namespace workerd::api::public_beta diff --git a/src/workerd/api/tests/BUILD.bazel b/src/workerd/api/tests/BUILD.bazel index 1024088fca5..c6251273d00 100644 --- a/src/workerd/api/tests/BUILD.bazel +++ b/src/workerd/api/tests/BUILD.bazel @@ -591,8 +591,6 @@ wd_test( src = "r2-jsrpc-test.wd-test", args = ["--experimental"], data = [ - "instrumentation-tail-worker.js", - "r2-instrumentation-test.js", "r2-jsrpc-passthrough.capnp", "r2-test.js", ], diff --git a/src/workerd/api/tests/r2-instrumentation-test.js b/src/workerd/api/tests/r2-instrumentation-test.js index 312ae91b579..3971fc68b4b 100644 --- a/src/workerd/api/tests/r2-instrumentation-test.js +++ b/src/workerd/api/tests/r2-instrumentation-test.js @@ -12,7 +12,7 @@ import { export default testTailHandler; export const test = { - async test(_ctrl, env) { + async test() { // Wait for all the tailStream executions to finish await Promise.allSettled(invocationPromises); @@ -1194,25 +1194,8 @@ export const test = { }, ]; - // The JSRPC transport also records a jsRpcSession span per call and does not set - // cloudflare.r2.response.success, so only the HTTP transport is compared exactly. - const projectSharedTags = (span) => - Object.fromEntries( - Object.entries(span).filter( - ([name]) => name !== 'cloudflare.r2.response.success' - ) - ); - // The expected spans come from the default test and deletePerKeyErrorParityTests, which run // before the other test exports. Spans from those later exports are not checked. - const sharedReceived = received - .filter((span) => span['cloudflare.binding.type'] === 'r2') - .map(projectSharedTags) - .slice(0, expected.length); - assert.deepStrictEqual(sharedReceived, expected.map(projectSharedTags)); - - if (env.R2_TRACE_TRANSPORT === 'http') { - assert.deepStrictEqual(received.slice(0, expected.length), expected); - } + assert.deepStrictEqual(received.slice(0, expected.length), expected); }, }; diff --git a/src/workerd/api/tests/r2-jsrpc-test.wd-test b/src/workerd/api/tests/r2-jsrpc-test.wd-test index 8158b1aaa11..27dc636d085 100644 --- a/src/workerd/api/tests/r2-jsrpc-test.wd-test +++ b/src/workerd/api/tests/r2-jsrpc-test.wd-test @@ -26,25 +26,8 @@ const unitTests :Workerd.Config = ( compatibilityFlags = ["experimental", "nodejs_compat", "service_binding_extra_handlers", "streams_enable_constructors", "r2_list_honor_include", "r2_binding_jsrpc"], - streamingTails = ["tail"], ) ), ( name = "r2-test", worker = Passthrough.passthroughWorker ), - ( name = "tail", worker = .tailWorker ), ], - extensions = [( - modules = [ - (name = "test:instrumentation-tail", esModule = embed "instrumentation-tail-worker.js"), - ], - )], -); - -const tailWorker :Workerd.Worker = ( - modules = [ - (name = "worker", esModule = embed "r2-instrumentation-test.js"), - ], - bindings = [ - (name = "R2_TRACE_TRANSPORT", text = "jsrpc"), - ], - compatibilityFlags = ["experimental", "nodejs_compat"], ); diff --git a/src/workerd/api/tests/r2-test.wd-test b/src/workerd/api/tests/r2-test.wd-test index 75e77e46e92..f44abdb5d7e 100644 --- a/src/workerd/api/tests/r2-test.wd-test +++ b/src/workerd/api/tests/r2-test.wd-test @@ -30,8 +30,5 @@ const tailWorker :Workerd.Worker = ( modules = [ (name = "worker", esModule = embed "r2-instrumentation-test.js") ], - bindings = [ - (name = "R2_TRACE_TRANSPORT", text = "http"), - ], compatibilityFlags = ["experimental", "nodejs_compat"], ); From 11d8caedd9679ebc09e1249b1582955d00f773f7 Mon Sep 17 00:00:00 2001 From: Carol Xu Date: Tue, 6 Oct 2026 15:40:10 -0500 Subject: [PATCH 07/13] Refactoring redundant serializable types out --- src/workerd/api/r2-bucket.c++ | 122 +++++++----------- src/workerd/api/r2-bucket.h | 116 +++++------------ src/workerd/api/r2-multipart.c++ | 9 +- src/workerd/api/r2-multipart.h | 9 +- src/workerd/api/r2-rpc.c++ | 37 +----- src/workerd/api/r2-rpc.h | 22 +--- src/workerd/api/r2.h | 23 ++-- .../api/tests/r2-instrumentation-test.js | 2 +- .../api/tests/r2-jsrpc-serialization-test.js | 13 ++ src/workerd/api/tests/r2-test.js | 2 +- src/workerd/io/worker-interface.capnp | 3 - 11 files changed, 112 insertions(+), 246 deletions(-) diff --git a/src/workerd/api/r2-bucket.c++ b/src/workerd/api/r2-bucket.c++ index c87d957b127..bb2d82b910c 100644 --- a/src/workerd/api/r2-bucket.c++ +++ b/src/workerd/api/r2-bucket.c++ @@ -332,25 +332,6 @@ kj::String buildEtagsString(kj::ArrayPtr etagArray) { }, ", "); } -R2Bucket::SerializableConditional makeSerializableConditional( - R2Bucket::UnwrappedConditional& conditional) { - R2Bucket::SerializableConditional rpcConditional{ - .secondsGranularity = conditional.secondsGranularity}; - KJ_IF_SOME(etags, conditional.etagMatches) { - rpcConditional.etagMatches = buildEtagsString(etags); - } - KJ_IF_SOME(etags, conditional.etagDoesNotMatch) { - rpcConditional.etagDoesNotMatch = buildEtagsString(etags); - } - KJ_IF_SOME(date, conditional.uploadedBefore) { - rpcConditional.uploadedBefore = date; - } - KJ_IF_SOME(date, conditional.uploadedAfter) { - rpcConditional.uploadedAfter = date; - } - return rpcConditional; -} - template void initOnlyIf(TraceContext& traceContext, jsg::Lock& js, Builder& builder, Options& o) { KJ_IF_SOME(i, o.onlyIf) { @@ -715,7 +696,8 @@ static jsg::Ref getResultFromSerializable( return js.alloc(kj::str(object->getName()), kj::str(object->getVersion()), object->getSize(), kj::str(object->getEtag()), object->getChecksums(), object->getUploaded(), object->getHttpMetadata(), kj::mv(customMetadata), object->getRange(), - kj::str(object->getStorageClass()), mapCopyString(object->getSSECKeyMd5()), kj::mv(body)); + kj::str(object->getStorageClass()), + object->getSSECKeyMd5().map([](kj::StringPtr s) { return kj::str(s); }), kj::mv(body)); } jsg::Ref headResultFromBackend( @@ -780,9 +762,12 @@ R2Bucket::getRpc(jsg::Lock& js, KJ_IF_SOME(o, options) { SerializableGetOptions normalized; - KJ_IF_SOME(i, o.onlyIf) { + KJ_IF_SOME(condition, o.onlyIf) { + // Parsed only for validation, so callers get the same errors as the non-RPC path (e.g. + // quoted or malformed ETags, invalid dates). The parsed result is intentionally unused: + // the original onlyIf value is forwarded below and the R2GW does its own parsing. UnwrappedConditional conditional = [&] { - KJ_SWITCH_ONEOF(i) { + KJ_SWITCH_ONEOF(condition) { KJ_CASE_ONEOF(value, Conditional) { return UnwrappedConditional(value); } @@ -793,11 +778,9 @@ R2Bucket::getRpc(jsg::Lock& js, KJ_UNREACHABLE; }(); - auto rpcConditional = makeSerializableConditional(conditional); - - KJ_SWITCH_ONEOF(i) { + KJ_SWITCH_ONEOF(condition) { KJ_CASE_ONEOF(conditional, Conditional) { - normalized.onlyIf = kj::mv(rpcConditional); + normalized.onlyIf = kj::mv(conditional); } KJ_CASE_ONEOF(headers, jsg::Ref) { normalized.onlyIf = kj::mv(headers); @@ -909,10 +892,8 @@ jsg::Promise>> R2Bucket::putRpc(jsg::Lo kj::Maybe value, jsg::Optional options, const jsg::TypeHandler>& rpcPropHandler, - const jsg::TypeHandler, - jsg::Optional, - double)>>& putFnHandler, + const jsg::TypeHandler, jsg::Optional, double)>>& putFnHandler, const jsg::TypeHandler>>& putResultHandler) { return js.evalNow([&] { auto cancelReader = kj::defer([&] { @@ -928,10 +909,11 @@ jsg::Promise>> R2Bucket::putRpc(jsg::Lo } }); + // Checksums are validated here so callers get the same errors as the non-RPC path, then + // forwarded unchanged; the R2GW decodes hex strings and buffers itself. bool hashAlreadySpecified = false; - const auto prepareChecksum = - [&](auto checksum, kj::StringPtr algorithm, size_t byteLength, - size_t hexLength) -> jsg::Optional, kj::String>> { + const auto validateChecksum = [&](auto checksum, kj::StringPtr algorithm, size_t byteLength, + size_t hexLength) -> decltype(checksum) { KJ_IF_SOME(c, checksum) { JSG_REQUIRE( !hashAlreadySpecified, TypeError, "You cannot specify multiple hashing algorithms."); @@ -941,7 +923,6 @@ jsg::Promise>> R2Bucket::putRpc(jsg::Lo auto bytes = buffer.getHandle(js); JSG_REQUIRE(bytes.size() == byteLength, TypeError, algorithm, " is ", byteLength, " bytes, not ", bytes.size()); - return kj::heapArray(bytes.asArrayPtr()); } KJ_CASE_ONEOF(text, jsg::NonCoercible) { JSG_REQUIRE(text.value.size() == hexLength, TypeError, algorithm, " is ", hexLength, @@ -949,17 +930,19 @@ jsg::Promise>> R2Bucket::putRpc(jsg::Lo auto decoded = kj::decodeHex(text.value); JSG_REQUIRE(!decoded.hadErrors, TypeError, "Provided ", algorithm, " wasn't a valid hex string"); - return kj::heapArray(decoded); } } } - return kj::none; + return kj::mv(checksum); }; - jsg::Optional rpcOptions; + jsg::Optional rpcOptions; KJ_IF_SOME(o, options) { - jsg::Optional>> onlyIf; + jsg::Optional>> onlyIf; KJ_IF_SOME(condition, o.onlyIf) { + // Parsed only for validation, so callers get the same errors as the non-RPC path (e.g. + // quoted or malformed ETags, invalid dates). The parsed result is intentionally unused: + // the original onlyIf value is forwarded below and the R2GW does its own parsing. UnwrappedConditional conditional = [&] { KJ_SWITCH_ONEOF(condition) { KJ_CASE_ONEOF(value, Conditional) { @@ -972,11 +955,9 @@ jsg::Promise>> R2Bucket::putRpc(jsg::Lo KJ_UNREACHABLE; }(); - auto rpcConditional = makeSerializableConditional(conditional); - KJ_SWITCH_ONEOF(condition) { KJ_CASE_ONEOF(conditional, Conditional) { - onlyIf = kj::mv(rpcConditional); + onlyIf = kj::mv(conditional); } KJ_CASE_ONEOF(headers, jsg::Ref) { onlyIf = kj::mv(headers); @@ -993,15 +974,15 @@ jsg::Promise>> R2Bucket::putRpc(jsg::Lo ssecKey = kj::mv(key); } - rpcOptions = SerializablePutOptions{ + rpcOptions = PutOptions{ .onlyIf = kj::mv(onlyIf), .httpMetadata = kj::mv(httpMetadata), .customMetadata = kj::mv(o.customMetadata), - .md5 = prepareChecksum(kj::mv(o.md5), "MD5"_kj, 16, 32), - .sha1 = prepareChecksum(kj::mv(o.sha1), "SHA-1"_kj, 20, 40), - .sha256 = prepareChecksum(kj::mv(o.sha256), "SHA-256"_kj, 32, 64), - .sha384 = prepareChecksum(kj::mv(o.sha384), "SHA-384"_kj, 48, 96), - .sha512 = prepareChecksum(kj::mv(o.sha512), "SHA-512"_kj, 64, 128), + .md5 = validateChecksum(kj::mv(o.md5), "MD5"_kj, 16, 32), + .sha1 = validateChecksum(kj::mv(o.sha1), "SHA-1"_kj, 20, 40), + .sha256 = validateChecksum(kj::mv(o.sha256), "SHA-256"_kj, 32, 64), + .sha384 = validateChecksum(kj::mv(o.sha384), "SHA-384"_kj, 48, 96), + .sha512 = validateChecksum(kj::mv(o.sha512), "SHA-512"_kj, 64, 128), .storageClass = kj::mv(o.storageClass), .ssecKey = kj::mv(ssecKey), }; @@ -1010,7 +991,7 @@ jsg::Promise>> R2Bucket::putRpc(jsg::Lo auto rpcMethod = getRpcMethod(js, "put"_kj); auto wrappedMethod = rpcPropHandler.wrap(js, kj::mv(rpcMethod)); auto putFn = KJ_ASSERT_NONNULL(putFnHandler.tryUnwrap(js, wrappedMethod)); - kj::Maybe rpcValue; + kj::Maybe rpcValue; double valueSize = 0; KJ_IF_SOME(v, value) { auto prepared = prepareR2RpcBody(js, v); @@ -1038,7 +1019,7 @@ jsg::Promise> R2Bucket::createMultipartUploadRpc(jsg const jsg::TypeHandler>& rpcPropHandler, const jsg::TypeHandler)>>& createFnHandler, - const jsg::TypeHandler>& uploadResultHandler) { + const jsg::TypeHandler>>& uploadIdHandler) { return js.evalNow([&] { R2MultipartUpload::Metadata metadata; KJ_IF_SOME(o, options) { @@ -1063,26 +1044,17 @@ jsg::Promise> R2Bucket::createMultipartUploadRpc(jsg } } - auto uploadPromise = callR2RpcMethod(js, + // The R2GW returns only the upload ID; the key, bucket, and metadata are already known here. + // NonCoercible so a non-string reply is rejected as malformed rather than stringified. + auto uploadPromise = callR2RpcMethod>(js, getRpcMethod(js, "createMultipartUpload"_kj), rpcPropHandler, createFnHandler, - uploadResultHandler, kj::str(key), kj::mv(options)); + uploadIdHandler, kj::str(key), kj::mv(options)); return uploadPromise.then(js, [bucket = JSG_THIS, key = kj::mv(key), metadata = kj::mv(metadata)]( - jsg::Lock& js, MultipartUploadBackendResult backend) mutable { - kj::String uploadId; - KJ_SWITCH_ONEOF(backend) { - KJ_CASE_ONEOF(id, kj::String) { - uploadId = kj::mv(id); - } - KJ_CASE_ONEOF(handle, SerializableMultipartUploadHandle) { - KJ_ASSERT(handle.key == key, - "Malformed R2 createMultipartUpload RPC result: handle key did not match request."); - uploadId = kj::mv(handle.uploadId); - } - } + jsg::Lock& js, jsg::NonCoercible uploadId) mutable { return js.alloc( - kj::mv(key), kj::mv(uploadId), kj::mv(bucket), kj::mv(metadata)); + kj::mv(key), kj::mv(uploadId.value), kj::mv(bucket), kj::mv(metadata)); }); }); } @@ -1629,32 +1601,31 @@ jsg::Promise R2Bucket::delete_(jsg::Lock& js, jsg::Promise R2Bucket::listRpc(jsg::Lock& js, jsg::Optional options, const jsg::TypeHandler>& rpcPropHandler, - const jsg::TypeHandler)>>& - listFnHandler, + const jsg::TypeHandler)>>& listFnHandler, const jsg::TypeHandler>& listResultHandler, CompatibilityFlags::Reader flags) { return js.evalNow([&] { const bool honorsIncludes = flags.getR2ListHonorIncludeFields(); - jsg::Optional rpcOptions; + jsg::Optional rpcOptions; if (options != kj::none || !honorsIncludes) { - SerializableListOptions normalized; - kj::Vector includes; + ListOptions normalized; + kj::Vector> includes; KJ_IF_SOME(o, options) { KJ_IF_SOME(limit, o.limit) { normalized.limit = limit; } KJ_IF_SOME(prefix, o.prefix) { - normalized.prefix = kj::mv(prefix.value); + normalized.prefix = kj::mv(prefix); } KJ_IF_SOME(cursor, o.cursor) { - normalized.cursor = kj::mv(cursor.value); + normalized.cursor = kj::mv(cursor); } KJ_IF_SOME(delimiter, o.delimiter) { - normalized.delimiter = kj::mv(delimiter.value); + normalized.delimiter = kj::mv(delimiter); } KJ_IF_SOME(startAfter, o.startAfter) { - normalized.startAfter = kj::mv(startAfter.value); + normalized.startAfter = kj::mv(startAfter); } KJ_IF_SOME(requestedIncludes, o.include) { for (auto& requested: requestedIncludes) { @@ -2133,7 +2104,7 @@ R2Bucket::SerializableHeadResult R2Bucket::HeadResult::toSerializable() const { .httpMetadata = kj::mv(rpcHttpMetadata), .customMetadata = kj::mv(rpcCustomMetadata), .range = kj::mv(rpcRange), - .ssecKeyMd5 = mapCopyString(ssecKeyMd5), + .ssecKeyMd5 = ssecKeyMd5.clone(), }; } @@ -2197,7 +2168,8 @@ jsg::Ref R2Bucket::GetResult::deserialize(jsg::Lock& js, return js.alloc(kj::str(object->getName()), kj::str(object->getVersion()), object->getSize(), kj::str(object->getEtag()), object->getChecksums(), object->getUploaded(), object->getHttpMetadata(), kj::mv(customMetadata), object->getRange(), - kj::str(object->getStorageClass()), mapCopyString(object->getSSECKeyMd5()), + kj::str(object->getStorageClass()), + object->getSSECKeyMd5().map([](kj::StringPtr s) { return kj::str(s); }), KJ_ASSERT_NONNULL(kj::mv(getResult.body)), bodyLength); } diff --git a/src/workerd/api/r2-bucket.h b/src/workerd/api/r2-bucket.h index e67093f8ad5..fe05568d316 100644 --- a/src/workerd/api/r2-bucket.h +++ b/src/workerd/api/r2-bucket.h @@ -250,68 +250,39 @@ class R2Bucket: public jsg::Object { JSG_STRUCT_TS_OVERRIDE(R2PutOptions); }; - // Serializable* types: the intermediate representation exchanged with the R2 gateway worker - // (R2GW) over JSRPC. They mirror the shape of the parameters the R2GW expects and the values - // it returns, and exist only so those values can be serialized across the JSRPC boundary. - // - // These are NOT customer-facing types and must never be returned to user code. The public - // counterparts (Conditional, GetOptions, HeadResult, GetResult, ListResult, ...) validate - // input with NonCoercible strings and expose methods / lazy accessors, none of which can be - // wrapped back to JavaScript or serialized. Public inputs are normalized into a Serializable* - // type before calling the R2GW, and Serializable* results are converted back into the public - // types before being handed to the caller. For the same reason these carry no TS overrides. - struct SerializableConditional { - jsg::Optional etagMatches; - jsg::Optional etagDoesNotMatch; - jsg::Optional uploadedBefore; - jsg::Optional uploadedAfter; - jsg::Optional secondsGranularity; - - JSG_STRUCT(etagMatches, etagDoesNotMatch, uploadedBefore, uploadedAfter, secondsGranularity); - }; - - struct SerializablePutOptions { - jsg::Optional>> onlyIf; + struct MultipartOptions { jsg::Optional>> httpMetadata; jsg::Optional> customMetadata; - jsg::Optional, kj::String>> md5; - jsg::Optional, kj::String>> sha1; - jsg::Optional, kj::String>> sha256; - jsg::Optional, kj::String>> sha384; - jsg::Optional, kj::String>> sha512; jsg::Optional storageClass; jsg::Optional, kj::String>> ssecKey; - JSG_STRUCT(onlyIf, - httpMetadata, - customMetadata, - md5, - sha1, - sha256, - sha384, - sha512, - storageClass, - ssecKey); + JSG_STRUCT(httpMetadata, customMetadata, storageClass, ssecKey); + JSG_STRUCT_TS_OVERRIDE(R2MultipartOptions); }; + // Serializable* types: values exchanged with the R2 gateway worker (R2GW) over JSRPC where the + // public type cannot be used directly. Most RPC inputs (PutOptions, ListOptions, Conditional) + // are sent as their public types; SerializableGetOptions exists only because of `range` (see + // below). + // + // The R2GW returns results as plain objects. Only a JSG_STRUCT can be unwrapped from a plain + // object, so the resource types (Checksums, HeadResult, GetResult) cannot receive them; the + // Serializable* result structs receive them instead and are converted into the public types + // before being handed to the caller. + // + // These are NOT customer-facing types and must never be returned to user code, so they carry + // no TS overrides. + struct SerializableGetOptions { - jsg::Optional>> onlyIf; + jsg::Optional>> onlyIf; + // Uses Dict rather than Range because the fast_jsg_struct compat flag sets absent Range fields + // to undefined. jsg::Optional, jsg::Ref>> range; jsg::Optional ssecKey; JSG_STRUCT(onlyIf, range, ssecKey); }; - struct MultipartOptions { - jsg::Optional>> httpMetadata; - jsg::Optional> customMetadata; - jsg::Optional storageClass; - jsg::Optional, kj::String>> ssecKey; - - JSG_STRUCT(httpMetadata, customMetadata, storageClass, ssecKey); - JSG_STRUCT_TS_OVERRIDE(R2MultipartOptions); - }; - // Object metadata as returned by the R2GW. HeadResult / Checksums are built from these. struct SerializableChecksums { jsg::Optional> md5; @@ -356,26 +327,6 @@ class R2Bucket: public jsg::Object { ssecKeyMd5); }; - struct SerializableListOptions { - jsg::Optional limit; - jsg::Optional prefix; - jsg::Optional cursor; - jsg::Optional delimiter; - jsg::Optional startAfter; - kj::Array include; - - JSG_STRUCT(limit, prefix, cursor, delimiter, startAfter, include); - }; - - struct SerializableListResult { - kj::Array objects; - bool truncated; - kj::Maybe cursor; - kj::Maybe> delimitedPrefixes; - - JSG_STRUCT(objects, truncated, cursor, delimitedPrefixes); - }; - struct SerializableGetResult { kj::String kind; SerializableHeadResult object; @@ -384,15 +335,6 @@ class R2Bucket: public jsg::Object { JSG_STRUCT(kind, object, body); }; - struct SerializableMultipartUploadHandle { - kj::String key; - kj::String uploadId; - - JSG_STRUCT(key, uploadId); - }; - - using MultipartUploadBackendResult = kj::OneOf; - class HeadResult: public jsg::Object { public: HeadResult(kj::String name, @@ -721,10 +663,8 @@ class R2Bucket: public jsg::Object { kj::Maybe value, jsg::Optional options, const jsg::TypeHandler>& rpcPropHandler, - const jsg::TypeHandler, - jsg::Optional, - double)>>& putFnHandler, + const jsg::TypeHandler, jsg::Optional, double)>>& putFnHandler, const jsg::TypeHandler>>& putResultHandler); jsg::Promise> createMultipartUploadRpc(jsg::Lock& js, kj::String key, @@ -732,12 +672,11 @@ class R2Bucket: public jsg::Object { const jsg::TypeHandler>& rpcPropHandler, const jsg::TypeHandler< jsg::Function)>>& createFnHandler, - const jsg::TypeHandler>& uploadResultHandler); + const jsg::TypeHandler>>& uploadIdHandler); jsg::Promise listRpc(jsg::Lock& js, jsg::Optional options, const jsg::TypeHandler>& rpcPropHandler, - const jsg::TypeHandler)>>& - listFnHandler, + const jsg::TypeHandler)>>& listFnHandler, const jsg::TypeHandler>& listResultHandler, CompatibilityFlags::Reader flags); jsg::Promise list(jsg::Lock& js, @@ -749,22 +688,25 @@ class R2Bucket: public jsg::Object { // The compatibility flag restricts the new transport to allowlisted // workers, because it is marked $experimental and EWC decides who // may opt in. + // Method order determines the generated TypeScript declaration order, so both branches + // declare resumeMultipartUpload in the same position. if (flags.getR2BindingsJsrpc()) { JSG_METHOD_NAMED(head, headRpc); JSG_METHOD_NAMED(get, getRpc); - JSG_METHOD_NAMED(delete, deleteRpc); JSG_METHOD_NAMED(put, putRpc); JSG_METHOD_NAMED(createMultipartUpload, createMultipartUploadRpc); + JSG_METHOD(resumeMultipartUpload); + JSG_METHOD_NAMED(delete, deleteRpc); JSG_METHOD_NAMED(list, listRpc); } else { JSG_METHOD(head); JSG_METHOD(get); - JSG_METHOD_NAMED(delete, delete_); JSG_METHOD(put); JSG_METHOD(createMultipartUpload); + JSG_METHOD(resumeMultipartUpload); + JSG_METHOD_NAMED(delete, delete_); JSG_METHOD(list); } - JSG_METHOD(resumeMultipartUpload); JSG_TS_ROOT(); JSG_TS_OVERRIDE({ // The order of these matters, since typescript tries to match function signatures in order diff --git a/src/workerd/api/r2-multipart.c++ b/src/workerd/api/r2-multipart.c++ index 801627d47b7..5fe038a0952 100644 --- a/src/workerd/api/r2-multipart.c++ +++ b/src/workerd/api/r2-multipart.c++ @@ -141,12 +141,9 @@ jsg::Promise R2MultipartUpload::uploadPartRpc(j R2PutValue value, jsg::Optional options, const jsg::TypeHandler>& rpcPropHandler, - const jsg::TypeHandler, - double)>>& uploadPartFnHandler, + const jsg::TypeHandler, double)>>& + uploadPartFnHandler, const jsg::TypeHandler>& uploadPartResultHandler) { return js.evalNow([&] { JSG_REQUIRE(partNumber >= 1 && partNumber <= 10000, TypeError, diff --git a/src/workerd/api/r2-multipart.h b/src/workerd/api/r2-multipart.h index 3528659b60b..0c05448240c 100644 --- a/src/workerd/api/r2-multipart.h +++ b/src/workerd/api/r2-multipart.h @@ -70,12 +70,9 @@ class R2MultipartUpload: public jsg::Object { R2PutValue value, jsg::Optional options, const jsg::TypeHandler>& rpcPropHandler, - const jsg::TypeHandler, - double)>>& uploadPartFnHandler, + const jsg::TypeHandler, double)>>& + uploadPartFnHandler, const jsg::TypeHandler>& uploadPartResultHandler); jsg::Promise abortRpc(jsg::Lock& js, const jsg::TypeHandler>& rpcPropHandler, diff --git a/src/workerd/api/r2-rpc.c++ b/src/workerd/api/r2-rpc.c++ index 49f56d4f1da..e390002743d 100644 --- a/src/workerd/api/r2-rpc.c++ +++ b/src/workerd/api/r2-rpc.c++ @@ -84,14 +84,14 @@ PreparedR2RpcBody prepareR2RpcBody(jsg::Lock& js, R2PutValue& value) { return {.value = makeR2RpcMemoryStream(js, bytes, kj::heap(kj::mv(text.value))), .size = static_cast(size)}; } - return {.value = kj::mv(text.value), .size = static_cast(size)}; + return {.value = kj::mv(text), .size = static_cast(size)}; } KJ_CASE_ONEOF(blob, jsg::Ref) { auto size = blob->getSize(); if (size > R2_RPC_INLINE_BODY_LIMIT) { // Blob bytes are V8-backed, so newMemorySource() must copy them into native memory. return { - .value = makeR2RpcMemoryStream(js, blob->getData()), .size = static_cast(size)}; + .value = makeR2RpcMemoryStream(js, blob->getData(js)), .size = static_cast(size)}; } return {.value = kj::mv(blob), .size = static_cast(size)}; } @@ -113,33 +113,6 @@ jsg::JsValue R2Error::getStack(jsg::Lock& js) { return jsg::JsObject(KJ_ASSERT_NONNULL(errorForStack).Get(js.v8Isolate)).get(js, "stack"_kj); } -void R2Error::serialize(jsg::Lock& js, - jsg::Serializer& serializer, - const jsg::TypeHandler& payloadHandler) { - requireR2RpcSerializer(serializer); - serializer.write(js, - jsg::JsValue(payloadHandler.wrap(js, - SerializableR2Error{ - .code = v4Code, - .message = kj::str(message), - .action = kj::str(KJ_REQUIRE_NONNULL(action)), - }))); -} - -jsg::Ref R2Error::deserialize(jsg::Lock& js, - rpc::SerializationTag tag, - jsg::Deserializer& deserializer, - const jsg::TypeHandler& payloadHandler) { - requireR2RpcDeserializer(deserializer); - auto payload = KJ_ASSERT_NONNULL(payloadHandler.tryUnwrap(js, deserializer.readValue(js)), - "Deserialization failed: invalid R2 error payload"); - auto result = js.alloc(payload.code, kj::mv(payload.message)); - result->action = kj::mv(payload.action); - result->errorForStack = v8::Global( - js.v8Isolate, v8::Exception::Error(v8::String::Empty(js.v8Isolate)).As()); - return result; -} - kj::Maybe R2Result::v4ErrorCode() { KJ_IF_SOME(e, toThrow) { return e->v4Code; @@ -157,9 +130,9 @@ kj::Maybe R2Result::getR2ErrorMessage() { void R2Result::throwIfError( kj::StringPtr action, const jsg::TypeHandler>& errorType) { KJ_IF_SOME(e, toThrow) { - // Keep the existing generic error behavior for HTTP-backed bindings. R2Error is serializable so - // an already-structured error can cross JSRPC, but changing this throw path would be a public - // compatibility change. + // Keep the existing generic error behavior for R2 bindings. Changing this throw path to + // throw a structured R2Error would be a public compatibility change, and R2Error would also + // need to be made serializable so it can cross JSRPC. #if 0 auto isolate = IoContext::current().getCurrentLock().getIsolate(); (*e)->action = kj::str(action); diff --git a/src/workerd/api/r2-rpc.h b/src/workerd/api/r2-rpc.h index 137a87b9f0e..33898df7664 100644 --- a/src/workerd/api/r2-rpc.h +++ b/src/workerd/api/r2-rpc.h @@ -76,14 +76,6 @@ jsg::Promise callR2RpcMethod(jsg::Lock& js, // to maintain ownership. class R2Error: public jsg::Object { public: - struct SerializableR2Error { - uint code; - kj::String message; - kj::String action; - - JSG_STRUCT(code, message, action); - }; - R2Error(uint v4Code, kj::String message): v4Code(v4Code), message(kj::mv(message)) {} constexpr kj::StringPtr getName() const { @@ -114,16 +106,6 @@ class R2Error: public jsg::Object { JSG_TS_ROOT(); } - void serialize(jsg::Lock& js, - jsg::Serializer& serializer, - const jsg::TypeHandler& payloadHandler); - static jsg::Ref deserialize(jsg::Lock& js, - rpc::SerializationTag tag, - jsg::Deserializer& deserializer, - const jsg::TypeHandler& payloadHandler); - - JSG_SERIALIZABLE(rpc::SerializationTag::R2_ERROR); - private: uint v4Code; kj::String message; @@ -138,11 +120,9 @@ class R2Error: public jsg::Object { using R2PutValue = kj::OneOf, jsg::NonCoercible, jsg::Ref>; -using SerializablePutValue = - kj::OneOf, kj::String, jsg::Ref>; struct PreparedR2RpcBody { - SerializablePutValue value; + R2PutValue value; double size; }; diff --git a/src/workerd/api/r2.h b/src/workerd/api/r2.h index c6b4ec1e8af..eb47e1cd908 100644 --- a/src/workerd/api/r2.h +++ b/src/workerd/api/r2.h @@ -9,24 +9,19 @@ namespace workerd::api::public_beta { #define EW_R2_PUBLIC_BETA_ISOLATE_TYPES \ - api::R2Error, api::R2Error::SerializableR2Error, api::public_beta::R2Bucket, \ - api::public_beta::R2MultipartUpload, api::public_beta::R2MultipartUpload::UploadedPart, \ - api::public_beta::R2Bucket::HeadResult, api::public_beta::R2Bucket::GetResult, \ - api::public_beta::R2Bucket::Range, api::public_beta::R2Bucket::Conditional, \ - api::public_beta::R2Bucket::GetOptions, api::public_beta::R2Bucket::PutOptions, \ - api::public_beta::R2Bucket::MultipartOptions, api::public_beta::R2Bucket::Checksums, \ - api::public_beta::R2Bucket::StringChecksums, api::public_beta::R2Bucket::HttpMetadata, \ - api::public_beta::R2Bucket::ListOptions, api::public_beta::R2Bucket::ListResult, \ + api::R2Error, api::public_beta::R2Bucket, api::public_beta::R2MultipartUpload, \ + api::public_beta::R2MultipartUpload::UploadedPart, api::public_beta::R2Bucket::HeadResult, \ + api::public_beta::R2Bucket::GetResult, api::public_beta::R2Bucket::Range, \ + api::public_beta::R2Bucket::Conditional, api::public_beta::R2Bucket::GetOptions, \ + api::public_beta::R2Bucket::PutOptions, api::public_beta::R2Bucket::MultipartOptions, \ + api::public_beta::R2Bucket::Checksums, api::public_beta::R2Bucket::StringChecksums, \ + api::public_beta::R2Bucket::HttpMetadata, api::public_beta::R2Bucket::ListOptions, \ + api::public_beta::R2Bucket::ListResult, \ api::public_beta::R2MultipartUpload::UploadPartOptions, \ api::public_beta::R2Bucket::SerializableChecksums, \ api::public_beta::R2Bucket::SerializableHeadResult, \ - api::public_beta::R2Bucket::SerializableConditional, \ - api::public_beta::R2Bucket::SerializablePutOptions, \ api::public_beta::R2Bucket::SerializableGetOptions, \ api::public_beta::R2Bucket::SerializableGetResult, \ - api::public_beta::R2Bucket::SerializableListOptions, \ - api::public_beta::R2Bucket::SerializableListResult, \ - api::public_beta::R2Bucket::ListBackendResult, \ - api::public_beta::R2Bucket::SerializableMultipartUploadHandle + api::public_beta::R2Bucket::ListBackendResult // The list of r2 types that are added to worker.c++'s JSG_DECLARE_ISOLATE_TYPE } // namespace workerd::api::public_beta diff --git a/src/workerd/api/tests/r2-instrumentation-test.js b/src/workerd/api/tests/r2-instrumentation-test.js index 3971fc68b4b..09fea51f6d6 100644 --- a/src/workerd/api/tests/r2-instrumentation-test.js +++ b/src/workerd/api/tests/r2-instrumentation-test.js @@ -73,7 +73,7 @@ export const test = { 'cloudflare.r2.request.size': 7n, 'cloudflare.r2.response.success': true, 'cloudflare.r2.response.etag': 'objectEtag', - 'cloudflare.r2.response.size': 123, + 'cloudflare.r2.response.size': 7, 'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z', 'cloudflare.r2.response.storage_class': 'Standard', 'cloudflare.r2.response.custom_metadata': true, diff --git a/src/workerd/api/tests/r2-jsrpc-serialization-test.js b/src/workerd/api/tests/r2-jsrpc-serialization-test.js index e76a132fc91..bcc42fed84f 100644 --- a/src/workerd/api/tests/r2-jsrpc-serialization-test.js +++ b/src/workerd/api/tests/r2-jsrpc-serialization-test.js @@ -34,6 +34,14 @@ export class ObservedR2Binding extends PassthroughR2Binding { return super.get(requestKey, options); } + async createMultipartUpload(requestKey, options) { + // The R2GW returns the upload ID as a bare string; an object must not be stringified into one. + if (requestKey === 'upload-handle-as-upload-id') { + return { key: requestKey, uploadId: 'upload-handle-id' }; + } + return super.createMultipartUpload(requestKey, options); + } + async checksums() { return (await this.env.REAL_BUCKET.head('multipleChecksums')).checksums; } @@ -177,5 +185,10 @@ export const nativeR2ResultTests = { 'rpc-multipart-abort-platform' ); await aborted.abort(); + + await assert.rejects( + () => env.BUCKET.createMultipartUpload('upload-handle-as-upload-id'), + { message: /^internal error; reference = \S+$/ } + ); }, }; diff --git a/src/workerd/api/tests/r2-test.js b/src/workerd/api/tests/r2-test.js index 9f2c5387a06..30dce5711ec 100644 --- a/src/workerd/api/tests/r2-test.js +++ b/src/workerd/api/tests/r2-test.js @@ -1455,7 +1455,7 @@ export class ObservedR2Binding extends WorkerEntrypoint { requestKey, options ); - return { key: upload.key, uploadId: upload.uploadId }; + return upload.uploadId; } uploadPart(requestKey, uploadId, partNumber, value, options, valueSize) { diff --git a/src/workerd/io/worker-interface.capnp b/src/workerd/io/worker-interface.capnp index 5df2b633fab..a00c9b543eb 100644 --- a/src/workerd/io/worker-interface.capnp +++ b/src/workerd/io/worker-interface.capnp @@ -562,9 +562,6 @@ enum SerializationTag { r2ObjectBody @21; # R2 object metadata with a transferred body stream. - - r2Error @22; - # A structured error produced by an R2 operation. } enum StreamEncoding { From 64020559ddd71b23724e3040c50301f6b9a79392 Mon Sep 17 00:00:00 2001 From: Carol Xu Date: Tue, 20 Oct 2026 22:22:58 -0400 Subject: [PATCH 08/13] add R2BindingService and return uploadId --- src/workerd/api/r2-bucket.c++ | 17 ++- src/workerd/api/r2-bucket.h | 10 +- src/workerd/api/tests/BUILD.bazel | 35 +++++ .../r2-binding-service/r2-binding-service.ts | 133 ++++++++++++++++++ .../r2-binding-service/r2-observed-binding.ts | 119 ++++++++++++++++ .../tests/r2-binding-service/tsconfig.json | 10 ++ .../api/tests/r2-jsrpc-passthrough.capnp | 7 +- .../api/tests/r2-jsrpc-serialization-test.js | 31 ++-- .../tests/r2-jsrpc-serialization-test.wd-test | 4 + src/workerd/api/tests/r2-test.js | 75 ---------- src/workerd/api/tests/tsconfig.json | 5 +- types/BUILD.bazel | 15 +- 12 files changed, 366 insertions(+), 95 deletions(-) create mode 100644 src/workerd/api/tests/r2-binding-service/r2-binding-service.ts create mode 100644 src/workerd/api/tests/r2-binding-service/r2-observed-binding.ts create mode 100644 src/workerd/api/tests/r2-binding-service/tsconfig.json diff --git a/src/workerd/api/r2-bucket.c++ b/src/workerd/api/r2-bucket.c++ index bb2d82b910c..549cc0c5c58 100644 --- a/src/workerd/api/r2-bucket.c++ +++ b/src/workerd/api/r2-bucket.c++ @@ -1019,7 +1019,7 @@ jsg::Promise> R2Bucket::createMultipartUploadRpc(jsg const jsg::TypeHandler>& rpcPropHandler, const jsg::TypeHandler)>>& createFnHandler, - const jsg::TypeHandler>>& uploadIdHandler) { + const jsg::TypeHandler>& createResultHandler) { return js.evalNow([&] { R2MultipartUpload::Metadata metadata; KJ_IF_SOME(o, options) { @@ -1044,17 +1044,20 @@ jsg::Promise> R2Bucket::createMultipartUploadRpc(jsg } } - // The R2GW returns only the upload ID; the key, bucket, and metadata are already known here. - // NonCoercible so a non-string reply is rejected as malformed rather than stringified. - auto uploadPromise = callR2RpcMethod>(js, + // The backend replies with only `{ uploadId }`; the key, bucket, and metadata are already + // known here. Any other reply shape, including a bare string or additional fields, is + // rejected as malformed. + auto uploadPromise = callR2RpcMethod(js, getRpcMethod(js, "createMultipartUpload"_kj), rpcPropHandler, createFnHandler, - uploadIdHandler, kj::str(key), kj::mv(options)); + createResultHandler, kj::str(key), kj::mv(options)); return uploadPromise.then(js, [bucket = JSG_THIS, key = kj::mv(key), metadata = kj::mv(metadata)]( - jsg::Lock& js, jsg::NonCoercible uploadId) mutable { + jsg::Lock& js, CreateMultipartUploadBackendResult result) mutable { + KJ_ASSERT(result.fields.size() == 1 && result.fields[0].name == "uploadId"_kj, + "Malformed R2 createMultipartUpload RPC result: expected only an uploadId."); return js.alloc( - kj::mv(key), kj::mv(uploadId.value), kj::mv(bucket), kj::mv(metadata)); + kj::mv(key), kj::mv(result.fields[0].value.value), kj::mv(bucket), kj::mv(metadata)); }); }); } diff --git a/src/workerd/api/r2-bucket.h b/src/workerd/api/r2-bucket.h index fe05568d316..51650fc67eb 100644 --- a/src/workerd/api/r2-bucket.h +++ b/src/workerd/api/r2-bucket.h @@ -585,6 +585,13 @@ class R2Bucket: public jsg::Object { JSG_STRUCT(objects, truncated, cursor, delimitedPrefixes); }; + // The backing entrypoint's createMultipartUpload() reply, which must be `{ uploadId }`. A named + // field rather than a bare string, so an implementation cannot return some other string (such + // as the key) by mistake. NonCoercible so that a non-string uploadId is rejected rather than + // stringified. A Dict rather than a JSG_STRUCT because structs must be registered in the + // isolate's type list, which is at the compiler's fold-expression nesting limit. + using CreateMultipartUploadBackendResult = jsg::Dict>; + struct ListOptions { jsg::Optional limit; jsg::Optional> prefix; @@ -672,7 +679,8 @@ class R2Bucket: public jsg::Object { const jsg::TypeHandler>& rpcPropHandler, const jsg::TypeHandler< jsg::Function)>>& createFnHandler, - const jsg::TypeHandler>>& uploadIdHandler); + const jsg::TypeHandler>& + createResultHandler); jsg::Promise listRpc(jsg::Lock& js, jsg::Optional options, const jsg::TypeHandler>& rpcPropHandler, diff --git a/src/workerd/api/tests/BUILD.bazel b/src/workerd/api/tests/BUILD.bazel index c6251273d00..fb0dc9f61da 100644 --- a/src/workerd/api/tests/BUILD.bazel +++ b/src/workerd/api/tests/BUILD.bazel @@ -1,6 +1,7 @@ load("@aspect_rules_js//js:defs.bzl", "js_binary") load("@aspect_rules_ts//ts:defs.bzl", "ts_config", "ts_project") load("@rules_shell//shell:sh_test.bzl", "sh_test") +load("//:build/lint_test.bzl", "lint_test") load("//:build/wasm_tools_parse.bzl", "wasm_tools_parse") load("//:build/wd_bench.bzl", "wd_bench") load("//:build/wd_test.bzl", "wd_cli_test", "wd_test") @@ -102,6 +103,35 @@ ts_project( deps = ["//src/node:node@tsproject"], ) +# Type-checks the R2 JSRPC binding contract and its test implementation against the Workers types +# in the checked-in snapshot. Kept apart from :ts_project because the snapshot's globals conflict +# with the DOM, WebWorker, and Node.js types that project uses. +ts_config( + name = "r2_binding_service_ts_config", + src = "r2-binding-service/tsconfig.json", + deps = ["@workerd//tools:base-tsconfig"], +) + +ts_project( + name = "r2_binding_service_ts_project", + srcs = glob(["r2-binding-service/*.ts"]), + composite = True, + tsconfig = ":r2_binding_service_ts_config", + deps = ["//types:generated_snapshot_experimental"], +) + +lint_test( + name = "r2_binding_service", + srcs = glob(["r2-binding-service/*.ts"]), + data = ["//types:generated-snapshot/experimental/index.d.ts"], + eslintrc_json = "@workerd//tools:base.eslint.config.mjs", + no_copy_to_bin = [ + "@workerd//tools:base.eslint.config.mjs", + "//types:generated-snapshot/experimental/index.d.ts", + ], + tsconfig_json = "r2-binding-service/tsconfig.json", +) + wd_test( src = "exported-handler-proxy-test.wd-test", data = ["exported-handler-proxy-test.js"], @@ -573,6 +603,7 @@ wd_test( data = [ "r2-jsrpc-serialization-test.js", "r2-test.js", + ":r2_binding_service_ts_project", ], ) @@ -583,6 +614,7 @@ wd_test( "r2-jsrpc-serialization-test.js", "r2-jsrpc-serialization-test.wd-test", "r2-test.js", + ":r2_binding_service_ts_project", ], lint = False, ) @@ -593,6 +625,7 @@ wd_test( data = [ "r2-jsrpc-passthrough.capnp", "r2-test.js", + ":r2_binding_service_ts_project", ], ) @@ -602,6 +635,7 @@ wd_test( data = [ "r2-jsrpc-passthrough.capnp", "r2-test.js", + ":r2_binding_service_ts_project", ], generate_all_compat_flags_variant = False, ) @@ -613,6 +647,7 @@ wd_test( data = [ "r2-jsrpc-passthrough.capnp", "r2-test.js", + ":r2_binding_service_ts_project", ], ) diff --git a/src/workerd/api/tests/r2-binding-service/r2-binding-service.ts b/src/workerd/api/tests/r2-binding-service/r2-binding-service.ts new file mode 100644 index 00000000000..f2b0e92d365 --- /dev/null +++ b/src/workerd/api/tests/r2-binding-service/r2-binding-service.ts @@ -0,0 +1,133 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +///////////////////////////////////////////////////////////////////////////// +// Binding contract to create a custom binding // +///////////////////////////////////////////////////////////////////////////// +// +// With the r2_binding_jsrpc compatibility flag, an R2 bucket binding that targets a Worker +// entrypoint calls these named JSRPC methods on it instead of speaking the R2 HTTP protocol. +// R2Bucket and R2MultipartUpload in src/workerd/api/r2-bucket.c++ and r2-multipart.c++ are the +// callers; this file describes what they send and what they accept back. +// +// Arguments are what the backend receives, which is not always what the caller passed to the +// public R2Bucket method: workerd validates and normalizes options first, and sends request +// bodies with their exact byte length as a separate argument. Absent option fields may arrive +// as explicit `undefined`. +// +// Results are the native R2 types, so a backend can return what a native R2 binding returns. +// Two constraints the types cannot express: an R2ObjectBody is rejected wherever only object +// metadata is expected, and a native R2Object returned from head, get, put, or +// completeMultipartUpload must include httpMetadata and customMetadata. + +// Request body passed to put and uploadPart. ArrayBufferViews arrive as ArrayBuffers, and +// binary, string, and Blob values above an inline size limit arrive as a ReadableStream. A +// stream's length may not be known on the receiving side; the accompanying valueSize argument +// gives its exact byte length. +export type R2BindingBody = ReadableStream | ArrayBuffer | string | Blob; + +// Range passed to get. Contains only the fields the caller set, so it may be empty. +export type R2BindingRange = { + offset?: number; + length?: number; + suffix?: number; +}; + +// Options for get. `range` is omitted for Headers without a Range header. `ssecKey` is always +// hex-encoded. +export type R2BindingGetOptions = { + onlyIf?: R2Conditional | Headers | undefined; + range?: R2BindingRange | Headers | undefined; + ssecKey?: string | undefined; +}; + +// Options for put. `httpMetadata` is always an object rather than Headers, and `ssecKey` is +// always hex-encoded. Checksums are forwarded as the caller passed them. +export type R2BindingPutOptions = { + onlyIf?: R2Conditional | Headers | undefined; + httpMetadata?: R2HTTPMetadata | undefined; + customMetadata?: Record | undefined; + md5?: ArrayBuffer | ArrayBufferView | string | undefined; + sha1?: ArrayBuffer | ArrayBufferView | string | undefined; + sha256?: ArrayBuffer | ArrayBufferView | string | undefined; + sha384?: ArrayBuffer | ArrayBufferView | string | undefined; + sha512?: ArrayBuffer | ArrayBufferView | string | undefined; + storageClass?: string | undefined; + ssecKey?: string | undefined; +}; + +// Options for list. Without the r2_list_honor_include compatibility flag, `include` always +// requests both kinds of metadata, whatever the caller asked for. +export type R2BindingListOptions = { + limit?: number | undefined; + prefix?: string | undefined; + cursor?: string | undefined; + delimiter?: string | undefined; + startAfter?: string | undefined; + include?: ('httpMetadata' | 'customMetadata')[] | undefined; +}; + +// Options for createMultipartUpload. `httpMetadata` is always an object rather than Headers, +// and `ssecKey` is always hex-encoded. +export type R2BindingMultipartOptions = { + httpMetadata?: R2HTTPMetadata | undefined; + customMetadata?: Record | undefined; + storageClass?: string | undefined; + ssecKey?: string | undefined; +}; + +// Options for uploadPart. `ssecKey` is always hex-encoded. +export type R2BindingUploadPartOptions = { + ssecKey?: string | undefined; +}; + +export type R2BindingService = { + head(key: string): Promise; + + // Returns object metadata without a body when an onlyIf precondition fails. + get( + key: string, + options: R2BindingGetOptions | undefined + ): Promise; + + // `value` is null for an empty put, in which case valueSize is 0. + put( + key: string, + value: R2BindingBody | null, + options: R2BindingPutOptions | undefined, + valueSize: number + ): Promise; + + // The result is ignored. + delete(keys: string | string[]): Promise; + + list(options: R2BindingListOptions | undefined): Promise; + + createMultipartUpload( + key: string, + options: R2BindingMultipartOptions | undefined + ): Promise<{ uploadId: string }>; + + // The returned partNumber is ignored in favour of the requested one, but must be present. + uploadPart( + key: string, + uploadId: string, + partNumber: number, + value: R2BindingBody, + options: R2BindingUploadPartOptions | undefined, + valueSize: number + ): Promise; + + abortMultipartUpload(key: string, uploadId: string): Promise; + + completeMultipartUpload( + key: string, + uploadId: string, + uploadedParts: R2UploadedPart[] + ): Promise; +}; + +///////////////////////////////////////////////////////////////////////////// +// End of the binding contract // +///////////////////////////////////////////////////////////////////////////// diff --git a/src/workerd/api/tests/r2-binding-service/r2-observed-binding.ts b/src/workerd/api/tests/r2-binding-service/r2-observed-binding.ts new file mode 100644 index 00000000000..b6ea66d16af --- /dev/null +++ b/src/workerd/api/tests/r2-binding-service/r2-observed-binding.ts @@ -0,0 +1,119 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +import { WorkerEntrypoint } from 'cloudflare:workers'; +import type { + R2BindingBody, + R2BindingGetOptions, + R2BindingListOptions, + R2BindingMultipartOptions, + R2BindingPutOptions, + R2BindingService, + R2BindingUploadPartOptions, +} from './r2-binding-service'; + +type Env = { + REAL_BUCKET: R2Bucket; +}; + +// JSRPC bodies can arrive as a stream whose length is known only from the separate size argument. +// The native binding requires a known-length stream, so restore it from that argument. +function withKnownLength( + value: T, + valueSize: number +): T | ReadableStream { + if (!(value instanceof ReadableStream)) { + return value; + } + const { readable, writable } = new FixedLengthStream(valueSize); + value.pipeTo(writable).catch(() => {}); + return readable; +} + +// Implements the R2 binding's JSRPC methods by forwarding each call to a native R2 binding, in +// the way a customer Worker can stand in for R2. The JSRPC tests target this entrypoint so the +// shared suite exercises the serialization of native R2 results and errors, while REAL_BUCKET +// reaches the same HTTP mock as the HTTP tests. HTTP requests are rejected to detect any +// transport fallback on the JSRPC hop. +export class ObservedR2Binding + extends WorkerEntrypoint + implements R2BindingService +{ + override fetch(): Response { + throw new Error('R2 JSRPC tests must not use HTTP'); + } + + head(key: string): Promise { + return this.env.REAL_BUCKET.head(key); + } + + get( + key: string, + options: R2BindingGetOptions | undefined + ): Promise { + // The native binding accepts a range with no fields set, which R2Range does not model. + return this.env.REAL_BUCKET.get(key, options as R2GetOptions | undefined); + } + + put( + key: string, + value: R2BindingBody | null, + options: R2BindingPutOptions | undefined, + valueSize: number + ): Promise { + return this.env.REAL_BUCKET.put( + key, + withKnownLength(value, valueSize), + options + ); + } + + delete(keys: string | string[]): Promise { + return this.env.REAL_BUCKET.delete(keys); + } + + list(options: R2BindingListOptions | undefined): Promise { + return this.env.REAL_BUCKET.list(options); + } + + async createMultipartUpload( + key: string, + options: R2BindingMultipartOptions | undefined + ): Promise<{ uploadId: string }> { + const upload = await this.env.REAL_BUCKET.createMultipartUpload( + key, + options + ); + return { uploadId: upload.uploadId }; + } + + uploadPart( + key: string, + uploadId: string, + partNumber: number, + value: R2BindingBody, + options: R2BindingUploadPartOptions | undefined, + valueSize: number + ): Promise { + return this.env.REAL_BUCKET.resumeMultipartUpload(key, uploadId).uploadPart( + partNumber, + withKnownLength(value, valueSize), + options + ); + } + + abortMultipartUpload(key: string, uploadId: string): Promise { + return this.env.REAL_BUCKET.resumeMultipartUpload(key, uploadId).abort(); + } + + completeMultipartUpload( + key: string, + uploadId: string, + uploadedParts: R2UploadedPart[] + ): Promise { + return this.env.REAL_BUCKET.resumeMultipartUpload(key, uploadId).complete( + uploadedParts + ); + } +} diff --git a/src/workerd/api/tests/r2-binding-service/tsconfig.json b/src/workerd/api/tests/r2-binding-service/tsconfig.json new file mode 100644 index 00000000000..9db40a53284 --- /dev/null +++ b/src/workerd/api/tests/r2-binding-service/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../../../../tools/base.tsconfig.json", + "compilerOptions": { + "lib": ["esnext"], + "exactOptionalPropertyTypes": false, + "allowJs": false + }, + "files": ["../../../../../types/generated-snapshot/experimental/index.d.ts"], + "include": ["*.ts"] +} diff --git a/src/workerd/api/tests/r2-jsrpc-passthrough.capnp b/src/workerd/api/tests/r2-jsrpc-passthrough.capnp index 233ea5c183d..6f33c0553ff 100644 --- a/src/workerd/api/tests/r2-jsrpc-passthrough.capnp +++ b/src/workerd/api/tests/r2-jsrpc-passthrough.capnp @@ -24,11 +24,14 @@ const userModules :List(Workerd.Worker.Module) = [ ]; # Exports ObservedR2Binding and the HTTP mock, but no test handlers, so the test runner does not -# run the tests exported by r2-test.js in this service. +# run the tests exported by r2-test.js in this service. ObservedR2Binding is compiled from +# r2-binding-service/r2-observed-binding.ts. const passthroughModules :List(Workerd.Worker.Module) = [ ( name = "worker", - esModule = "import mock, { ObservedR2Binding } from './r2-test.js'; export { ObservedR2Binding }; export default { fetch: mock.fetch };" ), + esModule = "import mock from './r2-test.js'; export { ObservedR2Binding } from './r2-observed-binding.js'; export default { fetch: mock.fetch };" ), ( name = "r2-test.js", esModule = embed "r2-test.js" ), + ( name = "r2-observed-binding.js", + esModule = embed "r2-binding-service/r2-observed-binding.js" ), ]; # r2_list_honor_include. diff --git a/src/workerd/api/tests/r2-jsrpc-serialization-test.js b/src/workerd/api/tests/r2-jsrpc-serialization-test.js index bcc42fed84f..d2968256b0d 100644 --- a/src/workerd/api/tests/r2-jsrpc-serialization-test.js +++ b/src/workerd/api/tests/r2-jsrpc-serialization-test.js @@ -3,7 +3,7 @@ // https://opensource.org/licenses/Apache-2.0 import assert from 'node:assert'; -import { ObservedR2Binding as PassthroughR2Binding } from './r2-test.js'; +import { ObservedR2Binding as PassthroughR2Binding } from './r2-observed-binding.js'; // Extends the shared passthrough so that some keys return a native R2 result of the wrong type // for the method, and adds methods that return native results to the test directly, without the @@ -35,9 +35,18 @@ export class ObservedR2Binding extends PassthroughR2Binding { } async createMultipartUpload(requestKey, options) { - // The R2GW returns the upload ID as a bare string; an object must not be stringified into one. - if (requestKey === 'upload-handle-as-upload-id') { - return { key: requestKey, uploadId: 'upload-handle-id' }; + // The backend must reply with `{ uploadId: string }`. + if (requestKey === 'upload-id-as-string') { + return 'upload-id'; + } + if (requestKey === 'upload-id-as-number') { + return { uploadId: 123 }; + } + if (requestKey === 'upload-id-missing') { + return {}; + } + if (requestKey === 'upload-id-with-key') { + return { key: requestKey, uploadId: 'upload-id' }; } return super.createMultipartUpload(requestKey, options); } @@ -186,9 +195,15 @@ export const nativeR2ResultTests = { ); await aborted.abort(); - await assert.rejects( - () => env.BUCKET.createMultipartUpload('upload-handle-as-upload-id'), - { message: /^internal error; reference = \S+$/ } - ); + for (const key of [ + 'upload-id-as-string', + 'upload-id-as-number', + 'upload-id-missing', + 'upload-id-with-key', + ]) { + await assert.rejects(() => env.BUCKET.createMultipartUpload(key), { + message: /^internal error; reference = \S+$/, + }); + } }, }; diff --git a/src/workerd/api/tests/r2-jsrpc-serialization-test.wd-test b/src/workerd/api/tests/r2-jsrpc-serialization-test.wd-test index f4a22070cc6..12b475b03dd 100644 --- a/src/workerd/api/tests/r2-jsrpc-serialization-test.wd-test +++ b/src/workerd/api/tests/r2-jsrpc-serialization-test.wd-test @@ -31,6 +31,8 @@ const testModules :List(Workerd.Worker.Module) = [ (name = "worker", esModule = "export { nativeR2ResultTests } from './test.js';"), (name = "test.js", esModule = embed "r2-jsrpc-serialization-test.js"), (name = "r2-test.js", esModule = embed "r2-test.js"), + (name = "r2-observed-binding.js", + esModule = embed "r2-binding-service/r2-observed-binding.js"), ]; # Exports no test handlers, so the test runner does not run the tests in r2-test.js here. @@ -40,6 +42,8 @@ const platformWorker :Workerd.Worker = ( esModule = "import mock from './r2-test.js'; export { ObservedR2Binding } from './test.js'; export default { fetch: mock.fetch };"), (name = "test.js", esModule = embed "r2-jsrpc-serialization-test.js"), (name = "r2-test.js", esModule = embed "r2-test.js"), + (name = "r2-observed-binding.js", + esModule = embed "r2-binding-service/r2-observed-binding.js"), ], bindings = [ (name = "REAL_BUCKET", r2Bucket = "platform"), diff --git a/src/workerd/api/tests/r2-test.js b/src/workerd/api/tests/r2-test.js index 30dce5711ec..1ccbef8e215 100644 --- a/src/workerd/api/tests/r2-test.js +++ b/src/workerd/api/tests/r2-test.js @@ -1405,81 +1405,6 @@ const testWorker = { }, }; -// JSRPC bodies can arrive as a stream whose length is known only from the separate size argument. -// The native binding requires a known-length stream, so restore it from that argument. -function withKnownLength(value, valueSize) { - if (!(value instanceof ReadableStream)) { - return value; - } - const { readable, writable } = new FixedLengthStream(valueSize); - value.pipeTo(writable).catch(() => {}); - return readable; -} - -// Implements the R2 binding's JSRPC methods by forwarding each call to a native R2 binding, in -// the way a customer Worker can stand in for R2. The JSRPC tests target this entrypoint so the -// shared suite exercises the serialization of native R2 results and errors, while REAL_BUCKET -// reaches the same HTTP mock as the HTTP tests. HTTP requests are rejected to detect any -// transport fallback on the JSRPC hop. -export class ObservedR2Binding extends WorkerEntrypoint { - fetch() { - throw new Error('R2 JSRPC tests must not use HTTP'); - } - - head(requestKey) { - return this.env.REAL_BUCKET.head(requestKey); - } - - get(requestKey, options) { - return this.env.REAL_BUCKET.get(requestKey, options); - } - - put(requestKey, value, options, valueSize) { - return this.env.REAL_BUCKET.put( - requestKey, - withKnownLength(value, valueSize), - options - ); - } - - delete(keys) { - return this.env.REAL_BUCKET.delete(keys); - } - - list(options) { - return this.env.REAL_BUCKET.list(options); - } - - async createMultipartUpload(requestKey, options) { - const upload = await this.env.REAL_BUCKET.createMultipartUpload( - requestKey, - options - ); - return upload.uploadId; - } - - uploadPart(requestKey, uploadId, partNumber, value, options, valueSize) { - return this.env.REAL_BUCKET.resumeMultipartUpload( - requestKey, - uploadId - ).uploadPart(partNumber, withKnownLength(value, valueSize), options); - } - - abortMultipartUpload(requestKey, uploadId) { - return this.env.REAL_BUCKET.resumeMultipartUpload( - requestKey, - uploadId - ).abort(); - } - - completeMultipartUpload(requestKey, uploadId, uploadedParts) { - return this.env.REAL_BUCKET.resumeMultipartUpload( - requestKey, - uploadId - ).complete(uploadedParts); - } -} - // Test that documents the behaviour of multi-key deletion where one fails and the other succeeds export const deletePerKeyErrorParityTests = { async test(_ctrl, env) { diff --git a/src/workerd/api/tests/tsconfig.json b/src/workerd/api/tests/tsconfig.json index a0e91d3a349..38a713f5175 100644 --- a/src/workerd/api/tests/tsconfig.json +++ b/src/workerd/api/tests/tsconfig.json @@ -9,5 +9,8 @@ "allowJs": false, "types": ["@types/node"] }, - "include": ["**/*.ts"] + "include": ["**/*.ts"], + // r2-binding-service has its own project; the other entries repeat the base config's exclude, + // which this one replaces. + "exclude": ["node_modules", "**/*.mjs", "r2-binding-service"] } diff --git a/types/BUILD.bazel b/types/BUILD.bazel index 14c92c27191..6326bbef72e 100644 --- a/types/BUILD.bazel +++ b/types/BUILD.bazel @@ -1,4 +1,4 @@ -load("@aspect_rules_js//js:defs.bzl", "js_binary", "js_run_binary") +load("@aspect_rules_js//js:defs.bzl", "js_binary", "js_library", "js_run_binary") load("//:build/wd_ts_project.bzl", "wd_ts_project") load("//:build/wd_ts_test.bzl", "wd_ts_test") load("//:build/wd_ts_type_test.bzl", "wd_ts_type_test") @@ -97,3 +97,16 @@ js_run_binary( wd_ts_type_test(src = src) for src in glob(["test/types/**/*.ts"]) ] + +# The checked-in experimental snapshot, for test code that type-checks against Workers types +# without the cost of regenerating them. CI keeps the snapshot equal to the generated output. +js_library( + name = "generated_snapshot_experimental", + srcs = ["generated-snapshot/experimental/index.d.ts"], + visibility = ["//src/workerd/api/tests:__pkg__"], +) + +exports_files( + ["generated-snapshot/experimental/index.d.ts"], + visibility = ["//src/workerd/api/tests:__pkg__"], +) From 47fffff17803e3fd98aca3cb454b6fb9a3b77ac7 Mon Sep 17 00:00:00 2001 From: Bradley Thwaites Date: Fri, 23 Oct 2026 09:34:51 -0400 Subject: [PATCH 09/13] Narrow named R2 JSRPC methods to accept only native R2 API result types --- src/workerd/api/r2-bucket.c++ | 108 +-- src/workerd/api/r2-bucket.h | 64 +- src/workerd/api/r2-rpc.h | 6 +- src/workerd/api/r2.h | 3 +- src/workerd/api/tests/BUILD.bazel | 22 - .../api/tests/r2-body-length-test.wd-test | 41 -- .../api/tests/r2-body-length-ts-test.wd-test | 25 - .../api/tests/r2-jsrpc-response-test.wd-test | 33 - src/workerd/api/tests/r2-test.js | 631 ------------------ 9 files changed, 48 insertions(+), 885 deletions(-) delete mode 100644 src/workerd/api/tests/r2-body-length-test.wd-test delete mode 100644 src/workerd/api/tests/r2-body-length-ts-test.wd-test delete mode 100644 src/workerd/api/tests/r2-jsrpc-response-test.wd-test diff --git a/src/workerd/api/r2-bucket.c++ b/src/workerd/api/r2-bucket.c++ index 549cc0c5c58..c1b7a9ed375 100644 --- a/src/workerd/api/r2-bucket.c++ +++ b/src/workerd/api/r2-bucket.c++ @@ -676,56 +676,23 @@ static uint64_t getResultBodyLength(R2Bucket::HeadResult& object) { return static_cast(bodyLength); } -static jsg::Ref getResultFromSerializable( - jsg::Lock& js, R2Bucket::SerializableGetResult rpc) { - KJ_ASSERT(rpc.kind == "body", "Malformed R2 get RPC result: expected a body result."); - auto body = KJ_ASSERT_NONNULL( - kj::mv(rpc.body), "Malformed R2 get RPC result: body result did not have a body."); - auto cancelReader = kj::defer( - [&] { body.forceCancel(js, js.error("Malformed R2 get RPC result.")).markAsHandled(js); }); - auto object = headResultFromSerializable(js, kj::mv(rpc.object), MissingMetadataPolicy::EMPTY); - - auto& context = IoContext::current(); - auto pipe = newIdentityPipe(getResultBodyLength(*object)); - auto pump = context.waitForDeferredProxy(body.pumpTo(js, kj::mv(pipe.out), EndStream::YES)); - body = JsReadableStream::create( - js, context, kj::heap(kj::mv(pipe.in), kj::mv(pump))); - cancelReader.cancel(); - - auto customMetadata = object->getCustomMetadata(); - return js.alloc(kj::str(object->getName()), kj::str(object->getVersion()), - object->getSize(), kj::str(object->getEtag()), object->getChecksums(), object->getUploaded(), - object->getHttpMetadata(), kj::mv(customMetadata), object->getRange(), - kj::str(object->getStorageClass()), - object->getSSECKeyMd5().map([](kj::StringPtr s) { return kj::str(s); }), kj::mv(body)); -} - jsg::Ref headResultFromBackend( jsg::Lock& js, R2Bucket::HeadBackendResult result, MissingMetadataPolicy policy) { - KJ_SWITCH_ONEOF(result) { - KJ_CASE_ONEOF(native, jsg::Ref) { - if (dynamic_cast(native.get()) != nullptr) { - auto body = static_cast(native.get())->getBody(js); - body.forceCancel(js, js.error("Malformed R2 RPC result: expected object metadata.")) - .markAsHandled(js); - KJ_FAIL_ASSERT( - "Malformed R2 RPC result: expected object metadata, received an object body."); - } - if (policy == MissingMetadataPolicy::EMPTY) { - bool hasHttpMetadata = native->getHttpMetadata() != kj::none; - bool hasCustomMetadata = native->getCustomMetadata() != kj::none; - KJ_ASSERT(hasHttpMetadata, - "Malformed R2 RPC result: object metadata did not include HTTP metadata."); - KJ_ASSERT(hasCustomMetadata, - "Malformed R2 RPC result: object metadata did not include custom metadata."); - } - return kj::mv(native); - } - KJ_CASE_ONEOF(rpc, R2Bucket::SerializableHeadResult) { - return headResultFromSerializable(js, kj::mv(rpc), policy); - } + if (dynamic_cast(result.get()) != nullptr) { + auto body = static_cast(result.get())->getBody(js); + body.forceCancel(js, js.error("Malformed R2 RPC result: expected object metadata.")) + .markAsHandled(js); + KJ_FAIL_ASSERT("Malformed R2 RPC result: expected object metadata, received an object body."); } - KJ_UNREACHABLE; + if (policy == MissingMetadataPolicy::EMPTY) { + bool hasHttpMetadata = result->getHttpMetadata() != kj::none; + bool hasCustomMetadata = result->getCustomMetadata() != kj::none; + KJ_ASSERT(hasHttpMetadata, + "Malformed R2 RPC result: object metadata did not include HTTP metadata."); + KJ_ASSERT(hasCustomMetadata, + "Malformed R2 RPC result: object metadata did not include custom metadata."); + } + return kj::mv(result); } jsg::Promise>> R2Bucket::headRpc(jsg::Lock& js, @@ -739,7 +706,7 @@ jsg::Promise>> R2Bucket::headRpc(jsg::L .then(js, [](jsg::Lock& js, kj::Maybe parsed) -> kj::Maybe> { - // A missing object is null, not an error: the gateway maps the 404 that + // A missing object is null, not an error: the backing entrypoint maps the 404 that // R2Result::objectNotFound() used to represent onto a null return. KJ_IF_SOME(backend, parsed) { return headResultFromBackend(js, kj::mv(backend)); @@ -765,7 +732,8 @@ R2Bucket::getRpc(jsg::Lock& js, KJ_IF_SOME(condition, o.onlyIf) { // Parsed only for validation, so callers get the same errors as the non-RPC path (e.g. // quoted or malformed ETags, invalid dates). The parsed result is intentionally unused: - // the original onlyIf value is forwarded below and the R2GW does its own parsing. + // the original onlyIf value is forwarded below and the backing entrypoint does its own + // parsing. UnwrappedConditional conditional = [&] { KJ_SWITCH_ONEOF(condition) { KJ_CASE_ONEOF(value, Conditional) { @@ -844,27 +812,7 @@ R2Bucket::getRpc(jsg::Lock& js, return kj::Maybe>(kj::mv(native)); } KJ_CASE_ONEOF(native, jsg::Ref) { - return headResultFromBackend( - js, HeadBackendResult(kj::mv(native)), MissingMetadataPolicy::EMPTY); - } - KJ_CASE_ONEOF(rpc, SerializableGetResult) { - if (rpc.kind == "metadata") { - KJ_IF_SOME(body, rpc.body) { - body.forceCancel( - js, js.error("Malformed R2 get RPC result: metadata result had a body.")); - KJ_FAIL_ASSERT("Malformed R2 get RPC result: metadata result had a body."); - } - return headResultFromSerializable( - js, kj::mv(rpc.object), MissingMetadataPolicy::EMPTY); - } - if (rpc.kind == "body") { - return kj::Maybe>(getResultFromSerializable(js, kj::mv(rpc))); - } - - KJ_IF_SOME(body, rpc.body) { - body.forceCancel(js, js.error("Malformed R2 get RPC result: unknown result kind.")); - } - KJ_FAIL_ASSERT("Malformed R2 get RPC result: unknown result kind ", rpc.kind, "."); + return headResultFromBackend(js, kj::mv(native), MissingMetadataPolicy::EMPTY); } } } @@ -910,7 +858,7 @@ jsg::Promise>> R2Bucket::putRpc(jsg::Lo }); // Checksums are validated here so callers get the same errors as the non-RPC path, then - // forwarded unchanged; the R2GW decodes hex strings and buffers itself. + // forwarded unchanged; the backing entrypoint decodes hex strings and buffers itself. bool hashAlreadySpecified = false; const auto validateChecksum = [&](auto checksum, kj::StringPtr algorithm, size_t byteLength, size_t hexLength) -> decltype(checksum) { @@ -942,7 +890,8 @@ jsg::Promise>> R2Bucket::putRpc(jsg::Lo KJ_IF_SOME(condition, o.onlyIf) { // Parsed only for validation, so callers get the same errors as the non-RPC path (e.g. // quoted or malformed ETags, invalid dates). The parsed result is intentionally unused: - // the original onlyIf value is forwarded below and the R2GW does its own parsing. + // the original onlyIf value is forwarded below and the backing entrypoint does its own + // parsing. UnwrappedConditional conditional = [&] { KJ_SWITCH_ONEOF(condition) { KJ_CASE_ONEOF(value, Conditional) { @@ -1605,7 +1554,7 @@ jsg::Promise R2Bucket::listRpc(jsg::Lock& js, jsg::Optional options, const jsg::TypeHandler>& rpcPropHandler, const jsg::TypeHandler)>>& listFnHandler, - const jsg::TypeHandler>& listResultHandler, + const jsg::TypeHandler>& listResultHandler, CompatibilityFlags::Reader flags) { return js.evalNow([&] { const bool honorsIncludes = flags.getR2ListHonorIncludeFields(); @@ -1649,19 +1598,12 @@ jsg::Promise R2Bucket::listRpc(jsg::Lock& js, rpcOptions = kj::mv(normalized); } - return callR2RpcMethod(js, getRpcMethod(js, "list"_kj), rpcPropHandler, + return callR2RpcMethod(js, getRpcMethod(js, "list"_kj), rpcPropHandler, listFnHandler, listResultHandler, kj::mv(rpcOptions)) - .then(js, [](jsg::Lock& js, ListBackendResult backend) -> ListResult { - ListResult result; - result.objects = KJ_MAP(object, backend.objects) { + .then(js, [](jsg::Lock& js, ListResult result) -> ListResult { + result.objects = KJ_MAP(object, result.objects) { return headResultFromBackend(js, kj::mv(object), MissingMetadataPolicy::ABSENT); }; - result.truncated = backend.truncated; - KJ_IF_SOME(cursor, backend.cursor) { - result.cursor = kj::mv(cursor); - } - result.delimitedPrefixes = - kj::mv(backend.delimitedPrefixes).orDefault(kj::heapArray(0)); return kj::mv(result); }); }); diff --git a/src/workerd/api/r2-bucket.h b/src/workerd/api/r2-bucket.h index 51650fc67eb..d85338b0a70 100644 --- a/src/workerd/api/r2-bucket.h +++ b/src/workerd/api/r2-bucket.h @@ -260,18 +260,14 @@ class R2Bucket: public jsg::Object { JSG_STRUCT_TS_OVERRIDE(R2MultipartOptions); }; - // Serializable* types: values exchanged with the R2 gateway worker (R2GW) over JSRPC where the - // public type cannot be used directly. Most RPC inputs (PutOptions, ListOptions, Conditional) - // are sent as their public types; SerializableGetOptions exists only because of `range` (see - // below). + // Internal JSRPC payload types. SerializableGetOptions is the normalized request shape used + // because the public Range type cannot represent the exact object sent to the backing + // entrypoint (see below). SerializableChecksums, SerializableHeadResult, and + // SerializableGetResult are the payloads used by the custom serializers for the corresponding + // native R2 resource types. RPC reconstructs those native resources before a named R2 method's + // result is unwrapped. // - // The R2GW returns results as plain objects. Only a JSG_STRUCT can be unwrapped from a plain - // object, so the resource types (Checksums, HeadResult, GetResult) cannot receive them; the - // Serializable* result structs receive them instead and are converted into the public types - // before being handed to the caller. - // - // These are NOT customer-facing types and must never be returned to user code, so they carry - // no TS overrides. + // These are not customer-facing types, so they carry no TS overrides. struct SerializableGetOptions { jsg::Optional>> onlyIf; @@ -283,7 +279,7 @@ class R2Bucket: public jsg::Object { JSG_STRUCT(onlyIf, range, ssecKey); }; - // Object metadata as returned by the R2GW. HeadResult / Checksums are built from these. + // Serialized Checksums payload. struct SerializableChecksums { jsg::Optional> md5; jsg::Optional> sha1; @@ -294,13 +290,8 @@ class R2Bucket: public jsg::Object { JSG_STRUCT(md5, sha1, sha256, sha384, sha512); }; - // Field names match the gateway's R2ObjectRpc, not HeadResult's members: the - // key arrives as `key` where HeadResult stores it as `name`. - // - // `kj::Maybe` rather than `jsg::Optional` throughout, because jsg::Optional - // accepts `undefined` but not `null`, and only kj::Maybe tolerates both. The - // gateway omits absent fields today, but that is an unenforced cross-repo - // invariant and a null would otherwise be a hard unwrap failure. + // The serialized field is `key`, while HeadResult stores it internally as `name`. + // `kj::Maybe` accepts both undefined and null for absent optional fields in a payload. struct SerializableHeadResult { kj::String key; kj::String version; @@ -556,9 +547,8 @@ class R2Bucket: public jsg::Object { } }; - using HeadBackendResult = kj::OneOf, SerializableHeadResult>; - using GetBackendResult = - kj::OneOf, jsg::Ref, SerializableGetResult>; + using HeadBackendResult = jsg::Ref; + using GetBackendResult = kj::OneOf, jsg::Ref>; struct ListResult { kj::Array> objects; @@ -576,15 +566,6 @@ class R2Bucket: public jsg::Object { )); }; - struct ListBackendResult { - kj::Array objects; - bool truncated; - kj::Maybe cursor; - kj::Maybe> delimitedPrefixes; - - JSG_STRUCT(objects, truncated, cursor, delimitedPrefixes); - }; - // The backing entrypoint's createMultipartUpload() reply, which must be `{ uploadId }`. A named // field rather than a bare string, so an implementation cannot return some other string (such // as the key) by mistake. NonCoercible so that a non-string uploadId is rejected rather than @@ -634,18 +615,11 @@ class R2Bucket: public jsg::Object { kj::OneOf> keys, const jsg::TypeHandler>& errorType); - // JSRPC equivalents of the above, selected by JSG_RESOURCE_TYPE when the r2_binding_jsrpc compatibility flag is - // on. It dispatches to the gateway's R2BindingEntrypoint instead of - // synthesising an HTTP request, then rebuilds the public result types from the - // plain data JSRPC delivers. - // - // These keep ordinary typed signatures rather than taking a raw - // v8::FunctionCallbackInfo the way KvNamespace::deleteBulk does. A raw-args - // passthrough cannot work here: it returns the JsRpcPromise directly, so the - // caller receives the gateway's plain data and R2Object's methods and sync - // accessors are gone. Reconstruction needs the resolved value, which needs a - // real promise, which needs a typed return, which needs an injected - // TypeHandler -- and TypeHandlers are only injected into typed signatures. + // JSRPC equivalents of the above, selected by JSG_RESOURCE_TYPE when the r2_binding_jsrpc + // compatibility flag is on. They dispatch to the backing entrypoint instead of synthesizing an + // HTTP request. Typed signatures provide the TypeHandlers that validate the entrypoint contract + // and unwrap the custom JSRPC thenable. Native R2 results are reconstructed by their custom RPC + // deserializers before these handlers receive them. jsg::Promise>> headRpc(jsg::Lock& js, kj::String key, const jsg::TypeHandler>& rpcPropHandler, @@ -685,7 +659,7 @@ class R2Bucket: public jsg::Object { jsg::Optional options, const jsg::TypeHandler>& rpcPropHandler, const jsg::TypeHandler)>>& listFnHandler, - const jsg::TypeHandler>& listResultHandler, + const jsg::TypeHandler>& listResultHandler, CompatibilityFlags::Reader flags); jsg::Promise list(jsg::Lock& js, jsg::Optional options, @@ -815,7 +789,7 @@ class R2Bucket: public jsg::Object { kj::LiteralStringConst operation, kj::Maybe key = kj::none); - // Look up a method on the gateway's entrypoint over this binding's subrequest + // Look up a method on the backing entrypoint over this binding's subrequest // channel. jsg::Ref getRpcMethod(jsg::Lock& js, kj::StringPtr methodName); diff --git a/src/workerd/api/r2-rpc.h b/src/workerd/api/r2-rpc.h index 33898df7664..c510d2a1aeb 100644 --- a/src/workerd/api/r2-rpc.h +++ b/src/workerd/api/r2-rpc.h @@ -44,8 +44,8 @@ jsg::Promise unwrapR2RpcPromise(jsg::Lock& js, return KJ_ASSERT_NONNULL( resultPromiseHandler.tryUnwrap(js, normalizedPromise.consumeHandle(js))); } else { - // Decode only fulfilled responses so gateway rejections retain their public API errors. - // Failures converting a response into the internal wire types are gateway contract violations. + // Decode only fulfilled responses so backing-entrypoint rejections retain their public API + // errors. Failures converting a response into the internal wire types are contract violations. return normalizedPromise.then(js, [&resultPromiseHandler](jsg::Lock& js, jsg::Value value) { auto fulfilled = js.resolvedPromise(kj::mv(value)); auto parsed = @@ -127,7 +127,7 @@ struct PreparedR2RpcBody { }; // Prepares the transport value and the exact byte length passed alongside it. The length is an -// internal argument to the gateway's named RPC method, not part of the public R2 API. +// internal argument to the backing entrypoint's named RPC method, not part of the public R2 API. PreparedR2RpcBody prepareR2RpcBody(jsg::Lock& js, R2PutValue& value); struct R2Result { diff --git a/src/workerd/api/r2.h b/src/workerd/api/r2.h index eb47e1cd908..daa976bb626 100644 --- a/src/workerd/api/r2.h +++ b/src/workerd/api/r2.h @@ -21,7 +21,6 @@ namespace workerd::api::public_beta { api::public_beta::R2Bucket::SerializableChecksums, \ api::public_beta::R2Bucket::SerializableHeadResult, \ api::public_beta::R2Bucket::SerializableGetOptions, \ - api::public_beta::R2Bucket::SerializableGetResult, \ - api::public_beta::R2Bucket::ListBackendResult + api::public_beta::R2Bucket::SerializableGetResult // The list of r2 types that are added to worker.c++'s JSG_DECLARE_ISOLATE_TYPE } // namespace workerd::api::public_beta diff --git a/src/workerd/api/tests/BUILD.bazel b/src/workerd/api/tests/BUILD.bazel index fb0dc9f61da..5d8507cf2f7 100644 --- a/src/workerd/api/tests/BUILD.bazel +++ b/src/workerd/api/tests/BUILD.bazel @@ -575,28 +575,6 @@ wd_test( ], ) -wd_test( - src = "r2-body-length-test.wd-test", - args = ["--experimental"], - data = ["r2-test.js"], -) - -wd_test( - src = "r2-body-length-ts-test.wd-test", - args = ["--experimental"], - data = [ - "r2-body-length-test.wd-test", - "r2-test.js", - ], - lint = False, -) - -wd_test( - src = "r2-jsrpc-response-test.wd-test", - args = ["--experimental"], - data = ["r2-test.js"], -) - wd_test( src = "r2-jsrpc-serialization-test.wd-test", args = ["--experimental"], diff --git a/src/workerd/api/tests/r2-body-length-test.wd-test b/src/workerd/api/tests/r2-body-length-test.wd-test deleted file mode 100644 index 669ff35145e..00000000000 --- a/src/workerd/api/tests/r2-body-length-test.wd-test +++ /dev/null @@ -1,41 +0,0 @@ -using Workerd = import "/workerd/workerd.capnp"; - -# Run the same body-length checks against one mock gateway using two callers: HTTP and JSRPC. -# The JSRPC caller also has a plain service binding to separate RPC behavior from R2 parsing. -# Only the body-length tests are exported here, keeping this matrix separate from other tests. -const unitTests :Workerd.Config = ( - services = [ - (name = "http", worker = ( - modules = .testModules, - bindings = [ - (name = "BUCKET", r2Bucket = (name = "gateway", entrypoint = "R2BodyLengthEntrypoint")), - (name = "R2_BODY_LENGTH_TRANSPORT", text = "http"), - ], - compatibilityFlags = ["nodejs_compat", "streams_enable_constructors"], - )), - (name = "jsrpc", worker = ( - modules = .testModules, - bindings = [ - (name = "BUCKET", r2Bucket = (name = "gateway", entrypoint = "R2BodyLengthEntrypoint")), - (name = "SERVICE", service = (name = "gateway", entrypoint = "R2BodyLengthEntrypoint")), - (name = "R2_BODY_LENGTH_TRANSPORT", text = "jsrpc"), - ], - compatibilityFlags = ["experimental", "nodejs_compat", "streams_enable_constructors", - "r2_binding_jsrpc", "rpc", "fetcher_no_get_put_delete"], - )), - (name = "gateway", worker = .gatewayWorker), - ], -); - -const gatewayWorker :Workerd.Worker = ( - modules = [ - (name = "worker", esModule = "export { R2BodyLengthEntrypoint } from './r2-test.js';"), - (name = "r2-test.js", esModule = embed "r2-test.js"), - ], - compatibilityFlags = ["nodejs_compat", "streams_enable_constructors"], -); - -const testModules :List(Workerd.Worker.Module) = [ - (name = "worker", esModule = "export { r2BodyLengthTests } from './r2-test.js';"), - (name = "r2-test.js", esModule = embed "r2-test.js"), -]; diff --git a/src/workerd/api/tests/r2-body-length-ts-test.wd-test b/src/workerd/api/tests/r2-body-length-ts-test.wd-test deleted file mode 100644 index 5387564b58b..00000000000 --- a/src/workerd/api/tests/r2-body-length-ts-test.wd-test +++ /dev/null @@ -1,25 +0,0 @@ -using Workerd = import "/workerd/workerd.capnp"; -using R2BodyLength = import "r2-body-length-test.wd-test"; - -# The TS backend needs bootstrap/hydration autogates. A separate config lets the normal -# R2 tests also exercise the legacy receiver with those autogates disabled. -const unitTests :Workerd.Config = ( - services = [ - (name = "jsrpc-ts", worker = ( - modules = R2BodyLength.testModules, - bindings = [ - (name = "BUCKET", r2Bucket = (name = "gateway", entrypoint = "R2BodyLengthEntrypoint")), - (name = "SERVICE", service = (name = "gateway", entrypoint = "R2BodyLengthEntrypoint")), - (name = "R2_BODY_LENGTH_TRANSPORT", text = "jsrpc-ts"), - ], - compatibilityFlags = ["experimental", "nodejs_compat", "streams_enable_constructors", - "r2_binding_jsrpc", "rpc", "fetcher_no_get_put_delete", - "typescript_implemented_streams"], - )), - (name = "gateway", worker = R2BodyLength.gatewayWorker), - ], - autogates = [ - "workerd-autogate-per-isolate-javascript-bootstrap", - "workerd-autogate-rpc-externals-hydration", - ], -); diff --git a/src/workerd/api/tests/r2-jsrpc-response-test.wd-test b/src/workerd/api/tests/r2-jsrpc-response-test.wd-test deleted file mode 100644 index 5c866746613..00000000000 --- a/src/workerd/api/tests/r2-jsrpc-response-test.wd-test +++ /dev/null @@ -1,33 +0,0 @@ -using Workerd = import "/workerd/workerd.capnp"; - -# Response decoding must report internal errors with either struct-wrapping implementation. -const unitTests :Workerd.Config = ( - services = [ - ( name = "fast-structs", - worker = ( - modules = [ - (name = "worker", esModule = "import { R2JsrpcResponseEntrypoint, testInvalidR2Responses } from './r2-test.js'; export { R2JsrpcResponseEntrypoint }; export const InvalidR2ResponseTests = { test: testInvalidR2Responses };"), - (name = "r2-test.js", esModule = embed "r2-test.js"), - ], - bindings = [ - (name = "BUCKET", r2Bucket = (name = "fast-structs", entrypoint = "R2JsrpcResponseEntrypoint")), - ], - compatibilityFlags = ["experimental", "nodejs_compat", "service_binding_extra_handlers", - "streams_enable_constructors", "r2_binding_jsrpc", "enable_fast_jsg_struct"], - ), - ), - ( name = "slow-structs", - worker = ( - modules = [ - (name = "worker", esModule = "import { R2JsrpcResponseEntrypoint, testInvalidR2Responses } from './r2-test.js'; export { R2JsrpcResponseEntrypoint }; export const InvalidR2ResponseTests = { test: testInvalidR2Responses };"), - (name = "r2-test.js", esModule = embed "r2-test.js"), - ], - bindings = [ - (name = "BUCKET", r2Bucket = (name = "slow-structs", entrypoint = "R2JsrpcResponseEntrypoint")), - ], - compatibilityFlags = ["experimental", "nodejs_compat", "service_binding_extra_handlers", - "streams_enable_constructors", "r2_binding_jsrpc", "disable_fast_jsg_struct"], - ), - ), - ], -); diff --git a/src/workerd/api/tests/r2-test.js b/src/workerd/api/tests/r2-test.js index 1ccbef8e215..5328317aec0 100644 --- a/src/workerd/api/tests/r2-test.js +++ b/src/workerd/api/tests/r2-test.js @@ -95,47 +95,6 @@ const HeadObject = { version: 'objectVersion', key, }; -const object = { - key: 'key', - version: 'version', - size: 0, - etag: 'etag', - uploaded: new Date(0), - storageClass: 'Standard', - checksums: {}, -}; -const invalidMetadata = { - negativeSize: { size: -1 }, - fractionalSize: { size: 0.5 }, - infiniteSize: { size: Infinity }, - nanSize: { size: NaN }, - negativeOffset: { range: { offset: -1 } }, - fractionalOffset: { range: { offset: 0.5 } }, - negativeLength: { range: { length: -1 } }, - fractionalLength: { range: { length: 0.5 } }, - negativeSuffix: { range: { suffix: -1 } }, - fractionalSuffix: { range: { suffix: 0.5 } }, - suffixWithOffset: { range: { suffix: 1, offset: 0 } }, - suffixWithLength: { range: { suffix: 1, length: 1 } }, - invalidDate: { uploaded: new Date(NaN) }, - invalidChecksum: { checksums: { md5: {} } }, -}; - -// Object metadata in the shape a JSRPC R2 gateway returns. -function buildRpcHead() { - return { - key, - version: objResponse.version, - size: Number(objResponse.size), - etag: objResponse.etag, - uploaded: new Date(Number(objResponse.uploaded)), - storageClass: objResponse.storageClass, - checksums: {}, - httpMetadata: {}, - customMetadata: {}, - }; -} - // The HTTP protocol reports failures with a status code and a CF-R2-Error header, which the // binding turns into an Error whose message is ": ()". function r2ErrorHeaders({ v4code, message }) { @@ -1845,594 +1804,4 @@ export const r2StringUploadPartTests = { }, }; -// The source-kind matrix uses the same four bytes and reports object.size = 4. It checks whether -// the stream itself knows its byte count: PUT receives only the body, not object.size. -const bodyLengthBytes = new TextEncoder().encode('data'); -const bodyLengthCopyCases = { - empty: { content: '', size: 0 }, - utf8: { content: 'café🌍', size: 9 }, - chunks: { content: 'data', size: 4 }, - bounded: { - content: 'at', - size: 4, - options: { range: { offset: 1, length: 2 } }, - range: { offset: 1, length: 2 }, - }, - offset: { - content: 'ta', - size: 4, - options: { range: { offset: 2 } }, - range: { offset: 2, length: 2 }, - }, - suffix: { - content: 'ta', - size: 4, - options: { range: { suffix: 2 } }, - range: { offset: 2, length: 2 }, - }, - shortened: { - content: 'ta', - size: 4, - options: { range: { offset: 2, length: 99 } }, - range: { offset: 2, length: 2 }, - }, - remaining: { content: 'ata', size: 3 }, -}; -let bodyLengthReleased; -let bodyLengthCancellation; - -function bodyLengthUploadBytes(requestKey) { - return requestKey === 'destination' - ? bodyLengthBytes - : new TextEncoder().encode(bodyLengthCopyCases[requestKey].content); -} - -// The HTTP gateway sends [JSON metadata][object bytes]. A buffer gives the HTTP transport -// a known length; wrapping the same bytes in an ordinary JS stream does not. -function buildBodyLengthResponse(mode) { - const metadata = new TextEncoder().encode( - JSON.stringify({ ...objResponse, size: String(bodyLengthBytes.byteLength) }) - ); - const payload = new Uint8Array( - metadata.byteLength + bodyLengthBytes.byteLength - ); - payload.set(metadata); - payload.set(bodyLengthBytes, metadata.byteLength); - return new Response( - mode === 'wrapped' - ? new ReadableStream({ - type: 'bytes', - start(controller) { - controller.enqueue(payload); - controller.close(); - }, - }) - : payload, - { headers: { 'cf-r2-metadata-size': String(metadata.byteLength) } } - ); -} - -async function createBodyLengthStream(mode, ctx) { - if (mode === 'native') { - // A response made from a fixed-size buffer already knows how many bytes it contains. - return new Response(bodyLengthBytes).body; - } - if (mode === 'fixed') { - // Explicitly give the stream its byte count using Cloudflare's FixedLengthStream. - const fixed = new FixedLengthStream(bodyLengthBytes.byteLength); - ctx.waitUntil(new Response(bodyLengthBytes).body.pipeTo(fixed.writable)); - return fixed.readable; - } - if (mode === 'expected') { - // Cloudflare's nonstandard expectedLength option also gives a JS stream a byte count. - return new ReadableStream({ - type: 'bytes', - expectedLength: bodyLengthBytes.byteLength, - start(controller) { - controller.enqueue(bodyLengthBytes.slice()); - controller.close(); - }, - }); - } - assert.strictEqual(mode, 'wrapped'); - // Model the gateway's splitMetadataPrefix(): read the JSON, then forward the remaining - // bytes through a new JS stream. Reading size from the JSON does not give that new stream - // a known length. metadataSize counts only the JSON prefix, not the four object bytes. - const response = buildBodyLengthResponse('native'); - const metadataSize = Number(response.headers.get('cf-r2-metadata-size')); - const reader = response.body.getReader({ mode: 'byob' }); - const { value } = await reader.readAtLeast( - metadataSize, - new Uint8Array(metadataSize) - ); - assert.strictEqual( - JSON.parse(new TextDecoder().decode(value)).size, - String(bodyLengthBytes.byteLength) - ); - return new ReadableStream({ - type: 'bytes', - async pull(controller) { - const next = await reader.read(new Uint8Array(64 * 1024)); - if (!next.done) { - controller.enqueue(next.value); - } else { - reader.releaseLock(); - controller.close(); - } - }, - async cancel(reason) { - try { - await reader.cancel(reason); - } finally { - reader.releaseLock(); - } - }, - }); -} - -// A mock gateway serving both HTTP and RPC. Successful uploads must contain exactly "data", -// so the copy tests check the actual bytes, not just whether PUT returns successfully. -export class R2BodyLengthEntrypoint extends WorkerEntrypoint { - releaseBody() { - bodyLengthReleased = true; - } - - waitForCancellation() { - return bodyLengthCancellation; - } - - async fetch(request) { - if (request.method === 'GET') { - const metadata = JSON.parse(request.headers.get('cf-r2-request')); - assert.strictEqual(metadata.method, 'get'); - return buildBodyLengthResponse(metadata.object); - } - assert.strictEqual(request.method, 'PUT'); - const metadataSize = Number(request.headers.get('cf-r2-metadata-size')); - const payload = new Uint8Array(await request.arrayBuffer()); - const metadata = JSON.parse( - new TextDecoder().decode(payload.subarray(0, metadataSize)) - ); - assert.strictEqual(metadata.method, 'put'); - assert.strictEqual(metadata.object, 'destination'); - assert.deepStrictEqual(payload.subarray(metadataSize), bodyLengthBytes); - return Response.json({ - ...objResponse, - name: 'destination', - size: String(bodyLengthBytes.byteLength), - }); - } - - async get(mode, options) { - const object = { ...buildRpcHead(), size: bodyLengthBytes.byteLength }; - if (mode === 'missing') { - return null; - } - if (mode === 'metadata') { - return { kind: 'metadata', object }; - } - if (mode === 'deferred') { - bodyLengthReleased = false; - return { - kind: 'body', - object, - body: new ReadableStream({ - async pull(controller) { - // The test gateway waits in its own request context; releaseBody() only flips - // a flag. This timer models delayed upstream I/O, not a wait in the R2 caller. - while (!bodyLengthReleased) { - await scheduler.wait(1); - } - controller.enqueue(bodyLengthBytes.slice()); - controller.close(); - }, - }), - }; - } - if (mode === 'cancel' || mode === 'invalid-size') { - if (mode === 'invalid-size') { - object.size = -1; - } - let cancelled; - bodyLengthCancellation = new Promise((resolve) => { - cancelled = resolve; - }); - return { - kind: 'body', - object, - body: new ReadableStream({ - pull(controller) { - controller.enqueue(bodyLengthBytes.slice()); - }, - cancel() { - cancelled(); - }, - }), - }; - } - if (mode === 'upstream-error') { - return { - kind: 'body', - object, - body: new ReadableStream({ - pull(controller) { - controller.error(new Error('body source failed')); - }, - }), - }; - } - const invalidSizes = { short: 5, long: 3, 'zero-long': 0, unsafe: 2 ** 53 }; - if (Object.hasOwn(invalidSizes, mode)) { - object.size = invalidSizes[mode]; - return { kind: 'body', object, body: new Response(bodyLengthBytes).body }; - } - if (mode === 'unsafe-range') { - object.range = { offset: 0, length: 2 ** 53 }; - return { kind: 'body', object, body: new Response(bodyLengthBytes).body }; - } - if (Object.hasOwn(bodyLengthCopyCases, mode)) { - const fixture = bodyLengthCopyCases[mode]; - assert.deepStrictEqual(options?.range, fixture.options?.range); - object.size = fixture.size; - if (fixture.range !== undefined) { - object.range = fixture.range; - } - const bytes = new TextEncoder().encode(fixture.content); - let offset = 0; - return { - kind: 'body', - object, - body: new ReadableStream({ - type: 'bytes', - pull(controller) { - if (offset < bytes.byteLength) { - controller.enqueue(bytes.slice(offset, ++offset)); - } else { - controller.close(); - } - }, - }), - }; - } - return { - kind: 'body', - object, - body: await createBodyLengthStream(mode, this.ctx), - }; - } - - async put(requestKey, value, options, valueSize) { - const expected = bodyLengthUploadBytes(requestKey); - // workerd supplies this hidden RPC argument from the stream's known length. - assert.strictEqual(valueSize, expected.byteLength); - assert.deepStrictEqual( - new Uint8Array(await new Response(value).arrayBuffer()), - expected - ); - return { - ...buildRpcHead(), - key: requestKey, - size: expected.byteLength, - }; - } - - async uploadPart( - requestKey, - uploadId, - partNumber, - value, - options, - valueSize - ) { - assert.strictEqual(uploadId, 'body-length-upload'); - const expected = bodyLengthUploadBytes(requestKey); - assert.strictEqual(valueSize, expected.byteLength); - assert.deepStrictEqual( - new Uint8Array(await new Response(value).arrayBuffer()), - expected - ); - return { partNumber, etag: 'body-length-etag' }; - } -} - -async function assertBodyLengthCopy(bucket, source, knownLength, label) { - // knownLength is the expected test outcome, not information supplied to PUT. PUT receives - // only the stream and must discover its remaining length itself; it cannot use object.size. - // Start PUT before any await, so the "immediate" cases cannot accidentally let RPC settle. - const copying = bucket.put('destination', source); - if (knownLength) { - const result = await copying.catch((error) => { - error.message = `${label}: ${error.message}`; - throw error; - }); - assert.strictEqual(result.key, 'destination', label); - assert.strictEqual(result.size, bodyLengthBytes.byteLength, label); - } else { - // Ordinary unsized streams are still invalid PUT inputs. Service RPC can also return - // a sized stream before its receiving endpoint has finished setting up. - await assert.rejects( - copying, - { - name: 'TypeError', - message: - 'Provided readable stream must have a known length (request/response body or readable half of FixedLengthStream)', - }, - label - ); - } -} - -export const r2BodyLengthTests = { - async test(controller, env, ctx) { - // Only the dedicated body-length configuration runs these checks; the main R2 suites - // also import this file but do not provide this binding. - if (env.R2_BODY_LENGTH_TRANSPORT === undefined) { - return; - } - if (env.R2_BODY_LENGTH_TRANSPORT === 'http') { - // HTTP control: native GET -> PUT succeeds because the original stream keeps its - // length tracking after the JSON prefix is read. The wrapped version fails even - // though both results still expose the correct object.size. - for (const mode of ['native', 'wrapped']) { - const result = await env.BUCKET.get(mode); - assert.strictEqual(result.size, bodyLengthBytes.byteLength); - await assertBodyLengthCopy( - env.BUCKET, - result.body, - mode === 'native', - `HTTP ${mode}` - ); - } - return; - } - - // Repeat the same copy attempt with four sources, all containing the four bytes "data": - // - native: a Response body backed by a byte array, with a known length. - // - wrapped: a new JS stream forwarding bytes after the gateway reads the metadata prefix; - // the metadata says size=4, but this forwarding stream has no declared length. - // - fixed: the readable side of a FixedLengthStream(4). - // - expected: a JS byte stream with expectedLength=4. - // - // For each source, first try PUT with a locally created stream. Then get a fresh stream - // through each binding and try PUT either immediately or after one event-loop turn. - // This separates the source's own length from what the receiving RPC stream can report. - for (const mode of ['native', 'wrapped', 'fixed', 'expected']) { - // Create the body in the caller, without receiving it over RPC first. All three - // known-length sources copy successfully; the ordinary wrapped stream is rejected. - await assertBodyLengthCopy( - env.BUCKET, - await createBodyLengthStream(mode, ctx), - mode !== 'wrapped', - `local ${mode}` - ); - // Both bindings call the same fake gateway's get(mode) over RPC. Both receive the - // stream and metadata saying size=4, but they process that result differently: - // - BUCKET runs R2's getRpc(), which uses the metadata to create a local fixed-length - // wrapper. Even a "wrapped" source can therefore be copied immediately. - // - SERVICE returns the RPC result directly. Its stream's length depends on asynchronous - // stream setup; ordinary service RPC does not interpret the sibling object.size field. - // - // Expected PUT outcomes ("sized" means native, fixed, or expected): - // Receiving path / source Immediate PUT PUT after an event-loop turn - // BUCKET / any succeeds succeeds - // SERVICE, legacy / sized length error succeeds - // SERVICE, legacy / wrapped length error length error - // SERVICE, TypeScript / any length error length error - // - // The legacy receiver can discover a declared length once stream setup completes. - // The TypeScript receiver snapshots the initially unknown length, so waiting does not - // help it. The dedicated jsrpc-ts configuration runs this same loop with that receiver. - for (const transport of ['BUCKET', 'SERVICE']) { - for (const timing of ['immediate', 'after-event-loop-turn']) { - // Fetch again for every attempt: a stream consumed or cancelled by one PUT must - // not be reused by the next. Awaiting get() alone need not finish RPC stream setup. - const result = await env[transport].get(mode); - // Metadata is correct in every case, including cases whose stream is still unsized. - assert.strictEqual( - transport === 'BUCKET' ? result.size : result.object.size, - bodyLengthBytes.byteLength - ); - if (timing === 'after-event-loop-turn') { - // Let pending RPC setup run without consuming body bytes. This is a diagnostic - // event-loop yield, not a suggested workaround for application code. - await scheduler.wait(0); - } - await assertBodyLengthCopy( - env.BUCKET, - result.body, - // This boolean selects the expected outcome from the table above. All uploads - // use BUCKET.put(); transport selects only how we obtained the source body. - transport === 'BUCKET' || - (env.R2_BODY_LENGTH_TRANSPORT !== 'jsrpc-ts' && - timing === 'after-event-loop-turn' && - mode !== 'wrapped'), - `${transport} ${mode} ${timing}` - ); - } - } - } - - // The returned range describes the actual body, even when the requested range is longer. - // Empty bodies and multibyte UTF-8 text also need byte counts, not character counts. - for (const [requestKey, fixture] of Object.entries(bodyLengthCopyCases)) { - const result = await env.BUCKET.get(requestKey, fixture.options); - assert.strictEqual(result.size, fixture.size); - assert.strictEqual(result.range?.offset, fixture.range?.offset); - assert.strictEqual(result.range?.length, fixture.range?.length); - const copied = await env.BUCKET.put(requestKey, result.body); - assert.strictEqual( - copied.size, - new TextEncoder().encode(fixture.content).byteLength - ); - } - - // GET must finish before any bytes are available. Releasing the source only afterwards - // catches implementations that wait for the first byte or buffer the whole body. - const deferred = await env.BUCKET.get('deferred'); - assert.strictEqual(deferred.bodyUsed, false); - assert.strictEqual(deferred.body.locked, false); - const copying = env.BUCKET.put('destination', deferred.body); - await env.SERVICE.releaseBody(); - assert.strictEqual((await copying).size, bodyLengthBytes.byteLength); - - // A partial read must reduce the known length. Tee gives us an undisturbed stream for - // RPC transfer; its length must describe only the three unread bytes, not all four. - const partial = await env.BUCKET.get('chunks'); - const reader = partial.body.getReader({ mode: 'byob' }); - const first = await reader.read(new Uint8Array(1)); - assert.deepStrictEqual(first.value, bodyLengthBytes.slice(0, 1)); - reader.releaseLock(); - const [remaining, discarded] = partial.body.tee(); - const discarding = discarded.cancel(); - assert.strictEqual((await env.BUCKET.put('remaining', remaining)).size, 3); - await discarding; - - // Multipart uploads use the same known-length requirement as PUT, including ranged bodies. - for (const requestKey of ['destination', 'bounded']) { - const fixture = bodyLengthCopyCases[requestKey]; - const upload = env.BUCKET.resumeMultipartUpload( - requestKey, - 'body-length-upload' - ); - const source = await env.BUCKET.get( - requestKey === 'destination' ? 'wrapped' : requestKey, - fixture?.options - ); - assert.deepStrictEqual(await upload.uploadPart(1, source.body), { - partNumber: 1, - etag: 'body-length-etag', - }); - } - - // Metadata is a promise about the bytes, not permission to truncate or pad a bad response. - for (const mode of ['short', 'long', 'zero-long']) { - const result = await env.BUCKET.get(mode); - // Read to EOF explicitly: text() can return immediately when the declared length is zero. - const reader = result.body.getReader(); - await assert.rejects( - async () => { - let chunk; - do { - chunk = await reader.read(); - } while (!chunk.done); - }, - { name: 'TypeError' }, - mode - ); - } - await assert.rejects(env.BUCKET.get('unsafe'), { - message: /^internal error; reference = \S+$/, - }); - await assert.rejects(env.BUCKET.get('unsafe-range'), { - message: /^internal error; reference = \S+$/, - }); - - // Wrapping must preserve upstream errors and let consumer cancellation reach the gateway. - await assert.rejects( - async () => (await env.BUCKET.get('upstream-error')).text(), - { - message: 'ReadableStream received over RPC disconnected prematurely.', - } - ); - const cancelled = await env.BUCKET.get('cancel'); - await cancelled.body.cancel('no longer needed'); - await env.SERVICE.waitForCancellation(); - - // A rejected metadata result must cancel its body rather than leave the gateway pumping. - await assert.rejects(env.BUCKET.get('invalid-size'), { - message: /^internal error; reference = \S+$/, - }); - await env.SERVICE.waitForCancellation(); - - // Results without bodies do not need a length wrapper. - assert.strictEqual(await env.BUCKET.get('missing'), null); - assert.strictEqual((await env.BUCKET.get('metadata')).body, undefined); - - // R2 bodies must remain usable with an application's own streaming transforms. - const transformed = (await env.BUCKET.get('wrapped')).body.pipeThrough( - new FixedLengthStream(bodyLengthBytes.byteLength) - ); - assert.strictEqual(await new Response(transformed).text(), 'data'); - }, -}; - -export class R2JsrpcResponseEntrypoint extends WorkerEntrypoint { - head(key) { - if (key === 'rejected') throw new TypeError('gateway failure'); - assert(Object.hasOwn(invalidMetadata, key)); - return { ...object, ...invalidMetadata[key] }; - } - - get(key) { - switch (key) { - case 'unknownKind': - return { kind: 'other', object }; - case 'metadataWithBody': - return { kind: 'metadata', object, body: new ReadableStream() }; - case 'missingBody': - return { kind: 'body', object }; - case 'invalidBody': - return { kind: 'body', object, body: 42 }; - case 'unsafeLength': - return { - kind: 'body', - object: { ...object, size: 2 ** 53 }, - body: new ReadableStream(), - }; - default: - assert.fail('Invalid user input must not reach the gateway'); - } - } - - put() { - return { ...object, ...invalidMetadata.invalidDate }; - } - - list() { - return { - objects: [{ ...object, ...invalidMetadata.invalidDate }], - truncated: false, - delimitedPrefixes: [], - }; - } - - completeMultipartUpload() { - return { ...object, ...invalidMetadata.invalidDate }; - } -} - -export async function testInvalidR2Responses(_controller, env) { - const internalError = { message: /^internal error; reference = \S+$/ }; - for (const key of Object.keys(invalidMetadata)) { - await assert.rejects(env.BUCKET.head(key), internalError, key); - } - for (const key of [ - 'unknownKind', - 'metadataWithBody', - 'missingBody', - 'invalidBody', - 'unsafeLength', - ]) { - await assert.rejects(env.BUCKET.get(key), internalError, key); - } - await assert.rejects(env.BUCKET.put('key', 'body'), internalError); - await assert.rejects(env.BUCKET.list(), internalError); - const upload = env.BUCKET.resumeMultipartUpload('key', 'uploadId'); - await assert.rejects(upload.complete([]), internalError); - - // A rejected gateway call and invalid user input retain their public error messages. - await assert.rejects(env.BUCKET.head('rejected'), { - name: 'TypeError', - message: 'gateway failure', - }); - await assert.rejects( - env.BUCKET.get('invalidInput', { range: { offset: -1 } }), - { - name: 'RangeError', - message: - 'Invalid range. Starting offset (-1) must be greater than or equal to 0.', - } - ); -} - export default testWorker; From 7426e204d80262a73e4f82d33c0e89509bc9b32e Mon Sep 17 00:00:00 2001 From: Bradley Thwaites Date: Fri, 23 Oct 2026 13:29:29 -0400 Subject: [PATCH 10/13] Remove R2 client-side stream length validation and valueSize RPC arguments --- src/workerd/api/r2-bucket.c++ | 13 ++++----- src/workerd/api/r2-bucket.h | 2 +- src/workerd/api/r2-multipart.c++ | 6 ++-- src/workerd/api/r2-multipart.h | 2 +- src/workerd/api/r2-rpc.c++ | 27 ++++++------------ src/workerd/api/r2-rpc.h | 11 ++------ .../r2-binding-service/r2-binding-service.ts | 17 ++++------- .../r2-binding-service/r2-observed-binding.ts | 28 +++---------------- src/workerd/api/tests/r2-test.js | 10 ++----- 9 files changed, 34 insertions(+), 82 deletions(-) diff --git a/src/workerd/api/r2-bucket.c++ b/src/workerd/api/r2-bucket.c++ index c1b7a9ed375..31947d37c7a 100644 --- a/src/workerd/api/r2-bucket.c++ +++ b/src/workerd/api/r2-bucket.c++ @@ -687,8 +687,8 @@ jsg::Ref headResultFromBackend( if (policy == MissingMetadataPolicy::EMPTY) { bool hasHttpMetadata = result->getHttpMetadata() != kj::none; bool hasCustomMetadata = result->getCustomMetadata() != kj::none; - KJ_ASSERT(hasHttpMetadata, - "Malformed R2 RPC result: object metadata did not include HTTP metadata."); + KJ_ASSERT( + hasHttpMetadata, "Malformed R2 RPC result: object metadata did not include HTTP metadata."); KJ_ASSERT(hasCustomMetadata, "Malformed R2 RPC result: object metadata did not include custom metadata."); } @@ -841,7 +841,7 @@ jsg::Promise>> R2Bucket::putRpc(jsg::Lo jsg::Optional options, const jsg::TypeHandler>& rpcPropHandler, const jsg::TypeHandler, jsg::Optional, double)>>& putFnHandler, + kj::String, kj::Maybe, jsg::Optional)>>& putFnHandler, const jsg::TypeHandler>>& putResultHandler) { return js.evalNow([&] { auto cancelReader = kj::defer([&] { @@ -941,15 +941,12 @@ jsg::Promise>> R2Bucket::putRpc(jsg::Lo auto wrappedMethod = rpcPropHandler.wrap(js, kj::mv(rpcMethod)); auto putFn = KJ_ASSERT_NONNULL(putFnHandler.tryUnwrap(js, wrappedMethod)); kj::Maybe rpcValue; - double valueSize = 0; KJ_IF_SOME(v, value) { - auto prepared = prepareR2RpcBody(js, v); - rpcValue = kj::mv(prepared.value); - valueSize = prepared.size; + rpcValue = prepareR2RpcBody(js, v); } cancelReader.cancel(); - auto rpcPromise = putFn(js, kj::mv(key), kj::mv(rpcValue), kj::mv(rpcOptions), valueSize); + auto rpcPromise = putFn(js, kj::mv(key), kj::mv(rpcValue), kj::mv(rpcOptions)); auto promise = unwrapR2RpcPromise>(js, kj::mv(rpcPromise), putResultHandler); return promise.then(js, diff --git a/src/workerd/api/r2-bucket.h b/src/workerd/api/r2-bucket.h index d85338b0a70..b1b4c7bb834 100644 --- a/src/workerd/api/r2-bucket.h +++ b/src/workerd/api/r2-bucket.h @@ -645,7 +645,7 @@ class R2Bucket: public jsg::Object { jsg::Optional options, const jsg::TypeHandler>& rpcPropHandler, const jsg::TypeHandler, jsg::Optional, double)>>& putFnHandler, + kj::String, kj::Maybe, jsg::Optional)>>& putFnHandler, const jsg::TypeHandler>>& putResultHandler); jsg::Promise> createMultipartUploadRpc(jsg::Lock& js, kj::String key, diff --git a/src/workerd/api/r2-multipart.c++ b/src/workerd/api/r2-multipart.c++ index 5fe038a0952..563b3883258 100644 --- a/src/workerd/api/r2-multipart.c++ +++ b/src/workerd/api/r2-multipart.c++ @@ -142,7 +142,7 @@ jsg::Promise R2MultipartUpload::uploadPartRpc(j jsg::Optional options, const jsg::TypeHandler>& rpcPropHandler, const jsg::TypeHandler, double)>>& + kj::String, kj::String, int, R2PutValue, jsg::Optional)>>& uploadPartFnHandler, const jsg::TypeHandler>& uploadPartResultHandler) { return js.evalNow([&] { @@ -155,11 +155,11 @@ jsg::Promise R2MultipartUpload::uploadPartRpc(j } } - auto prepared = prepareR2RpcBody(js, value); + auto rpcValue = prepareR2RpcBody(js, value); auto promise = callR2RpcMethod(js, bucket->getRpcMethod(js, "uploadPart"_kj), rpcPropHandler, uploadPartFnHandler, uploadPartResultHandler, kj::str(key), - kj::str(uploadId), partNumber, kj::mv(prepared.value), kj::mv(options), prepared.size); + kj::str(uploadId), partNumber, kj::mv(rpcValue), kj::mv(options)); return promise.then(js, [partNumber](jsg::Lock& js, UploadedPart uploadedPart) { return UploadedPart{.partNumber = partNumber, .etag = kj::mv(uploadedPart.etag)}; }); diff --git a/src/workerd/api/r2-multipart.h b/src/workerd/api/r2-multipart.h index 0c05448240c..a20c580e482 100644 --- a/src/workerd/api/r2-multipart.h +++ b/src/workerd/api/r2-multipart.h @@ -71,7 +71,7 @@ class R2MultipartUpload: public jsg::Object { jsg::Optional options, const jsg::TypeHandler>& rpcPropHandler, const jsg::TypeHandler, double)>>& + kj::String, kj::String, int, R2PutValue, jsg::Optional)>>& uploadPartFnHandler, const jsg::TypeHandler>& uploadPartResultHandler); jsg::Promise abortRpc(jsg::Lock& js, diff --git a/src/workerd/api/r2-rpc.c++ b/src/workerd/api/r2-rpc.c++ index e390002743d..f387d98c950 100644 --- a/src/workerd/api/r2-rpc.c++ +++ b/src/workerd/api/r2-rpc.c++ @@ -53,18 +53,10 @@ void requireR2RpcDeserializer(jsg::Deserializer& deserializer) { "R2 resources can only be deserialized for RPC."); } -PreparedR2RpcBody prepareR2RpcBody(jsg::Lock& js, R2PutValue& value) { +R2PutValue prepareR2RpcBody(jsg::Lock& js, R2PutValue& value) { KJ_SWITCH_ONEOF(value) { KJ_CASE_ONEOF(stream, JsReadableStream) { - auto size = stream.tryGetLength(js, StreamEncoding::IDENTITY); - - JSG_REQUIRE(size != kj::none, TypeError, - "Provided readable stream must have a known length (request/response body or readable " - "half of FixedLengthStream)"); - auto exactSize = KJ_ASSERT_NONNULL(size); - JSG_REQUIRE(exactSize <= 9007199254740991ull, RangeError, - "Provided readable stream is too large to represent its length exactly"); - return {.value = kj::mv(stream), .size = static_cast(exactSize)}; + return R2PutValue(kj::mv(stream)); } KJ_CASE_ONEOF(data, kj::Array) { auto size = data.size(); @@ -72,28 +64,25 @@ PreparedR2RpcBody prepareR2RpcBody(jsg::Lock& js, R2PutValue& value) { // Type-wrapper arrays may alias V8 memory, which cannot be pumped without the isolate lock. auto owned = kj::heapArray(data.asPtr()); auto view = owned.asPtr(); - return {.value = makeR2RpcMemoryStream(js, view, kj::heap(kj::mv(owned))), - .size = static_cast(size)}; + return R2PutValue(makeR2RpcMemoryStream(js, view, kj::heap(kj::mv(owned)))); } - return {.value = kj::mv(data), .size = static_cast(size)}; + return R2PutValue(kj::mv(data)); } KJ_CASE_ONEOF(text, jsg::NonCoercible) { auto size = text.value.size(); if (size > R2_RPC_INLINE_STRING_LIMIT) { auto bytes = text.value.asBytes(); - return {.value = makeR2RpcMemoryStream(js, bytes, kj::heap(kj::mv(text.value))), - .size = static_cast(size)}; + return R2PutValue(makeR2RpcMemoryStream(js, bytes, kj::heap(kj::mv(text.value)))); } - return {.value = kj::mv(text), .size = static_cast(size)}; + return R2PutValue(kj::mv(text)); } KJ_CASE_ONEOF(blob, jsg::Ref) { auto size = blob->getSize(); if (size > R2_RPC_INLINE_BODY_LIMIT) { // Blob bytes are V8-backed, so newMemorySource() must copy them into native memory. - return { - .value = makeR2RpcMemoryStream(js, blob->getData(js)), .size = static_cast(size)}; + return R2PutValue(makeR2RpcMemoryStream(js, blob->getData(js))); } - return {.value = kj::mv(blob), .size = static_cast(size)}; + return R2PutValue(kj::mv(blob)); } } KJ_UNREACHABLE; diff --git a/src/workerd/api/r2-rpc.h b/src/workerd/api/r2-rpc.h index c510d2a1aeb..b9654c654bb 100644 --- a/src/workerd/api/r2-rpc.h +++ b/src/workerd/api/r2-rpc.h @@ -121,14 +121,9 @@ class R2Error: public jsg::Object { using R2PutValue = kj::OneOf, jsg::NonCoercible, jsg::Ref>; -struct PreparedR2RpcBody { - R2PutValue value; - double size; -}; - -// Prepares the transport value and the exact byte length passed alongside it. The length is an -// internal argument to the backing entrypoint's named RPC method, not part of the public R2 API. -PreparedR2RpcBody prepareR2RpcBody(jsg::Lock& js, R2PutValue& value); +// Converts bodies that are too large for an inline JSRPC message into streams. Known stream lengths +// are preserved by the JSRPC stream transport. +R2PutValue prepareR2RpcBody(jsg::Lock& js, R2PutValue& value); struct R2Result { uint httpStatus; diff --git a/src/workerd/api/tests/r2-binding-service/r2-binding-service.ts b/src/workerd/api/tests/r2-binding-service/r2-binding-service.ts index f2b0e92d365..b7d5b2ed603 100644 --- a/src/workerd/api/tests/r2-binding-service/r2-binding-service.ts +++ b/src/workerd/api/tests/r2-binding-service/r2-binding-service.ts @@ -12,9 +12,8 @@ // callers; this file describes what they send and what they accept back. // // Arguments are what the backend receives, which is not always what the caller passed to the -// public R2Bucket method: workerd validates and normalizes options first, and sends request -// bodies with their exact byte length as a separate argument. Absent option fields may arrive -// as explicit `undefined`. +// public R2Bucket method: workerd validates and normalizes options first. Absent option fields may +// arrive as explicit `undefined`. // // Results are the native R2 types, so a backend can return what a native R2 binding returns. // Two constraints the types cannot express: an R2ObjectBody is rejected wherever only object @@ -22,9 +21,8 @@ // completeMultipartUpload must include httpMetadata and customMetadata. // Request body passed to put and uploadPart. ArrayBufferViews arrive as ArrayBuffers, and -// binary, string, and Blob values above an inline size limit arrive as a ReadableStream. A -// stream's length may not be known on the receiving side; the accompanying valueSize argument -// gives its exact byte length. +// binary, string, and Blob values above an inline size limit arrive as a ReadableStream. Known +// stream lengths are preserved across JSRPC. export type R2BindingBody = ReadableStream | ArrayBuffer | string | Blob; // Range passed to get. Contains only the fields the caller set, so it may be empty. @@ -91,12 +89,10 @@ export type R2BindingService = { options: R2BindingGetOptions | undefined ): Promise; - // `value` is null for an empty put, in which case valueSize is 0. put( key: string, value: R2BindingBody | null, - options: R2BindingPutOptions | undefined, - valueSize: number + options: R2BindingPutOptions | undefined ): Promise; // The result is ignored. @@ -115,8 +111,7 @@ export type R2BindingService = { uploadId: string, partNumber: number, value: R2BindingBody, - options: R2BindingUploadPartOptions | undefined, - valueSize: number + options: R2BindingUploadPartOptions | undefined ): Promise; abortMultipartUpload(key: string, uploadId: string): Promise; diff --git a/src/workerd/api/tests/r2-binding-service/r2-observed-binding.ts b/src/workerd/api/tests/r2-binding-service/r2-observed-binding.ts index b6ea66d16af..8b786c542be 100644 --- a/src/workerd/api/tests/r2-binding-service/r2-observed-binding.ts +++ b/src/workerd/api/tests/r2-binding-service/r2-observed-binding.ts @@ -17,20 +17,6 @@ type Env = { REAL_BUCKET: R2Bucket; }; -// JSRPC bodies can arrive as a stream whose length is known only from the separate size argument. -// The native binding requires a known-length stream, so restore it from that argument. -function withKnownLength( - value: T, - valueSize: number -): T | ReadableStream { - if (!(value instanceof ReadableStream)) { - return value; - } - const { readable, writable } = new FixedLengthStream(valueSize); - value.pipeTo(writable).catch(() => {}); - return readable; -} - // Implements the R2 binding's JSRPC methods by forwarding each call to a native R2 binding, in // the way a customer Worker can stand in for R2. The JSRPC tests target this entrypoint so the // shared suite exercises the serialization of native R2 results and errors, while REAL_BUCKET @@ -59,14 +45,9 @@ export class ObservedR2Binding put( key: string, value: R2BindingBody | null, - options: R2BindingPutOptions | undefined, - valueSize: number + options: R2BindingPutOptions | undefined ): Promise { - return this.env.REAL_BUCKET.put( - key, - withKnownLength(value, valueSize), - options - ); + return this.env.REAL_BUCKET.put(key, value, options); } delete(keys: string | string[]): Promise { @@ -93,12 +74,11 @@ export class ObservedR2Binding uploadId: string, partNumber: number, value: R2BindingBody, - options: R2BindingUploadPartOptions | undefined, - valueSize: number + options: R2BindingUploadPartOptions | undefined ): Promise { return this.env.REAL_BUCKET.resumeMultipartUpload(key, uploadId).uploadPart( partNumber, - withKnownLength(value, valueSize), + value, options ); } diff --git a/src/workerd/api/tests/r2-test.js b/src/workerd/api/tests/r2-test.js index 5328317aec0..ee4a6c84b2c 100644 --- a/src/workerd/api/tests/r2-test.js +++ b/src/workerd/api/tests/r2-test.js @@ -1629,13 +1629,9 @@ export const r2BindingApiTests = { 'Provided readable stream must have a known length (request/response body or readable half of FixedLengthStream)', } ); - // The JSRPC client cancels a stream it rejects; the HTTP client leaves it to the caller. - if (env.R2_TRANSPORT === 'jsrpc') { - assert.strictEqual( - unknownLengthCancelReason.message, - 'Stream cancelled because the associated put operation encountered an error.' - ); - } else { + // The direct HTTP path leaves a validation-failed stream with its caller. Releasing an + // RPC-transferred stream is handled by the RPC transport. + if (env.R2_TRANSPORT !== 'jsrpc') { assert.strictEqual(unknownLengthCancelReason, undefined); } }, From f199cef59c70c14eae7c917d8d09fba3bdc99e0b Mon Sep 17 00:00:00 2001 From: Bradley Thwaites Date: Mon, 26 Oct 2026 10:49:54 -0400 Subject: [PATCH 11/13] remove unusued multipart upload metadata snapshot --- src/workerd/api/r2-bucket.c++ | 22 +++------------- src/workerd/api/r2-bucket.h | 1 - src/workerd/api/r2-multipart.c++ | 7 ------ src/workerd/api/r2-multipart.h | 25 ++----------------- src/workerd/api/tests/BUILD.bazel | 7 ++++++ src/workerd/api/tests/r2-http-test.wd-test | 23 +++++++++++++++++ .../api/tests/r2-instrumentation-test.js | 4 +-- .../api/tests/r2-instrumentation-worker.js | 11 ++++++++ src/workerd/api/tests/r2-test.js | 1 - src/workerd/api/tests/r2-test.wd-test | 3 ++- src/workerd/io/compatibility-date.capnp | 7 ++++++ 11 files changed, 57 insertions(+), 54 deletions(-) create mode 100644 src/workerd/api/tests/r2-http-test.wd-test create mode 100644 src/workerd/api/tests/r2-instrumentation-worker.js diff --git a/src/workerd/api/r2-bucket.c++ b/src/workerd/api/r2-bucket.c++ index 31947d37c7a..a79623c366b 100644 --- a/src/workerd/api/r2-bucket.c++ +++ b/src/workerd/api/r2-bucket.c++ @@ -967,7 +967,6 @@ jsg::Promise> R2Bucket::createMultipartUploadRpc(jsg createFnHandler, const jsg::TypeHandler>& createResultHandler) { return js.evalNow([&] { - R2MultipartUpload::Metadata metadata; KJ_IF_SOME(o, options) { KJ_IF_SOME(metadata, o.httpMetadata) { auto normalized = normalizeHttpMetadata(js, kj::mv(metadata)); @@ -978,32 +977,19 @@ jsg::Promise> R2Bucket::createMultipartUploadRpc(jsg } } - KJ_IF_SOME(o, options) { - KJ_IF_SOME(httpMetadata, o.httpMetadata) { - metadata.httpMetadata = httpMetadata.get().clone(); - } - KJ_IF_SOME(customMetadata, o.customMetadata) { - metadata.customMetadata.fields = KJ_MAP(field, customMetadata.fields) { - return jsg::Dict::Field{ - .name = kj::str(field.name), .value = kj::str(field.value)}; - }; - } - } - - // The backend replies with only `{ uploadId }`; the key, bucket, and metadata are already - // known here. Any other reply shape, including a bare string or additional fields, is - // rejected as malformed. + // The backend replies with only `{ uploadId }`; the key and bucket are already known here. Any + // other reply shape, including a bare string or additional fields, is rejected as malformed. auto uploadPromise = callR2RpcMethod(js, getRpcMethod(js, "createMultipartUpload"_kj), rpcPropHandler, createFnHandler, createResultHandler, kj::str(key), kj::mv(options)); return uploadPromise.then(js, - [bucket = JSG_THIS, key = kj::mv(key), metadata = kj::mv(metadata)]( + [bucket = JSG_THIS, key = kj::mv(key)]( jsg::Lock& js, CreateMultipartUploadBackendResult result) mutable { KJ_ASSERT(result.fields.size() == 1 && result.fields[0].name == "uploadId"_kj, "Malformed R2 createMultipartUpload RPC result: expected only an uploadId."); return js.alloc( - kj::mv(key), kj::mv(result.fields[0].value.value), kj::mv(bucket), kj::mv(metadata)); + kj::mv(key), kj::mv(result.fields[0].value.value), kj::mv(bucket)); }); }); } diff --git a/src/workerd/api/r2-bucket.h b/src/workerd/api/r2-bucket.h index b1b4c7bb834..7404ca215ec 100644 --- a/src/workerd/api/r2-bucket.h +++ b/src/workerd/api/r2-bucket.h @@ -9,7 +9,6 @@ #include #include #include -#include namespace workerd::api { class Headers; diff --git a/src/workerd/api/r2-multipart.c++ b/src/workerd/api/r2-multipart.c++ index 563b3883258..400c760ff6c 100644 --- a/src/workerd/api/r2-multipart.c++ +++ b/src/workerd/api/r2-multipart.c++ @@ -129,13 +129,6 @@ jsg::Promise R2MultipartUpload::uploadPart(jsg: }); } -R2MultipartUpload::Metadata R2MultipartUpload::Metadata::clone() const { - return {.httpMetadata = httpMetadata.clone(), - .customMetadata = {.fields = KJ_MAP(field, customMetadata.fields) { - return jsg::Dict::Field{.name = kj::str(field.name), .value = kj::str(field.value)}; - }}}; -} - jsg::Promise R2MultipartUpload::uploadPartRpc(jsg::Lock& js, int partNumber, R2PutValue value, diff --git a/src/workerd/api/r2-multipart.h b/src/workerd/api/r2-multipart.h index a20c580e482..6685d753476 100644 --- a/src/workerd/api/r2-multipart.h +++ b/src/workerd/api/r2-multipart.h @@ -26,28 +26,10 @@ class R2MultipartUpload: public jsg::Object { JSG_STRUCT_TS_OVERRIDE(R2UploadPartOptions); }; - // The gateway's completion response omits create-time metadata. Created RPC uploads retain - // these values locally; resumed uploads use only the metadata returned by the gateway. - struct Metadata { - R2Bucket::HttpMetadata httpMetadata; - jsg::Dict customMetadata; - - Metadata clone() const; - - JSG_MEMORY_INFO(Metadata) { - tracker.trackField("httpMetadata", httpMetadata); - tracker.trackField("customMetadata", customMetadata); - } - }; - - R2MultipartUpload(kj::String key, - kj::String uploadId, - jsg::Ref bucket, - kj::Maybe metadata = kj::none) + R2MultipartUpload(kj::String key, kj::String uploadId, jsg::Ref bucket) : key(kj::mv(key)), uploadId(kj::mv(uploadId)), - bucket(kj::mv(bucket)), - metadata(kj::mv(metadata)) {} + bucket(kj::mv(bucket)) {} kj::StringPtr getKey() const { return key; @@ -103,7 +85,6 @@ class R2MultipartUpload: public jsg::Object { tracker.trackField("key", key); tracker.trackField("uploadId", uploadId); tracker.trackField("bucket", bucket); - tracker.trackField("metadata", metadata); } protected: @@ -112,8 +93,6 @@ class R2MultipartUpload: public jsg::Object { jsg::Ref bucket; private: - kj::Maybe metadata; - void visitForGc(jsg::GcVisitor& visitor) { visitor.visit(bucket); } diff --git a/src/workerd/api/tests/BUILD.bazel b/src/workerd/api/tests/BUILD.bazel index 5d8507cf2f7..128199da556 100644 --- a/src/workerd/api/tests/BUILD.bazel +++ b/src/workerd/api/tests/BUILD.bazel @@ -575,6 +575,12 @@ wd_test( ], ) +wd_test( + src = "r2-http-test.wd-test", + args = ["--experimental"], + data = ["r2-test.js"], +) + wd_test( src = "r2-jsrpc-serialization-test.wd-test", args = ["--experimental"], @@ -635,6 +641,7 @@ wd_test( data = [ "instrumentation-tail-worker.js", "r2-instrumentation-test.js", + "r2-instrumentation-worker.js", "r2-test.js", ], ) diff --git a/src/workerd/api/tests/r2-http-test.wd-test b/src/workerd/api/tests/r2-http-test.wd-test new file mode 100644 index 00000000000..1f75c76acaa --- /dev/null +++ b/src/workerd/api/tests/r2-http-test.wd-test @@ -0,0 +1,23 @@ +using Workerd = import "/workerd/workerd.capnp"; + +# Runs the complete R2 binding suite using the HTTP transport. The instrumented R2 test uses a +# restricted set of exports so that every emitted span can be checked exhaustively. +const unitTests :Workerd.Config = ( + services = [ + ( name = "r2-test", + worker = ( + modules = [ + ( name = "worker", esModule = embed "r2-test.js" ) + ], + bindings = [ + ( name = "BUCKET", r2Bucket = "r2-test" ), + ( name = "SERVICE", service = "r2-test" ), + ( name = "R2_TRANSPORT", text = "http" ), + ( name = "R2_LIST_HONOR_INCLUDE", text = "true" ), + ], + compatibilityFlags = ["nodejs_compat", "service_binding_extra_handlers", + "streams_enable_constructors", "r2_list_honor_include"], + ) + ), + ], +); diff --git a/src/workerd/api/tests/r2-instrumentation-test.js b/src/workerd/api/tests/r2-instrumentation-test.js index 09fea51f6d6..224dad911b7 100644 --- a/src/workerd/api/tests/r2-instrumentation-test.js +++ b/src/workerd/api/tests/r2-instrumentation-test.js @@ -1194,8 +1194,6 @@ export const test = { }, ]; - // The expected spans come from the default test and deletePerKeyErrorParityTests, which run - // before the other test exports. Spans from those later exports are not checked. - assert.deepStrictEqual(received.slice(0, expected.length), expected); + assert.deepStrictEqual(received, expected); }, }; diff --git a/src/workerd/api/tests/r2-instrumentation-worker.js b/src/workerd/api/tests/r2-instrumentation-worker.js new file mode 100644 index 00000000000..65ac1ecc7e6 --- /dev/null +++ b/src/workerd/api/tests/r2-instrumentation-worker.js @@ -0,0 +1,11 @@ +// Copyright (c) 2017-2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +// Only export the suites whose spans are checked by r2-instrumentation-test.js. The other shared +// R2 suites are exercised by the HTTP and JSRPC transport tests without instrumentation. +export { + default, + deletePerKeyErrorParityTests, + r2ValidationTests, +} from './r2-test.js'; diff --git a/src/workerd/api/tests/r2-test.js b/src/workerd/api/tests/r2-test.js index ee4a6c84b2c..225a42ea5ea 100644 --- a/src/workerd/api/tests/r2-test.js +++ b/src/workerd/api/tests/r2-test.js @@ -4,7 +4,6 @@ import assert from 'node:assert'; import { Buffer } from 'node:buffer'; -import { WorkerEntrypoint } from 'cloudflare:workers'; const key = 'basicKey'; const body = 'content'; diff --git a/src/workerd/api/tests/r2-test.wd-test b/src/workerd/api/tests/r2-test.wd-test index f44abdb5d7e..b0719d7fce9 100644 --- a/src/workerd/api/tests/r2-test.wd-test +++ b/src/workerd/api/tests/r2-test.wd-test @@ -5,7 +5,8 @@ const unitTests :Workerd.Config = ( ( name = "r2-test", worker = ( modules = [ - ( name = "worker", esModule = embed "r2-test.js" ) + ( name = "worker", esModule = embed "r2-instrumentation-worker.js" ), + ( name = "r2-test.js", esModule = embed "r2-test.js" ), ], bindings = [ ( name = "BUCKET", r2Bucket = "r2-test" ), diff --git a/src/workerd/io/compatibility-date.capnp b/src/workerd/io/compatibility-date.capnp index ede43238976..01078afa5ec 100644 --- a/src/workerd/io/compatibility-date.capnp +++ b/src/workerd/io/compatibility-date.capnp @@ -1768,4 +1768,11 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef { # When enabled, R2 bindings dispatch to the R2 gateway worker over JSRPC # instead of synthesising an HTTP request and calling `fetch`. Without this # flag, R2 bindings continue to use the HTTP transport. + # + # The binding must specify a named entrypoint implementing the R2 backend + # methods: head, get, put, delete, list, createMultipartUpload, uploadPart, + # abortMultipartUpload, and completeMultipartUpload. Object results use the + # corresponding native R2 API types; createMultipartUpload returns an object + # containing uploadId. The expected TypeScript interface is type-checked in + # src/workerd/api/tests/r2-binding-service/r2-binding-service.ts. } From dee191e5338ae182cea6ee8233036b24df627a87 Mon Sep 17 00:00:00 2001 From: Bradley Thwaites Date: Mon, 26 Oct 2026 12:31:48 -0400 Subject: [PATCH 12/13] Remove R2 RPC input validation - Preserve list.include compatibility - Replace multipart result dict with JSG struct - TS updates - Split tests --- .bazelrc | 2 +- src/workerd/api/r2-bucket.c++ | 280 ++---------------- src/workerd/api/r2-bucket.h | 32 +- src/workerd/api/r2-multipart.c++ | 15 - src/workerd/api/r2.h | 2 +- src/workerd/api/tests/BUILD.bazel | 28 +- .../r2-binding-service/r2-binding-service.ts | 72 +---- .../r2-binding-service/r2-observed-binding.ts | 23 +- src/workerd/api/tests/r2-jsrpc-input-test.js | 94 ++++++ .../api/tests/r2-jsrpc-input-test.wd-test | 41 +++ .../api/tests/r2-jsrpc-response-test.js | 84 ++++++ .../api/tests/r2-jsrpc-response-test.wd-test | 63 ++++ .../api/tests/r2-jsrpc-serialization-test.js | 209 ------------- .../tests/r2-resource-serialization-test.js | 74 +++++ ...=> r2-resource-serialization-test.wd-test} | 25 +- ...r2-resource-serialization-ts-test.wd-test} | 6 +- .../api/tests/r2-string-upload-test.js | 35 +++ .../api/tests/r2-string-upload-test.wd-test | 4 +- src/workerd/api/tests/r2-test.js | 90 +++--- 19 files changed, 526 insertions(+), 653 deletions(-) create mode 100644 src/workerd/api/tests/r2-jsrpc-input-test.js create mode 100644 src/workerd/api/tests/r2-jsrpc-input-test.wd-test create mode 100644 src/workerd/api/tests/r2-jsrpc-response-test.js create mode 100644 src/workerd/api/tests/r2-jsrpc-response-test.wd-test delete mode 100644 src/workerd/api/tests/r2-jsrpc-serialization-test.js create mode 100644 src/workerd/api/tests/r2-resource-serialization-test.js rename src/workerd/api/tests/{r2-jsrpc-serialization-test.wd-test => r2-resource-serialization-test.wd-test} (51%) rename src/workerd/api/tests/{r2-jsrpc-serialization-ts-test.wd-test => r2-resource-serialization-ts-test.wd-test} (78%) create mode 100644 src/workerd/api/tests/r2-string-upload-test.js diff --git a/.bazelrc b/.bazelrc index feb53f567b1..4ac48ed83a3 100644 --- a/.bazelrc +++ b/.bazelrc @@ -222,7 +222,7 @@ build --@capnp-cpp//src/capnp:gen_rust=True build --@capnp-cpp//src/kj:kj_enable_irequire=True # overriden in config=opt build --@capnp-cpp//src/kj:debug_memory=True -build --cxxopt="-fbracket-depth=512" --host_cxxopt="-fbracket-depth=512" +build --cxxopt="-fbracket-depth=1024" --host_cxxopt="-fbracket-depth=1024" # Additional Rust flags (see https://doc.rust-lang.org/rustc/codegen-options/index.html) # Need to disable debug-assertions for fastbuild, should be off automatically for opt. diff --git a/src/workerd/api/r2-bucket.c++ b/src/workerd/api/r2-bucket.c++ index a79623c366b..38726117a56 100644 --- a/src/workerd/api/r2-bucket.c++ +++ b/src/workerd/api/r2-bucket.c++ @@ -399,27 +399,6 @@ kj::Maybe buildSsecKey( return kj::none; } -static R2Bucket::HttpMetadata normalizeHttpMetadata( - jsg::Lock& js, kj::OneOf> metadata) { - KJ_SWITCH_ONEOF(metadata) { - KJ_CASE_ONEOF(value, R2Bucket::HttpMetadata) { - return kj::mv(value); - } - KJ_CASE_ONEOF(headers, jsg::Ref) { - return R2Bucket::HttpMetadata::fromRequestHeaders(js, *headers); - } - } - KJ_UNREACHABLE; -} - -/***************************** - * - * Helper functions to set various tags on traceContext. - * - addR2ResponseSpanTags - * - addHeadResultSpanTags - * - addListResultSpanTags - * - ******************************/ static void addR2ResponseSpanTags(TraceContext& traceContext, R2Result& r2Result) { traceContext.setTag("cloudflare.r2.response.success"_kjc, r2Result.success()); KJ_IF_SOME(e, r2Result.getR2ErrorMessage()) { @@ -721,88 +700,12 @@ R2Bucket::getRpc(jsg::Lock& js, kj::String key, jsg::Optional options, const jsg::TypeHandler>& rpcPropHandler, - const jsg::TypeHandler< - jsg::Function)>>& getFnHandler, + const jsg::TypeHandler)>>& + getFnHandler, const jsg::TypeHandler>>& getResultHandler) { return js.evalNow([&] { - jsg::Optional rpcOptions; - KJ_IF_SOME(o, options) { - SerializableGetOptions normalized; - - KJ_IF_SOME(condition, o.onlyIf) { - // Parsed only for validation, so callers get the same errors as the non-RPC path (e.g. - // quoted or malformed ETags, invalid dates). The parsed result is intentionally unused: - // the original onlyIf value is forwarded below and the backing entrypoint does its own - // parsing. - UnwrappedConditional conditional = [&] { - KJ_SWITCH_ONEOF(condition) { - KJ_CASE_ONEOF(value, Conditional) { - return UnwrappedConditional(value); - } - KJ_CASE_ONEOF(headers, jsg::Ref) { - return UnwrappedConditional(js, *headers); - } - } - KJ_UNREACHABLE; - }(); - - KJ_SWITCH_ONEOF(condition) { - KJ_CASE_ONEOF(conditional, Conditional) { - normalized.onlyIf = kj::mv(conditional); - } - KJ_CASE_ONEOF(headers, jsg::Ref) { - normalized.onlyIf = kj::mv(headers); - } - } - } - - KJ_IF_SOME(range, o.range) { - KJ_SWITCH_ONEOF(range) { - KJ_CASE_ONEOF(value, Range) { - kj::Vector::Field> fields; - KJ_IF_SOME(offset, value.offset) { - JSG_REQUIRE(offset >= 0, RangeError, "Invalid range. Starting offset (", offset, - ") must be greater than or equal to 0."); - JSG_REQUIRE(isWholeNumber(offset), RangeError, "Invalid range. Starting offset (", - offset, ") must be an integer, not floating point."); - fields.add(jsg::Dict::Field{.name = kj::str("offset"), .value = offset}); - } - KJ_IF_SOME(length, value.length) { - JSG_REQUIRE(length >= 0, RangeError, "Invalid range. Length (", length, - ") must be greater than or equal to 0."); - JSG_REQUIRE(isWholeNumber(length), RangeError, "Invalid range. Length (", length, - ") must be an integer, not floating point."); - fields.add(jsg::Dict::Field{.name = kj::str("length"), .value = length}); - } - KJ_IF_SOME(suffix, value.suffix) { - JSG_REQUIRE( - value.offset == kj::none, TypeError, "Suffix is incompatible with offset."); - JSG_REQUIRE( - value.length == kj::none, TypeError, "Suffix is incompatible with length."); - JSG_REQUIRE(suffix >= 0, RangeError, "Invalid suffix. Suffix (", suffix, - ") must be greater than or equal to 0."); - JSG_REQUIRE(isWholeNumber(suffix), RangeError, "Invalid range. Suffix (", suffix, - ") must be an integer, not floating point."); - fields.add(jsg::Dict::Field{.name = kj::str("suffix"), .value = suffix}); - } - normalized.range = jsg::Dict{.fields = fields.releaseAsArray()}; - } - KJ_CASE_ONEOF(headers, jsg::Ref) { - if (headers->getCommon(js, capnp::CommonHeaderName::RANGE) != kj::none) { - normalized.range = kj::mv(headers); - } - } - } - } - - KJ_IF_SOME(ssecKey, buildSsecKey(kj::mv(o.ssecKey))) { - normalized.ssecKey = kj::mv(ssecKey); - } - rpcOptions = kj::mv(normalized); - } - return callR2RpcMethod>(js, getRpcMethod(js, "get"_kj), - rpcPropHandler, getFnHandler, getResultHandler, kj::mv(key), kj::mv(rpcOptions)) + rpcPropHandler, getFnHandler, getResultHandler, kj::mv(key), kj::mv(options)) .then(js, [](jsg::Lock& js, kj::Maybe parsed) -> kj::OneOf>, jsg::Ref> { @@ -844,113 +747,16 @@ jsg::Promise>> R2Bucket::putRpc(jsg::Lo kj::String, kj::Maybe, jsg::Optional)>>& putFnHandler, const jsg::TypeHandler>>& putResultHandler) { return js.evalNow([&] { - auto cancelReader = kj::defer([&] { - KJ_IF_SOME(v, value) { - KJ_SWITCH_ONEOF(v) { - KJ_CASE_ONEOF(stream, JsReadableStream) { - stream.cancel(js, - js.error( - "Stream cancelled because the associated put operation encountered an error.")); - } - KJ_CASE_ONEOF_DEFAULT {} - } - } - }); - - // Checksums are validated here so callers get the same errors as the non-RPC path, then - // forwarded unchanged; the backing entrypoint decodes hex strings and buffers itself. - bool hashAlreadySpecified = false; - const auto validateChecksum = [&](auto checksum, kj::StringPtr algorithm, size_t byteLength, - size_t hexLength) -> decltype(checksum) { - KJ_IF_SOME(c, checksum) { - JSG_REQUIRE( - !hashAlreadySpecified, TypeError, "You cannot specify multiple hashing algorithms."); - hashAlreadySpecified = true; - KJ_SWITCH_ONEOF(c) { - KJ_CASE_ONEOF(buffer, jsg::JsRef) { - auto bytes = buffer.getHandle(js); - JSG_REQUIRE(bytes.size() == byteLength, TypeError, algorithm, " is ", byteLength, - " bytes, not ", bytes.size()); - } - KJ_CASE_ONEOF(text, jsg::NonCoercible) { - JSG_REQUIRE(text.value.size() == hexLength, TypeError, algorithm, " is ", hexLength, - " hex characters, not ", text.value.size()); - auto decoded = kj::decodeHex(text.value); - JSG_REQUIRE(!decoded.hadErrors, TypeError, "Provided ", algorithm, - " wasn't a valid hex string"); - } - } - } - return kj::mv(checksum); - }; - - jsg::Optional rpcOptions; - KJ_IF_SOME(o, options) { - jsg::Optional>> onlyIf; - KJ_IF_SOME(condition, o.onlyIf) { - // Parsed only for validation, so callers get the same errors as the non-RPC path (e.g. - // quoted or malformed ETags, invalid dates). The parsed result is intentionally unused: - // the original onlyIf value is forwarded below and the backing entrypoint does its own - // parsing. - UnwrappedConditional conditional = [&] { - KJ_SWITCH_ONEOF(condition) { - KJ_CASE_ONEOF(value, Conditional) { - return UnwrappedConditional(value); - } - KJ_CASE_ONEOF(headers, jsg::Ref) { - return UnwrappedConditional(js, *headers); - } - } - KJ_UNREACHABLE; - }(); - - KJ_SWITCH_ONEOF(condition) { - KJ_CASE_ONEOF(conditional, Conditional) { - onlyIf = kj::mv(conditional); - } - KJ_CASE_ONEOF(headers, jsg::Ref) { - onlyIf = kj::mv(headers); - } - } - } - - jsg::Optional>> httpMetadata; - KJ_IF_SOME(metadata, o.httpMetadata) { - httpMetadata = normalizeHttpMetadata(js, kj::mv(metadata)); - } - jsg::Optional, kj::String>> ssecKey; - KJ_IF_SOME(key, buildSsecKey(kj::mv(o.ssecKey))) { - ssecKey = kj::mv(key); - } - - rpcOptions = PutOptions{ - .onlyIf = kj::mv(onlyIf), - .httpMetadata = kj::mv(httpMetadata), - .customMetadata = kj::mv(o.customMetadata), - .md5 = validateChecksum(kj::mv(o.md5), "MD5"_kj, 16, 32), - .sha1 = validateChecksum(kj::mv(o.sha1), "SHA-1"_kj, 20, 40), - .sha256 = validateChecksum(kj::mv(o.sha256), "SHA-256"_kj, 32, 64), - .sha384 = validateChecksum(kj::mv(o.sha384), "SHA-384"_kj, 48, 96), - .sha512 = validateChecksum(kj::mv(o.sha512), "SHA-512"_kj, 64, 128), - .storageClass = kj::mv(o.storageClass), - .ssecKey = kj::mv(ssecKey), - }; - } - - auto rpcMethod = getRpcMethod(js, "put"_kj); - auto wrappedMethod = rpcPropHandler.wrap(js, kj::mv(rpcMethod)); - auto putFn = KJ_ASSERT_NONNULL(putFnHandler.tryUnwrap(js, wrappedMethod)); kj::Maybe rpcValue; KJ_IF_SOME(v, value) { rpcValue = prepareR2RpcBody(js, v); } - cancelReader.cancel(); - - auto rpcPromise = putFn(js, kj::mv(key), kj::mv(rpcValue), kj::mv(rpcOptions)); - auto promise = - unwrapR2RpcPromise>(js, kj::mv(rpcPromise), putResultHandler); - return promise.then(js, - [](jsg::Lock& js, kj::Maybe parsed) -> kj::Maybe> { + return callR2RpcMethod>(js, getRpcMethod(js, "put"_kj), + rpcPropHandler, putFnHandler, putResultHandler, kj::mv(key), kj::mv(rpcValue), + kj::mv(options)) + .then(js, + [](jsg::Lock& js, + kj::Maybe parsed) -> kj::Maybe> { KJ_IF_SOME(backend, parsed) { return headResultFromBackend(js, kj::mv(backend)); } @@ -967,18 +773,7 @@ jsg::Promise> R2Bucket::createMultipartUploadRpc(jsg createFnHandler, const jsg::TypeHandler>& createResultHandler) { return js.evalNow([&] { - KJ_IF_SOME(o, options) { - KJ_IF_SOME(metadata, o.httpMetadata) { - auto normalized = normalizeHttpMetadata(js, kj::mv(metadata)); - o.httpMetadata = kj::mv(normalized); - } - KJ_IF_SOME(ssecKey, buildSsecKey(kj::mv(o.ssecKey))) { - o.ssecKey = kj::mv(ssecKey); - } - } - - // The backend replies with only `{ uploadId }`; the key and bucket are already known here. Any - // other reply shape, including a bare string or additional fields, is rejected as malformed. + // The key and bucket are already known here, so the backend replies with `{ uploadId }`. auto uploadPromise = callR2RpcMethod(js, getRpcMethod(js, "createMultipartUpload"_kj), rpcPropHandler, createFnHandler, createResultHandler, kj::str(key), kj::mv(options)); @@ -986,10 +781,8 @@ jsg::Promise> R2Bucket::createMultipartUploadRpc(jsg return uploadPromise.then(js, [bucket = JSG_THIS, key = kj::mv(key)]( jsg::Lock& js, CreateMultipartUploadBackendResult result) mutable { - KJ_ASSERT(result.fields.size() == 1 && result.fields[0].name == "uploadId"_kj, - "Malformed R2 createMultipartUpload RPC result: expected only an uploadId."); return js.alloc( - kj::mv(key), kj::mv(result.fields[0].value.value), kj::mv(bucket)); + kj::mv(key), kj::mv(result.uploadId.value), kj::mv(bucket)); }); }); } @@ -1540,49 +1333,34 @@ jsg::Promise R2Bucket::listRpc(jsg::Lock& js, const jsg::TypeHandler>& listResultHandler, CompatibilityFlags::Reader flags) { return js.evalNow([&] { - const bool honorsIncludes = flags.getR2ListHonorIncludeFields(); - jsg::Optional rpcOptions; - if (options != kj::none || !honorsIncludes) { - ListOptions normalized; - kj::Vector> includes; - + if (!flags.getR2ListHonorIncludeFields()) { + bool hasOnlySupportedIncludes = true; KJ_IF_SOME(o, options) { - KJ_IF_SOME(limit, o.limit) { - normalized.limit = limit; - } - KJ_IF_SOME(prefix, o.prefix) { - normalized.prefix = kj::mv(prefix); - } - KJ_IF_SOME(cursor, o.cursor) { - normalized.cursor = kj::mv(cursor); - } - KJ_IF_SOME(delimiter, o.delimiter) { - normalized.delimiter = kj::mv(delimiter); - } - KJ_IF_SOME(startAfter, o.startAfter) { - normalized.startAfter = kj::mv(startAfter); - } - KJ_IF_SOME(requestedIncludes, o.include) { - for (auto& requested: requestedIncludes) { - JSG_REQUIRE(requested.value == "httpMetadata" || requested.value == "customMetadata", - RangeError, "Unsupported include value ", requested.value); - if (honorsIncludes) { - includes.add(kj::mv(requested.value)); + KJ_IF_SOME(includes, o.include) { + for (const auto& include: includes) { + if (include.value != "httpMetadata" && include.value != "customMetadata") { + hasOnlySupportedIncludes = false; + break; } } } } - if (!honorsIncludes) { - includes.add(kj::str("httpMetadata")); - includes.add(kj::str("customMetadata")); + if (hasOnlySupportedIncludes) { + if (options == kj::none) { + options = ListOptions{}; + } + KJ_IF_SOME(o, options) { + kj::Vector> includes; + includes.add(jsg::NonCoercible{kj::str("httpMetadata")}); + includes.add(jsg::NonCoercible{kj::str("customMetadata")}); + o.include = includes.releaseAsArray(); + } } - normalized.include = includes.releaseAsArray(); - rpcOptions = kj::mv(normalized); } return callR2RpcMethod(js, getRpcMethod(js, "list"_kj), rpcPropHandler, - listFnHandler, listResultHandler, kj::mv(rpcOptions)) + listFnHandler, listResultHandler, kj::mv(options)) .then(js, [](jsg::Lock& js, ListResult result) -> ListResult { result.objects = KJ_MAP(object, result.objects) { return headResultFromBackend(js, kj::mv(object), MissingMetadataPolicy::ABSENT); diff --git a/src/workerd/api/r2-bucket.h b/src/workerd/api/r2-bucket.h index 7404ca215ec..d940dc6547f 100644 --- a/src/workerd/api/r2-bucket.h +++ b/src/workerd/api/r2-bucket.h @@ -259,25 +259,13 @@ class R2Bucket: public jsg::Object { JSG_STRUCT_TS_OVERRIDE(R2MultipartOptions); }; - // Internal JSRPC payload types. SerializableGetOptions is the normalized request shape used - // because the public Range type cannot represent the exact object sent to the backing - // entrypoint (see below). SerializableChecksums, SerializableHeadResult, and + // Internal JSRPC payload types. SerializableChecksums, SerializableHeadResult, and // SerializableGetResult are the payloads used by the custom serializers for the corresponding // native R2 resource types. RPC reconstructs those native resources before a named R2 method's // result is unwrapped. // // These are not customer-facing types, so they carry no TS overrides. - struct SerializableGetOptions { - jsg::Optional>> onlyIf; - // Uses Dict rather than Range because the fast_jsg_struct compat flag sets absent Range fields - // to undefined. - jsg::Optional, jsg::Ref>> range; - jsg::Optional ssecKey; - - JSG_STRUCT(onlyIf, range, ssecKey); - }; - // Serialized Checksums payload. struct SerializableChecksums { jsg::Optional> md5; @@ -565,12 +553,14 @@ class R2Bucket: public jsg::Object { )); }; - // The backing entrypoint's createMultipartUpload() reply, which must be `{ uploadId }`. A named - // field rather than a bare string, so an implementation cannot return some other string (such - // as the key) by mistake. NonCoercible so that a non-string uploadId is rejected rather than - // stringified. A Dict rather than a JSG_STRUCT because structs must be registered in the - // isolate's type list, which is at the compiler's fold-expression nesting limit. - using CreateMultipartUploadBackendResult = jsg::Dict>; + // The backing entrypoint's createMultipartUpload() reply. A named field rather than a bare + // string makes it harder for an implementation to return some other string, such as the key, + // by mistake. NonCoercible rejects a non-string uploadId rather than stringifying it. + struct CreateMultipartUploadBackendResult { + jsg::NonCoercible uploadId; + + JSG_STRUCT(uploadId); + }; struct ListOptions { jsg::Optional limit; @@ -629,8 +619,8 @@ class R2Bucket: public jsg::Object { kj::String key, jsg::Optional options, const jsg::TypeHandler>& rpcPropHandler, - const jsg::TypeHandler)>>& getFnHandler, + const jsg::TypeHandler)>>& + getFnHandler, const jsg::TypeHandler>>& getResultHandler); jsg::Promise deleteRpc(jsg::Lock& js, kj::OneOf> keys, diff --git a/src/workerd/api/r2-multipart.c++ b/src/workerd/api/r2-multipart.c++ index 400c760ff6c..7c7e4492735 100644 --- a/src/workerd/api/r2-multipart.c++ +++ b/src/workerd/api/r2-multipart.c++ @@ -139,15 +139,6 @@ jsg::Promise R2MultipartUpload::uploadPartRpc(j uploadPartFnHandler, const jsg::TypeHandler>& uploadPartResultHandler) { return js.evalNow([&] { - JSG_REQUIRE(partNumber >= 1 && partNumber <= 10000, TypeError, - "Part number must be between 1 and 10000 (inclusive). Actual value was: ", partNumber); - - KJ_IF_SOME(o, options) { - KJ_IF_SOME(ssecKey, buildSsecKey(kj::mv(o.ssecKey))) { - o.ssecKey = kj::mv(ssecKey); - } - } - auto rpcValue = prepareR2RpcBody(js, value); auto promise = callR2RpcMethod(js, bucket->getRpcMethod(js, "uploadPart"_kj), @@ -227,12 +218,6 @@ jsg::Promise> R2MultipartUpload::completeRpc(jsg: kj::String, kj::String, kj::Array)>>& completeFnHandler, const jsg::TypeHandler>& completeResultHandler) { return js.evalNow([&] { - for (auto& part: uploadedParts) { - JSG_REQUIRE(part.partNumber >= 1 && part.partNumber <= 10000, TypeError, - "Part number must be between 1 and 10000 (inclusive). Actual value was: ", - part.partNumber); - } - auto promise = callR2RpcMethod(js, bucket->getRpcMethod(js, "completeMultipartUpload"_kj), rpcPropHandler, completeFnHandler, completeResultHandler, kj::str(key), kj::str(uploadId), kj::mv(uploadedParts)); diff --git a/src/workerd/api/r2.h b/src/workerd/api/r2.h index daa976bb626..c03acfba6df 100644 --- a/src/workerd/api/r2.h +++ b/src/workerd/api/r2.h @@ -17,10 +17,10 @@ namespace workerd::api::public_beta { api::public_beta::R2Bucket::Checksums, api::public_beta::R2Bucket::StringChecksums, \ api::public_beta::R2Bucket::HttpMetadata, api::public_beta::R2Bucket::ListOptions, \ api::public_beta::R2Bucket::ListResult, \ + api::public_beta::R2Bucket::CreateMultipartUploadBackendResult, \ api::public_beta::R2MultipartUpload::UploadPartOptions, \ api::public_beta::R2Bucket::SerializableChecksums, \ api::public_beta::R2Bucket::SerializableHeadResult, \ - api::public_beta::R2Bucket::SerializableGetOptions, \ api::public_beta::R2Bucket::SerializableGetResult // The list of r2 types that are added to worker.c++'s JSG_DECLARE_ISOLATE_TYPE } // namespace workerd::api::public_beta diff --git a/src/workerd/api/tests/BUILD.bazel b/src/workerd/api/tests/BUILD.bazel index 128199da556..9fc3cf4c988 100644 --- a/src/workerd/api/tests/BUILD.bazel +++ b/src/workerd/api/tests/BUILD.bazel @@ -582,21 +582,38 @@ wd_test( ) wd_test( - src = "r2-jsrpc-serialization-test.wd-test", + src = "r2-jsrpc-input-test.wd-test", + args = ["--experimental"], + data = ["r2-jsrpc-input-test.js"], + generate_all_compat_flags_variant = False, +) + +wd_test( + src = "r2-jsrpc-response-test.wd-test", + args = ["--experimental"], + data = [ + "r2-jsrpc-response-test.js", + "r2-test.js", + ":r2_binding_service_ts_project", + ], +) + +wd_test( + src = "r2-resource-serialization-test.wd-test", args = ["--experimental"], data = [ - "r2-jsrpc-serialization-test.js", + "r2-resource-serialization-test.js", "r2-test.js", ":r2_binding_service_ts_project", ], ) wd_test( - src = "r2-jsrpc-serialization-ts-test.wd-test", + src = "r2-resource-serialization-ts-test.wd-test", args = ["--experimental"], data = [ - "r2-jsrpc-serialization-test.js", - "r2-jsrpc-serialization-test.wd-test", + "r2-resource-serialization-test.js", + "r2-resource-serialization-test.wd-test", "r2-test.js", ":r2_binding_service_ts_project", ], @@ -630,6 +647,7 @@ wd_test( args = ["--experimental"], data = [ "r2-jsrpc-passthrough.capnp", + "r2-string-upload-test.js", "r2-test.js", ":r2_binding_service_ts_project", ], diff --git a/src/workerd/api/tests/r2-binding-service/r2-binding-service.ts b/src/workerd/api/tests/r2-binding-service/r2-binding-service.ts index b7d5b2ed603..3ace82aa18a 100644 --- a/src/workerd/api/tests/r2-binding-service/r2-binding-service.ts +++ b/src/workerd/api/tests/r2-binding-service/r2-binding-service.ts @@ -11,9 +11,10 @@ // R2Bucket and R2MultipartUpload in src/workerd/api/r2-bucket.c++ and r2-multipart.c++ are the // callers; this file describes what they send and what they accept back. // -// Arguments are what the backend receives, which is not always what the caller passed to the -// public R2Bucket method: workerd validates and normalizes options first. Absent option fields may -// arrive as explicit `undefined`. +// The backend receives the public R2 option types without client-side validation or normalization. +// It is responsible for validating them. The only compatibility adjustment is to list(): callers +// without r2_list_honor_include request both metadata fields when their include list is absent or +// contains only recognized values. // // Results are the native R2 types, so a backend can return what a native R2 binding returns. // Two constraints the types cannot express: an R2ObjectBody is rejected wherever only object @@ -25,84 +26,29 @@ // stream lengths are preserved across JSRPC. export type R2BindingBody = ReadableStream | ArrayBuffer | string | Blob; -// Range passed to get. Contains only the fields the caller set, so it may be empty. -export type R2BindingRange = { - offset?: number; - length?: number; - suffix?: number; -}; - -// Options for get. `range` is omitted for Headers without a Range header. `ssecKey` is always -// hex-encoded. -export type R2BindingGetOptions = { - onlyIf?: R2Conditional | Headers | undefined; - range?: R2BindingRange | Headers | undefined; - ssecKey?: string | undefined; -}; - -// Options for put. `httpMetadata` is always an object rather than Headers, and `ssecKey` is -// always hex-encoded. Checksums are forwarded as the caller passed them. -export type R2BindingPutOptions = { - onlyIf?: R2Conditional | Headers | undefined; - httpMetadata?: R2HTTPMetadata | undefined; - customMetadata?: Record | undefined; - md5?: ArrayBuffer | ArrayBufferView | string | undefined; - sha1?: ArrayBuffer | ArrayBufferView | string | undefined; - sha256?: ArrayBuffer | ArrayBufferView | string | undefined; - sha384?: ArrayBuffer | ArrayBufferView | string | undefined; - sha512?: ArrayBuffer | ArrayBufferView | string | undefined; - storageClass?: string | undefined; - ssecKey?: string | undefined; -}; - -// Options for list. Without the r2_list_honor_include compatibility flag, `include` always -// requests both kinds of metadata, whatever the caller asked for. -export type R2BindingListOptions = { - limit?: number | undefined; - prefix?: string | undefined; - cursor?: string | undefined; - delimiter?: string | undefined; - startAfter?: string | undefined; - include?: ('httpMetadata' | 'customMetadata')[] | undefined; -}; - -// Options for createMultipartUpload. `httpMetadata` is always an object rather than Headers, -// and `ssecKey` is always hex-encoded. -export type R2BindingMultipartOptions = { - httpMetadata?: R2HTTPMetadata | undefined; - customMetadata?: Record | undefined; - storageClass?: string | undefined; - ssecKey?: string | undefined; -}; - -// Options for uploadPart. `ssecKey` is always hex-encoded. -export type R2BindingUploadPartOptions = { - ssecKey?: string | undefined; -}; - export type R2BindingService = { head(key: string): Promise; // Returns object metadata without a body when an onlyIf precondition fails. get( key: string, - options: R2BindingGetOptions | undefined + options: R2GetOptions | undefined ): Promise; put( key: string, value: R2BindingBody | null, - options: R2BindingPutOptions | undefined + options: R2PutOptions | undefined ): Promise; // The result is ignored. delete(keys: string | string[]): Promise; - list(options: R2BindingListOptions | undefined): Promise; + list(options: R2ListOptions | undefined): Promise; createMultipartUpload( key: string, - options: R2BindingMultipartOptions | undefined + options: R2MultipartOptions | undefined ): Promise<{ uploadId: string }>; // The returned partNumber is ignored in favour of the requested one, but must be present. @@ -111,7 +57,7 @@ export type R2BindingService = { uploadId: string, partNumber: number, value: R2BindingBody, - options: R2BindingUploadPartOptions | undefined + options: R2UploadPartOptions | undefined ): Promise; abortMultipartUpload(key: string, uploadId: string): Promise; diff --git a/src/workerd/api/tests/r2-binding-service/r2-observed-binding.ts b/src/workerd/api/tests/r2-binding-service/r2-observed-binding.ts index 8b786c542be..ca578c3e576 100644 --- a/src/workerd/api/tests/r2-binding-service/r2-observed-binding.ts +++ b/src/workerd/api/tests/r2-binding-service/r2-observed-binding.ts @@ -3,15 +3,7 @@ // https://opensource.org/licenses/Apache-2.0 import { WorkerEntrypoint } from 'cloudflare:workers'; -import type { - R2BindingBody, - R2BindingGetOptions, - R2BindingListOptions, - R2BindingMultipartOptions, - R2BindingPutOptions, - R2BindingService, - R2BindingUploadPartOptions, -} from './r2-binding-service'; +import type { R2BindingBody, R2BindingService } from './r2-binding-service'; type Env = { REAL_BUCKET: R2Bucket; @@ -36,16 +28,15 @@ export class ObservedR2Binding get( key: string, - options: R2BindingGetOptions | undefined + options: R2GetOptions | undefined ): Promise { - // The native binding accepts a range with no fields set, which R2Range does not model. - return this.env.REAL_BUCKET.get(key, options as R2GetOptions | undefined); + return this.env.REAL_BUCKET.get(key, options); } put( key: string, value: R2BindingBody | null, - options: R2BindingPutOptions | undefined + options: R2PutOptions | undefined ): Promise { return this.env.REAL_BUCKET.put(key, value, options); } @@ -54,13 +45,13 @@ export class ObservedR2Binding return this.env.REAL_BUCKET.delete(keys); } - list(options: R2BindingListOptions | undefined): Promise { + list(options: R2ListOptions | undefined): Promise { return this.env.REAL_BUCKET.list(options); } async createMultipartUpload( key: string, - options: R2BindingMultipartOptions | undefined + options: R2MultipartOptions | undefined ): Promise<{ uploadId: string }> { const upload = await this.env.REAL_BUCKET.createMultipartUpload( key, @@ -74,7 +65,7 @@ export class ObservedR2Binding uploadId: string, partNumber: number, value: R2BindingBody, - options: R2BindingUploadPartOptions | undefined + options: R2UploadPartOptions | undefined ): Promise { return this.env.REAL_BUCKET.resumeMultipartUpload(key, uploadId).uploadPart( partNumber, diff --git a/src/workerd/api/tests/r2-jsrpc-input-test.js b/src/workerd/api/tests/r2-jsrpc-input-test.js new file mode 100644 index 00000000000..796057af6fc --- /dev/null +++ b/src/workerd/api/tests/r2-jsrpc-input-test.js @@ -0,0 +1,94 @@ +// Copyright (c) 2017-2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +import assert from 'node:assert'; +import { WorkerEntrypoint } from 'cloudflare:workers'; + +// Verifies that the named-method client does not reject or normalize malformed arguments before +// the custom backend receives them. Each method throws a recognizable server-side error after +// checking its arguments. +export class InputObservingR2Binding extends WorkerEntrypoint { + get(key, options) { + assert.strictEqual(key, 'invalid-get'); + assert.strictEqual(options.range.offset, -1); + throw new Error('server observed get'); + } + + put(key, value, options) { + assert.strictEqual(key, 'invalid-put'); + assert.strictEqual(value, 'body'); + assert.strictEqual(options.md5.byteLength, 1); + assert(options.httpMetadata instanceof Headers); + throw new Error('server observed put'); + } + + createMultipartUpload(key, options) { + assert.strictEqual(key, 'invalid-create'); + assert.strictEqual(options.ssecKey, 'bad'); + throw new Error('server observed createMultipartUpload'); + } + + uploadPart(key, uploadId, partNumber, value, options) { + assert.strictEqual(key, 'invalid-upload-part'); + assert.strictEqual(uploadId, 'upload-id'); + assert.strictEqual(partNumber, 0); + assert.strictEqual(value, 'body'); + assert.strictEqual(options.ssecKey, 'bad'); + throw new Error('server observed uploadPart'); + } + + completeMultipartUpload(key, uploadId, uploadedParts) { + assert.strictEqual(key, 'invalid-complete'); + assert.strictEqual(uploadId, 'upload-id'); + assert.deepStrictEqual(uploadedParts, [{ partNumber: 0, etag: 'invalid' }]); + throw new Error('server observed completeMultipartUpload'); + } + + list(options) { + assert.deepStrictEqual(options.include, ['unsupported']); + throw new Error('server observed list'); + } +} + +export const inputForwardingTests = { + async test(_controller, env) { + await assert.rejects( + env.BUCKET.get('invalid-get', { range: { offset: -1 } }), + { message: 'server observed get' } + ); + await assert.rejects( + env.BUCKET.put('invalid-put', 'body', { + md5: new Uint8Array(1), + httpMetadata: new Headers({ 'content-type': 'text/plain' }), + }), + { message: 'server observed put' } + ); + await assert.rejects( + env.BUCKET.createMultipartUpload('invalid-create', { ssecKey: 'bad' }), + { message: 'server observed createMultipartUpload' } + ); + + const uploadPart = env.BUCKET.resumeMultipartUpload( + 'invalid-upload-part', + 'upload-id' + ); + await assert.rejects(uploadPart.uploadPart(0, 'body', { ssecKey: 'bad' }), { + message: 'server observed uploadPart', + }); + + const complete = env.BUCKET.resumeMultipartUpload( + 'invalid-complete', + 'upload-id' + ); + await assert.rejects( + complete.complete([{ partNumber: 0, etag: 'invalid' }]), + { message: 'server observed completeMultipartUpload' } + ); + + // Unsupported includes reach the backend both before and after r2_list_honor_include. + await assert.rejects(env.BUCKET.list({ include: ['unsupported'] }), { + message: 'server observed list', + }); + }, +}; diff --git a/src/workerd/api/tests/r2-jsrpc-input-test.wd-test b/src/workerd/api/tests/r2-jsrpc-input-test.wd-test new file mode 100644 index 00000000000..8bb937be170 --- /dev/null +++ b/src/workerd/api/tests/r2-jsrpc-input-test.wd-test @@ -0,0 +1,41 @@ +using Workerd = import "/workerd/workerd.capnp"; + +# Runs the input-forwarding contract both before and after r2_list_honor_include. The backend has +# no native R2 binding: any expected R2 error must therefore come from its named method. +const unitTests :Workerd.Config = ( + services = [ + (name = "modern", worker = ( + modules = .testModules, + bindings = [ + (name = "BUCKET", + r2Bucket = (name = "backend", entrypoint = "InputObservingR2Binding")), + ], + compatibilityFlags = ["experimental", "nodejs_compat", "service_binding_extra_handlers", + "streams_enable_constructors", "r2_list_honor_include", + "r2_binding_jsrpc", "rpc", "fetcher_no_get_put_delete"], + )), + (name = "legacy-list", worker = ( + modules = .testModules, + bindings = [ + (name = "BUCKET", + r2Bucket = (name = "backend", entrypoint = "InputObservingR2Binding")), + ], + compatibilityFlags = ["experimental", "nodejs_compat", "service_binding_extra_handlers", + "streams_enable_constructors", "r2_binding_jsrpc", "rpc", + "fetcher_no_get_put_delete"], + )), + (name = "backend", worker = ( + modules = [ + (name = "worker", + esModule = "export { InputObservingR2Binding } from './test.js';"), + (name = "test.js", esModule = embed "r2-jsrpc-input-test.js"), + ], + compatibilityFlags = ["nodejs_compat", "streams_enable_constructors"], + )), + ], +); + +const testModules :List(Workerd.Worker.Module) = [ + (name = "worker", esModule = "export { inputForwardingTests } from './test.js';"), + (name = "test.js", esModule = embed "r2-jsrpc-input-test.js"), +]; diff --git a/src/workerd/api/tests/r2-jsrpc-response-test.js b/src/workerd/api/tests/r2-jsrpc-response-test.js new file mode 100644 index 00000000000..35f1864abb6 --- /dev/null +++ b/src/workerd/api/tests/r2-jsrpc-response-test.js @@ -0,0 +1,84 @@ +// Copyright (c) 2017-2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +import assert from 'node:assert'; +import mock from './r2-test.js'; +import { ObservedR2Binding as PassthroughR2Binding } from './r2-observed-binding.js'; + +// Returns native R2 resources that violate a named method's result contract. +export class InvalidR2ResponseEntrypoint extends PassthroughR2Binding { + async head(requestKey) { + if (requestKey === 'body-as-head') { + return this.env.REAL_BUCKET.get('rpc-json'); + } + if (requestKey === 'list-object-as-head') { + const listed = await this.env.REAL_BUCKET.list({ + prefix: 'httpMeta', + include: [], + }); + return listed.objects[0]; + } + return super.head(requestKey); + } + + async get(requestKey, options) { + if (requestKey === 'list-object-as-get') { + const listed = await this.env.REAL_BUCKET.list({ + prefix: 'httpMeta', + include: [], + }); + return listed.objects[0]; + } + return super.get(requestKey, options); + } + + async createMultipartUpload(requestKey, options) { + if (requestKey === 'upload-id-as-string') { + return 'upload-id'; + } + if (requestKey === 'upload-id-as-number') { + return { uploadId: 123 }; + } + if (requestKey === 'upload-id-missing') { + return {}; + } + if (requestKey === 'upload-id-with-key') { + return { key: requestKey, uploadId: 'upload-id' }; + } + return super.createMultipartUpload(requestKey, options); + } +} + +export default { + fetch: mock.fetch, + + async test(_controller, env) { + const internalError = { message: /^internal error; reference = \S+$/ }; + await assert.rejects(() => env.BUCKET.head('body-as-head'), internalError); + await assert.rejects( + () => env.BUCKET.head('list-object-as-head'), + internalError + ); + await assert.rejects( + () => env.BUCKET.get('list-object-as-get'), + internalError + ); + + for (const key of [ + 'upload-id-as-string', + 'upload-id-as-number', + 'upload-id-missing', + ]) { + await assert.rejects( + () => env.BUCKET.createMultipartUpload(key), + internalError + ); + } + + // Extra properties do not prevent construction of a valid local multipart resource. + const upload = await env.BUCKET.createMultipartUpload('upload-id-with-key'); + assert.strictEqual(upload.key, 'upload-id-with-key'); + assert.strictEqual(upload.uploadId, 'upload-id'); + }, +}; diff --git a/src/workerd/api/tests/r2-jsrpc-response-test.wd-test b/src/workerd/api/tests/r2-jsrpc-response-test.wd-test new file mode 100644 index 00000000000..c6ee35b8132 --- /dev/null +++ b/src/workerd/api/tests/r2-jsrpc-response-test.wd-test @@ -0,0 +1,63 @@ +using Workerd = import "/workerd/workerd.capnp"; + +# Response decoding reports malformed native R2 resources as internal contract errors with either +# JSG struct implementation. Each platform Worker also serves the native R2 HTTP mock used to +# construct the resources returned by InvalidR2ResponseEntrypoint. +const unitTests :Workerd.Config = ( + services = [ + (name = "fast-structs", worker = ( + modules = .testModules, + bindings = [ + (name = "BUCKET", + r2Bucket = (name = "fast-platform", entrypoint = "InvalidR2ResponseEntrypoint")), + ], + compatibilityFlags = ["experimental", "nodejs_compat", "service_binding_extra_handlers", + "streams_enable_constructors", "r2_list_honor_include", "r2_binding_jsrpc", + "rpc", "fetcher_no_get_put_delete", "enable_fast_jsg_struct"], + )), + (name = "fast-platform", worker = ( + modules = .platformModules, + bindings = [ + (name = "REAL_BUCKET", r2Bucket = "fast-platform"), + (name = "R2_LIST_HONOR_INCLUDE", text = "true"), + ], + compatibilityFlags = ["nodejs_compat", "streams_enable_constructors", + "r2_list_honor_include", "enable_fast_jsg_struct"], + )), + (name = "slow-structs", worker = ( + modules = .testModules, + bindings = [ + (name = "BUCKET", + r2Bucket = (name = "slow-platform", entrypoint = "InvalidR2ResponseEntrypoint")), + ], + compatibilityFlags = ["experimental", "nodejs_compat", "service_binding_extra_handlers", + "streams_enable_constructors", "r2_list_honor_include", "r2_binding_jsrpc", + "rpc", "fetcher_no_get_put_delete", "disable_fast_jsg_struct"], + )), + (name = "slow-platform", worker = ( + modules = .platformModules, + bindings = [ + (name = "REAL_BUCKET", r2Bucket = "slow-platform"), + (name = "R2_LIST_HONOR_INCLUDE", text = "true"), + ], + compatibilityFlags = ["nodejs_compat", "streams_enable_constructors", + "r2_list_honor_include", "disable_fast_jsg_struct"], + )), + ], +); + +const testModules :List(Workerd.Worker.Module) = [ + (name = "worker", esModule = embed "r2-jsrpc-response-test.js"), + (name = "r2-test.js", esModule = embed "r2-test.js"), + (name = "r2-observed-binding.js", + esModule = embed "r2-binding-service/r2-observed-binding.js"), +]; + +const platformModules :List(Workerd.Worker.Module) = [ + (name = "worker", + esModule = "import mock from './r2-test.js'; export { InvalidR2ResponseEntrypoint } from './test.js'; export default { fetch: mock.fetch };"), + (name = "test.js", esModule = embed "r2-jsrpc-response-test.js"), + (name = "r2-test.js", esModule = embed "r2-test.js"), + (name = "r2-observed-binding.js", + esModule = embed "r2-binding-service/r2-observed-binding.js"), +]; diff --git a/src/workerd/api/tests/r2-jsrpc-serialization-test.js b/src/workerd/api/tests/r2-jsrpc-serialization-test.js deleted file mode 100644 index d2968256b0d..00000000000 --- a/src/workerd/api/tests/r2-jsrpc-serialization-test.js +++ /dev/null @@ -1,209 +0,0 @@ -// Copyright (c) 2017-2026 Cloudflare, Inc. -// Licensed under the Apache 2.0 license found in the LICENSE file or at: -// https://opensource.org/licenses/Apache-2.0 - -import assert from 'node:assert'; -import { ObservedR2Binding as PassthroughR2Binding } from './r2-observed-binding.js'; - -// Extends the shared passthrough so that some keys return a native R2 result of the wrong type -// for the method, and adds methods that return native results to the test directly, without the -// R2 binding's JSRPC client in between. -export class ObservedR2Binding extends PassthroughR2Binding { - async head(requestKey) { - if (requestKey === 'body-as-head') { - return this.env.REAL_BUCKET.get('rpc-json'); - } - if (requestKey === 'list-object-as-head') { - const listed = await this.env.REAL_BUCKET.list({ - prefix: 'httpMeta', - include: [], - }); - return listed.objects[0]; - } - return super.head(requestKey); - } - - async get(requestKey, options) { - if (requestKey === 'list-object-as-get') { - const listed = await this.env.REAL_BUCKET.list({ - prefix: 'httpMeta', - include: [], - }); - return listed.objects[0]; - } - return super.get(requestKey, options); - } - - async createMultipartUpload(requestKey, options) { - // The backend must reply with `{ uploadId: string }`. - if (requestKey === 'upload-id-as-string') { - return 'upload-id'; - } - if (requestKey === 'upload-id-as-number') { - return { uploadId: 123 }; - } - if (requestKey === 'upload-id-missing') { - return {}; - } - if (requestKey === 'upload-id-with-key') { - return { key: requestKey, uploadId: 'upload-id' }; - } - return super.createMultipartUpload(requestKey, options); - } - - async checksums() { - return (await this.env.REAL_BUCKET.head('multipleChecksums')).checksums; - } - - getNative(requestKey) { - return this.env.REAL_BUCKET.get(requestKey); - } - - echoNative(object) { - return object; - } -} - -export const nativeR2ResultTests = { - async test(_controller, env) { - const checksums = await env.PLATFORM.checksums(); - assert.throws(() => structuredClone(checksums), { name: 'DataCloneError' }); - const checksumJson = checksums.toJSON(); - assert.strictEqual(checksumJson.md5, '9a0364b9e99bb480dd25e1f0284c8555'); - assert.strictEqual( - checksumJson.sha1, - '2a0364b9e99bb480dd25e1f0284c855511223344' - ); - assert.strictEqual( - checksumJson.sha256, - '3a0364b9e99bb480dd25e1f0284c8555112233445566778899aabbccddeeff00' - ); - - const head = await env.BUCKET.head('multipleChecksums'); - assert.throws(() => structuredClone(head), { name: 'DataCloneError' }); - assert.strictEqual(head.key, 'basicKey'); - assert.strictEqual(typeof head.writeHttpMetadata, 'function'); - const headChecksumJson = head.checksums.toJSON(); - assert.strictEqual( - headChecksumJson.md5, - '9a0364b9e99bb480dd25e1f0284c8555' - ); - assert.strictEqual( - headChecksumJson.sha1, - '2a0364b9e99bb480dd25e1f0284c855511223344' - ); - assert.strictEqual( - headChecksumJson.sha256, - '3a0364b9e99bb480dd25e1f0284c8555112233445566778899aabbccddeeff00' - ); - await assert.rejects(() => env.BUCKET.head('body-as-head'), { - message: /^internal error; reference = \S+$/, - }); - await assert.rejects(() => env.BUCKET.head('list-object-as-head'), { - message: /^internal error; reference = \S+$/, - }); - - const directGet = await env.PLATFORM.getNative('rpc-json'); - assert.throws(() => structuredClone(directGet), { name: 'DataCloneError' }); - assert.deepStrictEqual(await directGet.json(), { ok: true }); - assert.strictEqual(directGet.bodyUsed, true); - - const object = await env.BUCKET.get('rpc-json'); - assert.strictEqual(typeof object.text, 'function'); - assert.strictEqual(object.bodyUsed, false); - const objectBody = object.body; - assert.strictEqual(object.body, objectBody); - assert.strictEqual(object.bodyUsed, false); - assert.strictEqual(objectBody.locked, false); - assert.deepStrictEqual(await object.json(), { ok: true }); - assert.strictEqual(object.body, objectBody); - assert.strictEqual(object.bodyUsed, true); - - // Each consumption method must work without first accessing the body getter. - for (const method of ['arrayBuffer', 'bytes', 'text', 'blob']) { - const result = await env.PLATFORM.getNative('rpc-lazy-body'); - const consumed = await result[method](); - assert.strictEqual(await new Response(consumed).text(), 'content'); - assert.strictEqual(result.bodyUsed, true); - } - - // A deserialized native result must be serializable again before its body is accessed. - const echoed = await env.PLATFORM.echoNative( - await env.PLATFORM.getNative('rpc-json') - ); - assert.deepStrictEqual(await echoed.json(), { ok: true }); - - // Both service RPC and an R2 interceptor preserve the length for immediate streaming copies. - for (const get of [ - () => env.PLATFORM.getNative('rpc-lazy-body'), - () => env.BUCKET.get('rpc-lazy-body'), - ]) { - const source = await get(); - const copied = await env.BUCKET.put('basicKey', source.body); - assert.strictEqual(copied.size, 7); - assert.strictEqual(source.bodyUsed, true); - } - - const metadataOnly = await env.BUCKET.get('rpc-conditional-metadata'); - assert.strictEqual(metadataOnly.key, 'basicKey'); - assert.strictEqual(metadataOnly.body, undefined); - await assert.rejects(() => env.BUCKET.get('list-object-as-get'), { - message: /^internal error; reference = \S+$/, - }); - - const put = await env.BUCKET.put('platform-put', 'content'); - assert.strictEqual(put.key, 'basicKey'); - assert.strictEqual(typeof put.writeHttpMetadata, 'function'); - - const listed = await env.BUCKET.list({ - prefix: 'rpc-metadata', - include: ['httpMetadata', 'customMetadata'], - }); - assert.strictEqual(listed.objects.length, 1); - assert.strictEqual(listed.objects[0].key, 'basicKey'); - assert.strictEqual(typeof listed.objects[0].writeHttpMetadata, 'function'); - - const listedWithoutMetadata = await env.BUCKET.list({ - prefix: 'httpMeta', - include: [], - }); - assert.strictEqual( - listedWithoutMetadata.objects[0].httpMetadata, - undefined - ); - assert.strictEqual( - listedWithoutMetadata.objects[0].customMetadata, - undefined - ); - - const upload = await env.BUCKET.createMultipartUpload( - 'rpc-multipart-platform' - ); - assert.strictEqual(upload.key, 'rpc-multipart-platform'); - assert.strictEqual(upload.uploadId, 'rpc-multipart-platform-id'); - const part = await upload.uploadPart(1, 'content'); - assert.strictEqual( - part.etag, - 'rpc-multipart-platform/rpc-multipart-platform-id/1' - ); - const completed = await upload.complete([part]); - assert.strictEqual(completed.key, 'rpc-multipart-platform'); - assert.strictEqual(completed.version, 'rpc-multipart-platform-id'); - - const aborted = await env.BUCKET.createMultipartUpload( - 'rpc-multipart-abort-platform' - ); - await aborted.abort(); - - for (const key of [ - 'upload-id-as-string', - 'upload-id-as-number', - 'upload-id-missing', - 'upload-id-with-key', - ]) { - await assert.rejects(() => env.BUCKET.createMultipartUpload(key), { - message: /^internal error; reference = \S+$/, - }); - } - }, -}; diff --git a/src/workerd/api/tests/r2-resource-serialization-test.js b/src/workerd/api/tests/r2-resource-serialization-test.js new file mode 100644 index 00000000000..dbffd31beb0 --- /dev/null +++ b/src/workerd/api/tests/r2-resource-serialization-test.js @@ -0,0 +1,74 @@ +// Copyright (c) 2017-2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +import assert from 'node:assert'; +import { ObservedR2Binding as PassthroughR2Binding } from './r2-observed-binding.js'; + +// Exposes native R2 resources through an ordinary service binding so this suite exercises their +// JSRPC serializers without involving the R2 binding's named-method client. +export class ResourceSerializationEntrypoint extends PassthroughR2Binding { + async checksums() { + return (await this.env.REAL_BUCKET.head('multipleChecksums')).checksums; + } + + headNative(requestKey) { + return this.env.REAL_BUCKET.head(requestKey); + } + + getNative(requestKey) { + return this.env.REAL_BUCKET.get(requestKey); + } + + echoNative(object) { + return object; + } +} + +export const resourceSerializationTests = { + async test(_controller, env) { + const checksums = await env.PLATFORM.checksums(); + assert.throws(() => structuredClone(checksums), { name: 'DataCloneError' }); + const checksumJson = checksums.toJSON(); + assert.strictEqual(checksumJson.md5, '9a0364b9e99bb480dd25e1f0284c8555'); + assert.strictEqual( + checksumJson.sha1, + '2a0364b9e99bb480dd25e1f0284c855511223344' + ); + assert.strictEqual( + checksumJson.sha256, + '3a0364b9e99bb480dd25e1f0284c8555112233445566778899aabbccddeeff00' + ); + + const head = await env.PLATFORM.headNative('multipleChecksums'); + assert.throws(() => structuredClone(head), { name: 'DataCloneError' }); + assert.strictEqual(head.key, 'basicKey'); + assert.strictEqual(typeof head.writeHttpMetadata, 'function'); + assert.deepStrictEqual(head.checksums.toJSON(), checksumJson); + + const directGet = await env.PLATFORM.getNative('rpc-json'); + assert.throws(() => structuredClone(directGet), { name: 'DataCloneError' }); + assert.deepStrictEqual(await directGet.json(), { ok: true }); + assert.strictEqual(directGet.bodyUsed, true); + + // Each consumption method must work without first accessing the body getter. + for (const method of ['arrayBuffer', 'bytes', 'text', 'blob']) { + const result = await env.PLATFORM.getNative('rpc-lazy-body'); + const consumed = await result[method](); + assert.strictEqual(await new Response(consumed).text(), 'content'); + assert.strictEqual(result.bodyUsed, true); + } + + // A deserialized native result must be serializable again before its body is accessed. + const echoed = await env.PLATFORM.echoNative( + await env.PLATFORM.getNative('rpc-json') + ); + assert.deepStrictEqual(await echoed.json(), { ok: true }); + + // The resource serializer preserves a body stream's known length across a service RPC hop. + const source = await env.PLATFORM.getNative('rpc-lazy-body'); + const copied = await env.BUCKET.put('basicKey', source.body); + assert.strictEqual(copied.size, 7); + assert.strictEqual(source.bodyUsed, true); + }, +}; diff --git a/src/workerd/api/tests/r2-jsrpc-serialization-test.wd-test b/src/workerd/api/tests/r2-resource-serialization-test.wd-test similarity index 51% rename from src/workerd/api/tests/r2-jsrpc-serialization-test.wd-test rename to src/workerd/api/tests/r2-resource-serialization-test.wd-test index 12b475b03dd..f004389553f 100644 --- a/src/workerd/api/tests/r2-jsrpc-serialization-test.wd-test +++ b/src/workerd/api/tests/r2-resource-serialization-test.wd-test @@ -1,23 +1,16 @@ using Workerd = import "/workerd/workerd.capnp"; -# The user-facing R2 binding uses JSRPC to call a platform Worker. That Worker lacks -# r2_binding_jsrpc, so its real R2 binding reaches the HTTP mock in r2-test.js over HTTP, and -# returns the resulting native R2 objects across the JSRPC hop. Keeping this topology in a -# dedicated config prevents the native-resource round trips from being conflated with the HTTP -# transport. -# -# The platform Worker also serves the HTTP mock: its REAL_BUCKET points back at itself, as in -# r2-jsrpc-passthrough.capnp. It uses its own modules because it exports the ObservedR2Binding -# subclass from r2-jsrpc-serialization-test.js, which adds methods for this test. +# Native R2 resources cross an ordinary service binding independently of the named-method R2 +# binding transport. The platform Worker obtains them from its native HTTP R2 binding. const unitTests :Workerd.Config = ( services = [ (name = "user", worker = ( modules = .testModules, bindings = [ (name = "BUCKET", - r2Bucket = (name = "platform", entrypoint = "ObservedR2Binding")), + r2Bucket = (name = "platform", entrypoint = "ResourceSerializationEntrypoint")), (name = "PLATFORM", - service = (name = "platform", entrypoint = "ObservedR2Binding")), + service = (name = "platform", entrypoint = "ResourceSerializationEntrypoint")), ], compatibilityFlags = ["experimental", "nodejs_compat", "streams_enable_constructors", "r2_list_honor_include", "r2_binding_jsrpc", "rpc", @@ -28,19 +21,19 @@ const unitTests :Workerd.Config = ( ); const testModules :List(Workerd.Worker.Module) = [ - (name = "worker", esModule = "export { nativeR2ResultTests } from './test.js';"), - (name = "test.js", esModule = embed "r2-jsrpc-serialization-test.js"), + (name = "worker", esModule = "export { resourceSerializationTests } from './test.js';"), + (name = "test.js", esModule = embed "r2-resource-serialization-test.js"), (name = "r2-test.js", esModule = embed "r2-test.js"), (name = "r2-observed-binding.js", esModule = embed "r2-binding-service/r2-observed-binding.js"), ]; -# Exports no test handlers, so the test runner does not run the tests in r2-test.js here. +# Exports no test handlers, so the test runner does not run the tests in the dependency modules. const platformWorker :Workerd.Worker = ( modules = [ (name = "worker", - esModule = "import mock from './r2-test.js'; export { ObservedR2Binding } from './test.js'; export default { fetch: mock.fetch };"), - (name = "test.js", esModule = embed "r2-jsrpc-serialization-test.js"), + esModule = "import mock from './r2-test.js'; export { ResourceSerializationEntrypoint } from './test.js'; export default { fetch: mock.fetch };"), + (name = "test.js", esModule = embed "r2-resource-serialization-test.js"), (name = "r2-test.js", esModule = embed "r2-test.js"), (name = "r2-observed-binding.js", esModule = embed "r2-binding-service/r2-observed-binding.js"), diff --git a/src/workerd/api/tests/r2-jsrpc-serialization-ts-test.wd-test b/src/workerd/api/tests/r2-resource-serialization-ts-test.wd-test similarity index 78% rename from src/workerd/api/tests/r2-jsrpc-serialization-ts-test.wd-test rename to src/workerd/api/tests/r2-resource-serialization-ts-test.wd-test index e0d84622ed4..717e599e243 100644 --- a/src/workerd/api/tests/r2-jsrpc-serialization-ts-test.wd-test +++ b/src/workerd/api/tests/r2-resource-serialization-ts-test.wd-test @@ -1,5 +1,5 @@ using Workerd = import "/workerd/workerd.capnp"; -using NativeR2 = import "r2-jsrpc-serialization-test.wd-test"; +using NativeR2 = import "r2-resource-serialization-test.wd-test"; # Native R2 object bodies cross the RPC deserializer's no-JavaScript graph-read scope. # The TypeScript stream receiver requires its streams to be hydrated before that scope. @@ -9,9 +9,9 @@ const unitTests :Workerd.Config = ( modules = NativeR2.testModules, bindings = [ (name = "BUCKET", - r2Bucket = (name = "platform", entrypoint = "ObservedR2Binding")), + r2Bucket = (name = "platform", entrypoint = "ResourceSerializationEntrypoint")), (name = "PLATFORM", - service = (name = "platform", entrypoint = "ObservedR2Binding")), + service = (name = "platform", entrypoint = "ResourceSerializationEntrypoint")), ], compatibilityFlags = ["experimental", "nodejs_compat", "streams_enable_constructors", "r2_list_honor_include", "r2_binding_jsrpc", "rpc", diff --git a/src/workerd/api/tests/r2-string-upload-test.js b/src/workerd/api/tests/r2-string-upload-test.js new file mode 100644 index 00000000000..9a2e3fea0be --- /dev/null +++ b/src/workerd/api/tests/r2-string-upload-test.js @@ -0,0 +1,35 @@ +// Copyright (c) 2017-2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +import assert from 'node:assert'; +import { stringUploadCases } from './r2-test.js'; + +function stringUploadValue({ size, suffix }) { + return ( + 'a'.repeat(size - new TextEncoder().encode(suffix).byteLength) + suffix + ); +} + +export const r2StringPutTests = { + async test(_ctrl, env) { + for (const [key, fixture] of Object.entries(stringUploadCases)) { + const result = await env.BUCKET.put(key, stringUploadValue(fixture)); + assert.strictEqual(result.key, key); + assert.strictEqual(result.size, fixture.size); + } + }, +}; + +export const r2StringUploadPartTests = { + async test(_ctrl, env) { + for (const [key, fixture] of Object.entries(stringUploadCases)) { + const upload = env.BUCKET.resumeMultipartUpload(key, 'string-upload-id'); + const result = await upload.uploadPart(1, stringUploadValue(fixture)); + assert.deepStrictEqual(result, { + partNumber: 1, + etag: `string-upload-${fixture.size}`, + }); + } + }, +}; diff --git a/src/workerd/api/tests/r2-string-upload-test.wd-test b/src/workerd/api/tests/r2-string-upload-test.wd-test index faab23807bb..bd40cee3474 100644 --- a/src/workerd/api/tests/r2-string-upload-test.wd-test +++ b/src/workerd/api/tests/r2-string-upload-test.wd-test @@ -9,7 +9,6 @@ const unitTests :Workerd.Config = ( modules = .testModules, bindings = [ (name = "BUCKET", r2Bucket = "r2-test"), - (name = "R2_STRING_UPLOAD_TEST", text = "true"), ], compatibilityFlags = ["nodejs_compat", "streams_enable_constructors"], )), @@ -17,7 +16,6 @@ const unitTests :Workerd.Config = ( modules = .testModules, bindings = [ (name = "BUCKET", r2Bucket = (name = "r2-test", entrypoint = "ObservedR2Binding")), - (name = "R2_STRING_UPLOAD_TEST", text = "true"), ], compatibilityFlags = ["experimental", "nodejs_compat", "streams_enable_constructors", "r2_binding_jsrpc"], @@ -27,6 +25,6 @@ const unitTests :Workerd.Config = ( ); const testModules :List(Workerd.Worker.Module) = [ - (name = "worker", esModule = "export { r2StringPutTests, r2StringUploadPartTests } from './r2-test.js';"), + (name = "worker", esModule = embed "r2-string-upload-test.js"), (name = "r2-test.js", esModule = embed "r2-test.js"), ]; diff --git a/src/workerd/api/tests/r2-test.js b/src/workerd/api/tests/r2-test.js index 225a42ea5ea..f27088ab890 100644 --- a/src/workerd/api/tests/r2-test.js +++ b/src/workerd/api/tests/r2-test.js @@ -8,7 +8,7 @@ import { Buffer } from 'node:buffer'; const key = 'basicKey'; const body = 'content'; const rpcStreamBody = 'café'; -const stringUploadCases = { +export const stringUploadCases = { 'string-upload-below-limit': { size: (8 << 20) - 1, suffix: 'Ā' }, 'string-upload-at-limit': { size: 8 << 20, suffix: 'Ā' }, 'string-upload-above-limit': { size: (8 << 20) + 1, suffix: 'Ā' }, @@ -1433,23 +1433,39 @@ export const r2ValidationTests = { { message: 'You cannot specify multiple hashing algorithms.' } ); - const invalidOptionsStream = new FixedLengthStream(1); - const invalidOptionsWriter = invalidOptionsStream.writable.getWriter(); - const invalidOptionsWriting = invalidOptionsWriter.write( - new Uint8Array([1]) - ); - await assert.rejects( - env.BUCKET.put('must-not-call', invalidOptionsStream.readable, { - onlyIf: { etagMatches: '"quoted"' }, - }), - { - message: 'Conditional ETag should not be wrapped in quotes ("quoted").', - } - ); - await assert.rejects(invalidOptionsWriting, { - message: - 'Stream cancelled because the associated put operation encountered an error.', - }); + if (env.R2_TRANSPORT === 'jsrpc') { + await assert.rejects( + env.BUCKET.put('must-not-call', body, { + onlyIf: { etagMatches: '"quoted"' }, + }), + { + message: + 'Conditional ETag should not be wrapped in quotes ("quoted").', + } + ); + } else { + // The HTTP client owns this validation and cancels a stream it cannot send. The JSRPC + // client forwards the call, so stream cancellation is instead owned by the backend and the + // generic RPC stream transport. + const invalidOptionsStream = new FixedLengthStream(1); + const invalidOptionsWriter = invalidOptionsStream.writable.getWriter(); + const invalidOptionsWriting = invalidOptionsWriter.write( + new Uint8Array([1]) + ); + await assert.rejects( + env.BUCKET.put('must-not-call', invalidOptionsStream.readable, { + onlyIf: { etagMatches: '"quoted"' }, + }), + { + message: + 'Conditional ETag should not be wrapped in quotes ("quoted").', + } + ); + await assert.rejects(invalidOptionsWriting, { + message: + 'Stream cancelled because the associated put operation encountered an error.', + }); + } await assert.rejects( env.BUCKET.createMultipartUpload('must-not-call', { @@ -1519,6 +1535,13 @@ export const r2BindingApiTests = { assert.strictEqual(await lazyBody.text(), body); assert.strictEqual(lazyBody.bodyUsed, true); + if (env.R2_TRANSPORT === 'jsrpc') { + const copySource = await env.BUCKET.get('rpc-lazy-body'); + const copied = await env.BUCKET.put('basicKey', copySource.body); + assert.strictEqual(copied.size, body.length); + assert.strictEqual(copySource.bodyUsed, true); + } + const bytesResult = await env.BUCKET.get('rpc-bytes'); assert.deepStrictEqual( await bytesResult.bytes(), @@ -1768,35 +1791,4 @@ export const r2MultipartApiTests = { }, }; -function stringUploadValue({ size, suffix }) { - return ( - 'a'.repeat(size - new TextEncoder().encode(suffix).byteLength) + suffix - ); -} - -export const r2StringPutTests = { - async test(_ctrl, env) { - if (env.R2_STRING_UPLOAD_TEST !== 'true') return; - for (const [key, fixture] of Object.entries(stringUploadCases)) { - const result = await env.BUCKET.put(key, stringUploadValue(fixture)); - assert.strictEqual(result.key, key); - assert.strictEqual(result.size, fixture.size); - } - }, -}; - -export const r2StringUploadPartTests = { - async test(_ctrl, env) { - if (env.R2_STRING_UPLOAD_TEST !== 'true') return; - for (const [key, fixture] of Object.entries(stringUploadCases)) { - const upload = env.BUCKET.resumeMultipartUpload(key, 'string-upload-id'); - const result = await upload.uploadPart(1, stringUploadValue(fixture)); - assert.deepStrictEqual(result, { - partNumber: 1, - etag: `string-upload-${fixture.size}`, - }); - } - }, -}; - export default testWorker; From fcbdd084d6274c2775dac412e52d9067f791bcda Mon Sep 17 00:00:00 2001 From: Bradley Thwaites Date: Wed, 28 Oct 2026 14:00:02 -0400 Subject: [PATCH 13/13] Add test cases for r2 jsrpc stream cancellation with jsrpc-readable-cancel-propagation --- compile_flags.txt | 2 +- doxyfile | 2 +- src/workerd/api/tests/r2-jsrpc-input-test.js | 116 +++++++++++++++++- .../api/tests/r2-jsrpc-input-test.wd-test | 1 + 4 files changed, 113 insertions(+), 8 deletions(-) diff --git a/compile_flags.txt b/compile_flags.txt index 54cdb5bb728..4a757f2967e 100644 --- a/compile_flags.txt +++ b/compile_flags.txt @@ -136,4 +136,4 @@ -Wunused-lambda-capture -Wunused-variable -no-canonical-prefixes --fbracket-depth=512 +-fbracket-depth=1024 diff --git a/doxyfile b/doxyfile index f9f49a065e4..580f6060810 100644 --- a/doxyfile +++ b/doxyfile @@ -1140,7 +1140,7 @@ CLANG_OPTIONS = -std=c++23 \ -stdlib=libc++ \ -xc++ \ -nostdinc \ - -fbracket-depth=512 \ + -fbracket-depth=1024 \ -D_FORTIFY_SOURCE=1 \ -DCAPNP_VERSION=11000 \ -DDEBUG \ diff --git a/src/workerd/api/tests/r2-jsrpc-input-test.js b/src/workerd/api/tests/r2-jsrpc-input-test.js index 796057af6fc..8a7b9d4732d 100644 --- a/src/workerd/api/tests/r2-jsrpc-input-test.js +++ b/src/workerd/api/tests/r2-jsrpc-input-test.js @@ -5,6 +5,45 @@ import assert from 'node:assert'; import { WorkerEntrypoint } from 'cloudflare:workers'; +const REMOTE_RELEASED_MESSAGE = + 'ReadableStream sent over RPC was canceled or released by the remote execution context.'; +const STREAM_CANCEL_MESSAGE = 'server canceled put body'; + +function makeObservedStream(chunk) { + const canceled = Promise.withResolvers(); + let cancelCalls = 0; + const stream = new ReadableStream({ + start(controller) { + if (chunk !== undefined) controller.enqueue(chunk); + }, + cancel(reason) { + ++cancelCalls; + canceled.resolve(reason); + }, + }); + return { + stream, + whenCanceled: canceled.promise, + get cancelCalls() { + return cancelCalls; + }, + }; +} + +async function awaitCancellation(observed) { + // A timer keeps the caller's execution context alive while the rejected RPC invocation tears + // down and releases its received stream. + const pending = Symbol('pending'); + for (let attempt = 0; attempt < 100; attempt++) { + const outcome = await Promise.race([ + observed.whenCanceled, + scheduler.wait(50).then(() => pending), + ]); + if (outcome !== pending) return outcome; + } + assert.fail('the origin never observed the backend releasing the stream'); +} + // Verifies that the named-method client does not reject or normalize malformed arguments before // the custom backend receives them. Each method throws a recognizable server-side error after // checking its arguments. @@ -15,12 +54,32 @@ export class InputObservingR2Binding extends WorkerEntrypoint { throw new Error('server observed get'); } - put(key, value, options) { - assert.strictEqual(key, 'invalid-put'); - assert.strictEqual(value, 'body'); - assert.strictEqual(options.md5.byteLength, 1); - assert(options.httpMetadata instanceof Headers); - throw new Error('server observed put'); + async put(key, value, options) { + switch (key) { + case 'invalid-put': + assert.strictEqual(value, 'body'); + assert.strictEqual(options.md5.byteLength, 1); + assert(options.httpMetadata instanceof Headers); + throw new Error('server observed put'); + case 'cancel-stream': + assert(value instanceof ReadableStream); + await value.cancel(new Error(STREAM_CANCEL_MESSAGE)); + throw new Error('server canceled stream'); + case 'throw-before-consuming-stream': + assert(value instanceof ReadableStream); + throw new Error('server rejected stream before consuming it'); + case 'throw-during-stream': { + assert(value instanceof ReadableStream); + const reader = value.getReader(); + const first = await reader.read(); + assert.strictEqual(first.done, false); + assert.deepStrictEqual(first.value, new Uint8Array([1, 2, 3])); + reader.releaseLock(); + throw new Error('server rejected stream during consumption'); + } + default: + assert.fail(`unexpected put key: ${key}`); + } } createMultipartUpload(key, options) { @@ -90,5 +149,50 @@ export const inputForwardingTests = { await assert.rejects(env.BUCKET.list({ include: ['unsupported'] }), { message: 'server observed list', }); + + const explicitlyCanceled = makeObservedStream(); + await assert.rejects( + env.BUCKET.put('cancel-stream', explicitlyCanceled.stream), + { message: 'server canceled stream' } + ); + const explicitReason = await awaitCancellation(explicitlyCanceled); + assert.strictEqual(explicitlyCanceled.cancelCalls, 1); + assert(explicitReason instanceof Error); + assert.strictEqual(explicitReason.message, STREAM_CANCEL_MESSAGE); + + const rejectedBeforeConsumption = makeObservedStream(); + await assert.rejects( + env.BUCKET.put( + 'throw-before-consuming-stream', + rejectedBeforeConsumption.stream + ), + { message: 'server rejected stream before consuming it' } + ); + const beforeConsumptionReason = await awaitCancellation( + rejectedBeforeConsumption + ); + assert.strictEqual(rejectedBeforeConsumption.cancelCalls, 1); + assert(beforeConsumptionReason instanceof Error); + assert.strictEqual( + beforeConsumptionReason.message, + REMOTE_RELEASED_MESSAGE + ); + + const rejectedDuringConsumption = makeObservedStream( + new Uint8Array([1, 2, 3]) + ); + await assert.rejects( + env.BUCKET.put('throw-during-stream', rejectedDuringConsumption.stream), + { message: 'server rejected stream during consumption' } + ); + const duringConsumptionReason = await awaitCancellation( + rejectedDuringConsumption + ); + assert.strictEqual(rejectedDuringConsumption.cancelCalls, 1); + assert(duringConsumptionReason instanceof Error); + assert.strictEqual( + duringConsumptionReason.message, + REMOTE_RELEASED_MESSAGE + ); }, }; diff --git a/src/workerd/api/tests/r2-jsrpc-input-test.wd-test b/src/workerd/api/tests/r2-jsrpc-input-test.wd-test index 8bb937be170..58d72148af5 100644 --- a/src/workerd/api/tests/r2-jsrpc-input-test.wd-test +++ b/src/workerd/api/tests/r2-jsrpc-input-test.wd-test @@ -33,6 +33,7 @@ const unitTests :Workerd.Config = ( compatibilityFlags = ["nodejs_compat", "streams_enable_constructors"], )), ], + autogates = ["workerd-autogate-jsrpc-readable-cancel-propagation"], ); const testModules :List(Workerd.Worker.Module) = [