diff --git a/cloud/inference-sdk/direct.mdx b/cloud/inference-sdk/direct.mdx new file mode 100644 index 0000000..3b9a2c0 --- /dev/null +++ b/cloud/inference-sdk/direct.mdx @@ -0,0 +1,98 @@ +--- +title: "SDK" +icon: "braces" +description: "Experimental SDK notes for direct model endpoints." +--- + + + Direct model endpoints are experimental and are not recommended for new integrations or production verification workflows. Use the NEAR AI Cloud Gateway instead. + + +Independent requests to the same direct hostname can reach different serving instances. An attestation report does not establish deployment evidence for a later completion, and a later signature lookup can return `404`. Do not use direct response signatures as a production verification gate. + +Use `DirectInferenceClient` only when maintaining an existing direct integration. + +## Install + +The package requires Node.js 24 or later. + +```bash +npm install @nearai/inference-sdk@next +``` + +## Send and verify an E2EE Chat Completion + +```ts +import { DirectInferenceClient } from '@nearai/inference-sdk/node'; + +const baseUrl = 'https://glm-5-3-flash.completions.near.ai/v1'; +const model = 'z-ai/glm-5.3-flash'; +const apiKey = process.env.NEARAI_API_KEY; +if (!apiKey) throw new Error('NEARAI_API_KEY is required'); + +const client = new DirectInferenceClient({ + baseUrl, + apiKey, + e2ee: true, +}); + +const completion = await client.chat.completions.create({ + model, + messages: [{ role: 'user', content: 'Reply with the word ok.' }], +}); + +console.log(completion.choices[0]?.message.content); + +const receipt = await client.verifyResponse(completion.id); +console.log(`Verified ${receipt.signatureKind} response.`); +``` + +Before sending the completion, the SDK verifies each model-attestation report returned by that endpoint. It does not request Gateway evidence. It encrypts supported Chat fields to a key from the verified model evidence and decrypts the response automatically. + + + `verifyResponse()` is explicit. For a streaming completion, consume the entire stream first, then call it with the completion ID so the SDK can verify the exact response bytes. + + +## Add OHTTP + +Set `ohttp: true` to encapsulate Chat requests and responses with OHTTP. OHTTP uses the SDK's default Ed25519 signing algorithm; it is not compatible with `signingAlgo: 'ecdsa'`. + +```ts +const client = new DirectInferenceClient({ + baseUrl, + apiKey, + e2ee: true, + ohttp: true, +}); +``` + +The Chat and `verifyResponse()` calls stay the same. The SDK verifies the direct endpoint's signed OHTTP configuration before sending Chat; a missing or invalid configuration blocks the request. For a direct endpoint, both OHTTP and E2EE are scoped to the model service. + + + OHTTP applies only to Chat. Attestation and signature requests remain regular HTTPS. The configured endpoint or proxy must expose `/ohttp` at the same origin. Authorization and custom headers sent on the outer request, along with the client's network address, are not hidden by OHTTP. + + +## Node, browser, and proxy integrations + +The example uses `@nearai/inference-sdk/node`. Direct endpoint TLS binding is not currently enabled; normal HTTPS certificate validation still applies. + +For a browser or an application proxy, import from `@nearai/inference-sdk`. Configure the same explicit `baseUrl` for the endpoint your application uses. + +## Examples + +See the [Inference SDK examples and setup instructions](https://github.com/nearai/inference-sdk/tree/main/examples#direct-model-endpoints) for runnable direct-endpoint projects. + +- [DirectInferenceClient](https://github.com/nearai/inference-sdk/blob/main/examples/example-js/direct/client.ts): streaming and non-streaming Chat with E2EE and response verification. +- [OpenAI SDK integration](https://github.com/nearai/inference-sdk/blob/main/examples/example-js/direct/client-openai-sdk.ts): use `DirectInferenceClient.fetch` with the OpenAI client. +- [Manual verification](https://github.com/nearai/inference-sdk/blob/main/examples/example-js/direct/bare.ts): verify model attestations and response signatures directly. This example sends plaintext Chat bodies over HTTPS without E2EE. + +## Next steps + + + + Understand the attestation and response-signature checks behind this flow. + + + Learn the protocol when you need to manage encryption yourself. + + diff --git a/cloud/inference-sdk/index.mdx b/cloud/inference-sdk/index.mdx new file mode 100644 index 0000000..eaf31eb --- /dev/null +++ b/cloud/inference-sdk/index.mdx @@ -0,0 +1,103 @@ +--- +title: "SDK" +icon: "braces" +description: "Use the TypeScript SDK for verified, end-to-end encrypted Chat Completions." +--- + +## Install + +The package requires Node.js 24 or later. + +```bash +npm install @nearai/inference-sdk@next +``` + +## E2EE + +End-to-end encryption (E2EE) is disabled by default in `InferenceClient`. Set `e2ee: true` to enable it for a model with supported model attestation. When enabled, the SDK verifies the required attestation evidence, encrypts supported Chat fields to a key from the verified model evidence, and decrypts the response automatically. + +```ts +import { InferenceClient } from '@nearai/inference-sdk/node'; + +const baseUrl = 'https://cloud-api.near.ai/v1'; +const model = 'z-ai/glm-5.3-flash'; +const apiKey = process.env.NEARAI_API_KEY; +if (!apiKey) throw new Error('NEARAI_API_KEY is required'); + +const client = new InferenceClient({ + baseUrl, + apiKey, + e2ee: true, +}); + +const completion = await client.chat.completions.create({ + model, + messages: [{ role: 'user', content: 'Reply with the word ok.' }], +}); + +console.log(completion.choices[0]?.message.content); +``` + +## Verification + +`InferenceClient` verifies the Gateway and every returned NEAR model-attestation candidate before sending an E2EE Chat Completion. Call `verifyResponse()` after a completion to verify its response signature with the request and response bytes captured by the client. + +```ts +const receipt = await client.verifyResponse(completion.id); +console.log(`Verified ${receipt.signatureKind} response.`); +``` + + + `verifyResponse()` is explicit. For a streaming completion, consume the entire stream first, then call it with the completion ID so the SDK can verify the exact response bytes. + + +### Set a TCB policy + +By default, the SDK accepts the Intel TCB statuses `UpToDate` and `OutOfDate`. To require `UpToDate`, apply a policy to both the Gateway and model evidence: + +```ts +const strict = { + policy: { acceptedTcbStatuses: ['UpToDate'] as const }, +}; + +const client = new InferenceClient({ + baseUrl, + apiKey, + e2ee: true, + gatewayVerification: strict, + modelVerification: strict, +}); +``` + +### Runtime behavior + +For Node, import from `@nearai/inference-sdk/node`. It verifies the Gateway TLS peer and binds later requests to the verified Gateway identity. For a browser or application proxy, import from `@nearai/inference-sdk`; browser Fetch cannot read the TLS peer certificate, so that entry point does not perform Gateway TLS binding. + +For a manual verification flow or a custom policy, see [Verification](/cloud/verification). + +## OHTTP + +Set `ohttp: true` to encapsulate Chat requests and responses with OHTTP. OHTTP uses the SDK's default Ed25519 signing algorithm; it is not compatible with `signingAlgo: 'ecdsa'`. + +```ts +const client = new InferenceClient({ + baseUrl, + apiKey, + e2ee: true, + ohttp: true, +}); +``` + +The Chat and `verifyResponse()` calls stay the same. Before sending Chat, the SDK verifies the Gateway's signed OHTTP configuration; a missing or invalid configuration blocks the request. OHTTP encapsulates the exchange to the Gateway, while E2EE encrypts supported Chat fields to the verified model key. + + + OHTTP applies only to Chat. Attestation and signature requests remain regular HTTPS. The configured endpoint or proxy must expose `/ohttp` at the same origin. Authorization and custom headers sent on the outer request, along with the client's network address, are not hidden by OHTTP. + + +## Examples + +See the [Inference SDK examples and setup instructions](https://github.com/nearai/inference-sdk/tree/main/examples) for runnable projects. + +- [InferenceClient](https://github.com/nearai/inference-sdk/blob/main/examples/example-js/gateway/client.ts): streaming and non-streaming Chat, response verification, and Gateway image provenance checks. +- [OpenAI SDK integration](https://github.com/nearai/inference-sdk/blob/main/examples/example-js/gateway/client-openai-sdk.ts): use `InferenceClient.fetch` with the OpenAI client. +- [Manual verification and E2EE](https://github.com/nearai/inference-sdk/blob/main/examples/example-js/gateway/bare.ts): work directly with attestation, encryption, and signature-verification helpers. diff --git a/docs.json b/docs.json index adeb0d7..433f920 100644 --- a/docs.json +++ b/docs.json @@ -87,7 +87,8 @@ ] } ] - } + }, + "cloud/inference-sdk/index" ] }, { @@ -134,7 +135,13 @@ { "group": "Experimental", "pages": [ - "cloud/experimental/direct-completions" + { + "group": "Direct Completions", + "root": "cloud/experimental/direct-completions", + "pages": [ + "cloud/inference-sdk/direct" + ] + } ] } ]