From a41c42cf5d24d629eea5ad1f51281900f040463f Mon Sep 17 00:00:00 2001 From: Andrew Date: Sun, 8 Mar 2026 18:51:26 -0500 Subject: [PATCH] ai: add shimmy as alternative LLM backend Shimmy is a lightweight Rust-based OpenAI-compatible API server for GGUF models. It supports CPU and GPU inference (CUDA, Vulkan, OpenCL, MLX). - Add shimmy component alongside existing ollama - OpenWebUI and n8n can use either backend (ollama preferred, shimmy fallback) - CPU-only by default, auto-detects GPU when enabled - Health checks on /health endpoint Users can choose between ollama (more features) or shimmy (lighter weight). --- Pulumi.yaml | 14 ++++++++ components/ai/AI.md | 10 ++++-- components/ai/index.ts | 27 +++++++++----- components/ai/shimmy/shimmy.md | 66 ++++++++++++++++++++++++++++++++++ components/ai/shimmy/shimmy.ts | 59 ++++++++++++++++++++++++++++++ src/config.ts | 11 +++++- 6 files changed, 174 insertions(+), 13 deletions(-) create mode 100644 components/ai/shimmy/shimmy.md create mode 100644 components/ai/shimmy/shimmy.ts diff --git a/Pulumi.yaml b/Pulumi.yaml index 1004f41..d46761b 100644 --- a/Pulumi.yaml +++ b/Pulumi.yaml @@ -162,6 +162,20 @@ config: # LLM parameter tuning ollama:OLLAMA_CONTEXT_LENGTH: '16536' + shimmy:enabled: false + shimmy:hostname: shimmy + shimmy:storageSize: 50Gi + shimmy:storageOnly: false + shimmy:storageClass: longhorn-gpu + # shimmy:fromVolume: shimmy + # Set to 'nvidia' or 'amd' to enable GPU support, Use '' for CPU-only mode + shimmy:gpu: nvidia + # Override GPU version for AMD ROCm compatibility (shimmy:gpu=amd) + # shimmy:HSA_OVERRIDE_GFX_VERSION: "11.0.2" + # shimmy:HCC_AMDGPU_TARGETS: "gfx1103" + # Override shimmy image tag (default: latest) + # shimmy:image: latest + automatic1111:enabled: false automatic1111:hostname: 'automatic1111' automatic1111:image: universonic/stable-diffusion-webui:full diff --git a/components/ai/AI.md b/components/ai/AI.md index e16da4f..23b2cf5 100644 --- a/components/ai/AI.md +++ b/components/ai/AI.md @@ -14,17 +14,21 @@ pulumi config set nvidia-gpu-operator:enabled true pulumi config set cert-manager:enabled true pulumi config set amd-gpu-operator:enabled true -# Install Ollama and OpenWebUI -pulumi config set ollama:enabled true # uses any NVidia node by default +# Install Ollama or Shimmy (both provide OpenAI-compatible LLM API) +pulumi config set ollama:enabled true # uses any NVidia node by default +# OR +pulumi config set shimmy:enabled true # lighter weight alternative + pulumi config set open-webui:enabled true pulumi up ``` -> Note: GPU memory is limited so you might encounter out-of-memory errors when loading new models. Only enable components you need. You can check processes using GPU with `nvidia-smi` or `nvtop`. Check Ollama section on how to stop models to free up memory. +> Note: GPU memory is limited so you might encounter out-of-memory errors when loading new models. Only enable components you need. You can check processes using GPU with `nvidia-smi` or `nvtop`. ## Components - [Ollama](./ollama/ollama.md) - Run large language models locally with GPU support. +- [Shimmy](./shimmy/shimmy.md) - Lightweight OpenAI-compatible API server for GGUF models (Rust-based). - [Open-WebUI](./open-webui/open-webui.md) - User-friendly web interface for LLMs with RAG support. - [InvokeAI](./invokeai/invokeai.md) - Professional-grade generative AI toolkit for image creation. - [N8n](./n8n/n8n.md) - Workflow automation tool with native AI integration. diff --git a/components/ai/index.ts b/components/ai/index.ts index d8cd1db..3533176 100644 --- a/components/ai/index.ts +++ b/components/ai/index.ts @@ -5,11 +5,13 @@ import { InvokeAi } from './invokeai/invokeai'; import { KubeAi } from './kubeai/kubeai'; import { N8n } from './n8n/n8n'; import { Ollama } from './ollama/ollama'; +import { Shimmy } from './shimmy/shimmy'; import { OpenWebUI } from './open-webui/open-webui'; import { SDNext } from './sdnext/sdnext'; export class AIModule extends pulumi.ComponentResource { private readonly ollama?: Ollama; + private readonly shimmy?: Shimmy; private readonly kubeAI?: KubeAi; private readonly openWebUI?: OpenWebUI; private readonly automatic1111?: Automatic1111; @@ -24,6 +26,7 @@ export class AIModule extends pulumi.ComponentResource { ...this.invokeAi?.app.network.endpoints, kubeai: this.kubeAI?.serviceUrl, ollama: this.ollama?.endpointUrl, + shimmy: this.shimmy?.endpointUrl, 'open-webui': this.openWebUI?.endpointUrl, ...this.sdnext?.app.network.endpoints, ...this.n8n?.app.network.endpoints, @@ -33,6 +36,7 @@ export class AIModule extends pulumi.ComponentResource { ...this.invokeAi?.app.network.clusterEndpoints, kubeai: this.kubeAI?.serviceUrl, ollama: this.ollama?.serviceUrl, + shimmy: this.shimmy?.serviceUrl, ...this.sdnext?.app.network.clusterEndpoints, ...this.n8n?.app.network.clusterEndpoints, }, @@ -52,6 +56,10 @@ export class AIModule extends pulumi.ComponentResource { this.ollama = new Ollama('ollama', { parent: this }); } + if (config.isEnabled('shimmy')) { + this.shimmy = new Shimmy('shimmy', { parent: this }); + } + if (config.isEnabled('automatic1111')) { this.automatic1111 = new Automatic1111('automatic1111', { parent: this }); } @@ -65,10 +73,11 @@ export class AIModule extends pulumi.ComponentResource { } if (config.isEnabled('open-webui')) { + const llmUrl = this.ollama?.serviceUrl ?? this.shimmy?.serviceUrl; this.openWebUI = new OpenWebUI( 'open-webui', { - ollamaUrl: this.ollama?.serviceUrl, + ollamaUrl: llmUrl, openAiUrl: this.kubeAI?.serviceUrl, automatic1111Url: this.sdnext?.app.network.clusterEndpoints.sdnext ?? @@ -76,9 +85,12 @@ export class AIModule extends pulumi.ComponentResource { }, { parent: this, - dependsOn: [this.ollama, this.kubeAI, this.automatic1111].filter( - x => x !== undefined, - ), + dependsOn: [ + this.ollama, + this.shimmy, + this.kubeAI, + this.automatic1111, + ].filter(x => x !== undefined), }, ); } @@ -88,11 +100,8 @@ export class AIModule extends pulumi.ComponentResource { } if (config.isEnabled('n8n')) { - this.n8n = new N8n( - 'n8n', - { ollamaUrl: this.ollama?.serviceUrl }, - { parent: this }, - ); + const llmUrl = this.ollama?.serviceUrl ?? this.shimmy?.serviceUrl; + this.n8n = new N8n('n8n', { ollamaUrl: llmUrl }, { parent: this }); } } } diff --git a/components/ai/shimmy/shimmy.md b/components/ai/shimmy/shimmy.md new file mode 100644 index 0000000..a8622b8 --- /dev/null +++ b/components/ai/shimmy/shimmy.md @@ -0,0 +1,66 @@ +# Shimmy + +| | | +| ------------ | ---------------------------------------------- | +| Homepage | https://github.com/Michael-A-Kuykendall/shimmy | +| Source | https://github.com/Michael-A-Kuykendall/shimmy | +| Docker Image | ghcr.io/michael-a-kuykendall/shimmy | +| Endpoints | `https://shimmy./` | + +Shimmy is an OpenAI-compatible API server for GGUF models written in Rust. It auto-discovers models and supports CPU and GPU inference (CUDA, Vulkan, OpenCL, MLX). + +```sh +# (Optional) Use CPU-only mode (default when no GPU operator installed) +pulumi config set shimmy:gpu "" + +# (Recommended) Or enable GPU support (requires installing the appropriate operator first) +# pulumi config set nvidia-gpu-operator:enabled true +# pulumi config set shimmy:gpu nvidia +# or +# pulumi config set amd-gpu-operator:enabled true +# pulumi config set shimmy:gpu amd + +# (Optional) Increase volume size for bigger models (default: 50Gi) +pulumi config set shimmy:storageSize "100Gi" + +# (Optional) Override shimmy image tag (default: latest) +pulumi config set shimmy:image "latest" + +# Enable Shimmy +pulumi config set shimmy:enabled true +pulumi up +``` + +Models will be stored on local Longhorn volume with no replication across nodes. + +## Usage + +Shimmy provides an OpenAI-compatible API on port 11434. Configure your tools to use the endpoint: + +```sh +# Get the endpoint +pulumi stack output --json | jq -r '.ai.endpoints.shimmy' + +# For OpenAI-compatible tools, append /v1/ to the URL +# Example: https://shimmy.yourdomain.com/v1/chat/completions +``` + +## Models + +Shimmy auto-discovers GGUF models from the `/app/models` directory. Add models by: + +1. Downloading GGUF files to the models volume +2. Models are automatically detected and available via the API + +Find GGUF models at: + +- Hugging Face: https://huggingface.co/models?library=gguf +- TheBloke quantizations: https://huggingface.co/TheBloke + +## GPU Support + +- **CPU-only**: Set `shimmy:gpu ""` or leave unset (default) +- **NVIDIA**: Install nvidia-gpu-operator, set `shimmy:gpu nvidia` +- **AMD**: Install amd-gpu-operator, set `shimmy:gpu amd` + +Shimmy auto-detects GPU backends (CUDA, Vulkan, OpenCL, MLX). diff --git a/components/ai/shimmy/shimmy.ts b/components/ai/shimmy/shimmy.ts new file mode 100644 index 0000000..d314b27 --- /dev/null +++ b/components/ai/shimmy/shimmy.ts @@ -0,0 +1,59 @@ +import * as pulumi from '@pulumi/pulumi'; +import { Application } from '@orangelab/application'; +import { config } from '@orangelab/config'; + +export class Shimmy extends pulumi.ComponentResource { + public readonly endpointUrl?: string; + public readonly serviceUrl?: string; + + private readonly app: Application; + + constructor( + private name: string, + opts?: pulumi.ResourceOptions, + ) { + super('orangelab:ai:Shimmy', name, {}, opts); + + const hostname = config.require(name, 'hostname'); + + this.app = new Application(this, name).addStorage(); + + if (this.app.storageOnly) return; + + const httpEndpointInfo = this.app.network.getHttpEndpointInfo(); + const amdGpu = this.app.gpu === 'amd'; + const gfxVersion = config.get(this.name, 'HSA_OVERRIDE_GFX_VERSION'); + const amdTargets = config.get(this.name, 'HCC_AMDGPU_TARGETS'); + + const imageTag = config.get(this.name, 'image') ?? 'latest'; + const commandArgs = ['serve', '--bind', '0.0.0.0:11434']; + if (!this.app.gpu) { + commandArgs.push('--gpu-backend', 'cpu'); + } else { + commandArgs.push('--gpu-backend', 'auto'); + } + + this.app.addDeployment({ + ports: [{ name: 'http', port: 11434 }], + volumeMounts: [{ mountPath: '/app/models' }], + commandArgs, + image: `ghcr.io/michael-a-kuykendall/shimmy:${imageTag}`, + resources: { + requests: { memory: '512Mi' }, + limits: { memory: '2Gi' }, + }, + env: { + RUST_LOG: this.app.debug ? 'debug' : 'info', + SHIMMY_PORT: '11434', + SHIMMY_HOST: '0.0.0.0', + SHIMMY_BASE_GGUF: '/app/models', + HSA_OVERRIDE_GFX_VERSION: amdGpu && gfxVersion ? gfxVersion : undefined, + HCC_AMDGPU_TARGETS: amdGpu && amdTargets ? amdTargets : undefined, + }, + healthChecks: true, + }); + + this.endpointUrl = httpEndpointInfo.url; + this.serviceUrl = `http://${hostname}.shimmy:11434`; + } +} diff --git a/src/config.ts b/src/config.ts index bb3d2a5..fe7a870 100644 --- a/src/config.ts +++ b/src/config.ts @@ -2,7 +2,16 @@ import * as pulumi from '@pulumi/pulumi'; const moduleDependencies: Record = { - ai: ['automatic1111', 'invokeai', 'kubeai', 'n8n', 'ollama', 'open-webui', 'sdnext'], + ai: [ + 'automatic1111', + 'invokeai', + 'kubeai', + 'n8n', + 'ollama', + 'shimmy', + 'open-webui', + 'sdnext', + ], bitcoin: ['bitcoin-core', 'bitcoin-knots', 'electrs', 'mempool'], data: ['cloudnative-pg', 'mariadb-operator'], dev: ['debug'],