Preview

Preview Feature — This feature is currently in preview and under active development. APIs and functionality may change. We recommend testing thoroughly before using in production.

Agent Resilience

Build reliable AI agents by leveraging Ductape's existing resilience infrastructure. Agents can use product-level healthchecks, quotas, and fallbacks to create production-ready agents that gracefully handle failures, distribute load across providers, and maintain high availability.

Overview

Agent resilience in Ductape works by referencing existing resilience configurations defined at the product level:

Pattern	Purpose	Use Case
Healthchecks	Monitor provider availability	Detect outages, track latency
Quotas	Weighted load distribution	Balance traffic across providers
Fallbacks	Sequential failover	Automatic failover on errors

Important

Agents do not define their own healthchecks, quotas, or fallbacks. Instead, they reference configurations that already exist at the product level. This ensures consistency across your entire product and avoids duplication.

---

Setting Up Resilience

Step 1: Define Resilience at the Product Level

First, create your resilience configurations using the Ductape resilience API:

TypeScript

import Ductape from '@ductape/sdk';

const ductape = new Ductape({...});

// Create a healthcheck
await ductape.resilience.healthcheck.create('my-product', {
  tag: 'anthropic-health',
  name: 'Anthropic API Health',
  probe: {
    type: 'app',
    app: 'anthropic-app',
    event: 'health-check',
  },
  interval: 30000, // Check every 30 seconds
  retries: 2,
  envs: [
    { slug: 'production' },
    { slug: 'staging' },
  ],
  onFailure: {
    notifications: [{
      notification: 'alerts',
      message: 'llm-provider-down',
      channels: {
        email: { recipients: ['oncall@company.com'] },
      },
    }],
  },
});

// Create a quota for load balancing
await ductape.resilience.quota.create('my-product', {
  tag: 'llm-quota',
  name: 'LLM Provider Quota',
  input: {
    prompt: { type: 'string', required: true },
  },
  options: [
    {
      provider: 'anthropic-primary',
      app: 'anthropic-app',
      event: 'generate',
      quota: 70, // 70% of traffic
      healthcheck: 'anthropic-health',
      retries: 2,
    },
    {
      provider: 'openai-secondary',
      app: 'openai-app',
      event: 'generate',
      quota: 30, // 30% of traffic
      healthcheck: 'openai-health',
      retries: 2,
    },
  ],
});

// Create a fallback chain
await ductape.resilience.fallback.create('my-product', {
  tag: 'llm-fallback',
  name: 'LLM Fallback Chain',
  input: {
    prompt: { type: 'string', required: true },
  },
  options: [
    {
      provider: 'anthropic-primary',
      app: 'anthropic-app',
      event: 'generate',
      healthcheck: 'anthropic-health',
      retries: 3,
    },
    {
      provider: 'openai-fallback',
      app: 'openai-app',
      event: 'generate',
      healthcheck: 'openai-health',
      retries: 2,
    },
  ],
});

Java

import app.ductape.sdk.Ductape;
import app.ductape.sdk.core.EnvType;
import app.ductape.sdk.core.RequestContext;

Map<String, Object> ductape = new Ductape(Map.of(...));

// Create a healthcheck
ductape.resilience.healthcheck.create('my-product', Map.of(
  "tag", "anthropic-health",
  "name", "Anthropic API Health",
  probe: Map.of(
    "type", "app",
    "app", "anthropic-app",
    "event", "health-check"
  ),
  "interval", 30000, // Check every 30 seconds
  "retries", 2,
  envs: [
    Map.of( "slug", "production" ),
    Map.of( "slug", "staging" ),
  ],
  onFailure: Map.of(
    notifications: [Map.of(
      "notification", "alerts",
      "message", "llm-provider-down",
      channels: Map.of(
        email: Map.of( recipients: ['oncall@company.com'] )
      )
    )]
  )
));

// Create a quota for load balancing
ductape.resilience.quota.create('my-product', Map.of(
  "tag", "llm-quota",
  "name", "LLM Provider Quota",
  input: Map.of(
    prompt: Map.of( "type", "string", "required", true )
  ),
  options: [
    Map.of(
      "provider", "anthropic-primary",
      "app", "anthropic-app",
      "event", "generate",
      "quota", 70, // 70% of traffic
      "healthcheck", "anthropic-health",
      "retries", 2
    ),
    Map.of(
      "provider", "openai-secondary",
      "app", "openai-app",
      "event", "generate",
      "quota", 30, // 30% of traffic
      "healthcheck", "openai-health",
      "retries", 2
    ),
  ]
));

// Create a fallback chain
ductape.resilience.fallback.create('my-product', Map.of(
  "tag", "llm-fallback",
  "name", "LLM Fallback Chain",
  input: Map.of(
    prompt: Map.of( "type", "string", "required", true )
  ),
  options: [
    Map.of(
      "provider", "anthropic-primary",
      "app", "anthropic-app",
      "event", "generate",
      "healthcheck", "anthropic-health",
      "retries", 3
    ),
    Map.of(
      "provider", "openai-fallback",
      "app", "openai-app",
      "event", "generate",
      "healthcheck", "openai-health",
      "retries", 2
    ),
  ]
));

Go

import (
	"context"
	"github.com/ductape/ductape/sdk/go/core"
	ductapesdk "github.com/ductape/ductape/sdk/go/ductape"
)

ductape := new Ductape({...});

// Create a healthcheck
client.resilience.healthcheck.create('my-product', {
  "tag": "anthropic-health",
  "name": "Anthropic API Health",
  probe: {
    "type": "app",
    "app": "anthropic-app",
    "event": "health-check",
  },
  "interval": 30000, // Check every 30 seconds
  "retries": 2,
  envs: [
    { "slug": "production" },
    { "slug": "staging" },
  ],
  onFailure: {
    notifications: [{
      "notification": "alerts",
      "message": "llm-provider-down",
      channels: {
        email: { recipients: ['oncall@company.com'] },
      },
    }],
  },
});

// Create a quota for load balancing
client.resilience.quota.create('my-product', {
  "tag": "llm-quota",
  "name": "LLM Provider Quota",
  input: {
    prompt: { "type": "string", "required": true },
  },
  options: [
    {
      "provider": "anthropic-primary",
      "app": "anthropic-app",
      "event": "generate",
      "quota": 70, // 70% of traffic
      "healthcheck": "anthropic-health",
      "retries": 2,
    },
    {
      "provider": "openai-secondary",
      "app": "openai-app",
      "event": "generate",
      "quota": 30, // 30% of traffic
      "healthcheck": "openai-health",
      "retries": 2,
    },
  ],
});

// Create a fallback chain
client.resilience.fallback.create('my-product', {
  "tag": "llm-fallback",
  "name": "LLM Fallback Chain",
  input: {
    prompt: { "type": "string", "required": true },
  },
  options: [
    {
      "provider": "anthropic-primary",
      "app": "anthropic-app",
      "event": "generate",
      "healthcheck": "anthropic-health",
      "retries": 3,
    },
    {
      "provider": "openai-fallback",
      "app": "openai-app",
      "event": "generate",
      "healthcheck": "openai-health",
      "retries": 2,
    },
  ],
});

.NET

using Ductape.Sdk;
using Ductape.Sdk.Core;

var ductape = new Ductape({...});

// Create a healthcheck
await ductape.resilience.healthcheck.create('my-product', {
  ["tag"] = "anthropic-health",
  ["name"] = "Anthropic API Health",
  probe: {
    ["type"] = "app",
    ["app"] = "anthropic-app",
    ["event"] = "health-check",
  },
  ["interval"] = 30000, // Check every 30 seconds
  ["retries"] = 2,
  envs: [
    { ["slug"] = "production" },
    { ["slug"] = "staging" },
  ],
  onFailure: {
    notifications: [{
      ["notification"] = "alerts",
      ["message"] = "llm-provider-down",
      channels: {
        email: { recipients: ['oncall@company.com'] },
      },
    }],
  },
});

// Create a quota for load balancing
await ductape.resilience.quota.create('my-product', {
  ["tag"] = "llm-quota",
  ["name"] = "LLM Provider Quota",
  input: {
    prompt: { ["type"] = "string", ["required"] = true },
  },
  options: [
    {
      ["provider"] = "anthropic-primary",
      ["app"] = "anthropic-app",
      ["event"] = "generate",
      ["quota"] = 70, // 70% of traffic
      ["healthcheck"] = "anthropic-health",
      ["retries"] = 2,
    },
    {
      ["provider"] = "openai-secondary",
      ["app"] = "openai-app",
      ["event"] = "generate",
      ["quota"] = 30, // 30% of traffic
      ["healthcheck"] = "openai-health",
      ["retries"] = 2,
    },
  ],
});

// Create a fallback chain
await ductape.resilience.fallback.create('my-product', {
  ["tag"] = "llm-fallback",
  ["name"] = "LLM Fallback Chain",
  input: {
    prompt: { ["type"] = "string", ["required"] = true },
  },
  options: [
    {
      ["provider"] = "anthropic-primary",
      ["app"] = "anthropic-app",
      ["event"] = "generate",
      ["healthcheck"] = "anthropic-health",
      ["retries"] = 3,
    },
    {
      ["provider"] = "openai-fallback",
      ["app"] = "openai-app",
      ["event"] = "generate",
      ["healthcheck"] = "openai-health",
      ["retries"] = 2,
    },
  ],
});

Step 2: Reference Resilience in Your Agent

Now reference these configurations in your agent definition:

TypeScript

const agent = await ductape.agents.define({
  tag: 'resilient-agent',
  name: 'Resilient Agent',
  model: 'claude-model',
  systemPrompt: 'You are a helpful assistant.',
  tools: [...],

  // Reference existing resilience configurations
  resilience: {
    defaults: {
      healthcheck: 'anthropic-health', // Check health before operations
      fallback: 'llm-fallback',        // Use fallback chain by default
    },
  },
});

Java

Map<String, Object> agent = ductape.agents.define(Map.of(
  "tag", "resilient-agent",
  "name", "Resilient Agent",
  "model", "claude-model",
  "systemPrompt", "You are a helpful assistant.",
  tools: [...],

  // Reference existing resilience configurations
  resilience: Map.of(
    defaults: Map.of(
      "healthcheck", "anthropic-health", // Check health before operations
      "fallback", "llm-fallback",        // Use fallback chain by default
    )
  )
));

Go

agent := client.agents.define({
  "tag": "resilient-agent",
  "name": "Resilient Agent",
  "model": "claude-model",
  "systemPrompt": "You are a helpful assistant.",
  tools: [...],

  // Reference existing resilience configurations
  resilience: {
    defaults: {
      "healthcheck": "anthropic-health", // Check health before operations
      "fallback": "llm-fallback",        // Use fallback chain by default
    },
  },
});

.NET

var agent = await ductape.agents.define({
  ["tag"] = "resilient-agent",
  ["name"] = "Resilient Agent",
  ["model"] = "claude-model",
  ["systemPrompt"] = "You are a helpful assistant.",
  tools: [...],

  // Reference existing resilience configurations
  resilience: {
    defaults: {
      ["healthcheck"] = "anthropic-health", // Check health before operations
      ["fallback"] = "llm-fallback",        // Use fallback chain by default
    },
  },
});

---

Using Resilience in Tools

Checking Health Status

TypeScript

{
  tag: 'smart-query',
  description: 'Query with health-aware provider selection',
  parameters: {...},
  handler: async (ctx, params) => {
    // Check provider health before making a decision
    const healthStatus = await ctx.resilience.healthcheck.status();

    if (healthStatus['anthropic-health']?.status === 'healthy') {
      ctx.log.info('Using Anthropic provider');
      // Use primary provider
    } else if (healthStatus['openai-health']?.status === 'healthy') {
      ctx.log.info('Falling back to OpenAI');
      // Use fallback provider
    } else {
      throw new Error('All LLM providers are unavailable');
    }

    // ... perform the query
  },
}

Java

Map.of(
  "tag", "smart-query",
  "description", "Query with health-aware provider selection",
  parameters: Map.of(...),
  handler: async (ctx, params) => Map.of(
    // Check provider health before making a decision
    Map<String, Object> healthStatus = ctx.resilience.healthcheck.status();

    if (healthStatus['anthropic-health']?.status === 'healthy') Map.of(
      ctx.log.info('Using Anthropic provider');
      // Use primary provider
    ) else if (healthStatus['openai-health']?.status === 'healthy') Map.of(
      ctx.log.info('Falling back to OpenAI');
      // Use fallback provider
    ) else Map.of(
      throw new Error('All LLM providers are unavailable');
    )

    // ... perform the query
  )
)

Go

import "context"

{
  "tag": "smart-query",
  "description": "Query with health-aware provider selection",
  parameters: {...},
  handler: async (ctx, params) => {
    // Check provider health before making a decision
    healthStatus := ctx.resilience.healthcheck.status();

    if (healthStatus['anthropic-health']?.status === 'healthy') {
      ctx.log.info('Using Anthropic provider');
      // Use primary provider
    } else if (healthStatus['openai-health']?.status === 'healthy') {
      ctx.log.info('Falling back to OpenAI');
      // Use fallback provider
    } else {
      throw new Error('All LLM providers are unavailable');
    }

    // ... perform the query
  },
}

.NET

{
  ["tag"] = "smart-query",
  ["description"] = "Query with health-aware provider selection",
  parameters: {...},
  handler: async (ctx, params) => {
    // Check provider health before making a decision
    var healthStatus = await ctx.resilience.healthcheck.status();

    if (healthStatus['anthropic-health']?.status === 'healthy') {
      ctx.log.info('Using Anthropic provider');
      // Use primary provider
    } else if (healthStatus['openai-health']?.status === 'healthy') {
      ctx.log.info('Falling back to OpenAI');
      // Use fallback provider
    } else {
      throw new Error('All LLM providers are unavailable');
    }

    // ... perform the query
  },
}

Running Through Quotas

TypeScript

{
  tag: 'analyze-text',
  description: 'Analyze text using load-balanced LLM providers',
  parameters: {
    text: { type: 'string', description: 'Text to analyze', required: true },
  },
  handler: async (ctx, params) => {
    // Run through quota - automatically selects provider based on weight
    const result = await ctx.resilience.quota.run({
      tag: 'llm-quota',
      input: {
        prompt: `Analyze the following text: ${params.text}`,
      },
    });

    return result;
  },
}

Java

Map.of(
  "tag", "analyze-text",
  "description", "Analyze text using load-balanced LLM providers",
  parameters: Map.of(
    text: Map.of( "type", "string", "description", "Text to analyze", "required", true )
  ),
  handler: async (ctx, params) => Map.of(
    // Run through quota - automatically selects provider based on weight
    Map<String, Object> result = ctx.resilience.quota.run(Map.of(
      "tag", "llm-quota",
      input: Map.of(
        prompt: `Analyze the following text: $Map.of(params.text)`
      )
    ));

    return result;
  )
)

Go

import "context"

{
  "tag": "analyze-text",
  "description": "Analyze text using load-balanced LLM providers",
  parameters: {
    text: { "type": "string", "description": "Text to analyze", "required": true },
  },
  handler: async (ctx, params) => {
    // Run through quota - automatically selects provider based on weight
    result := ctx.resilience.quota.run({
      "tag": "llm-quota",
      input: {
        prompt: `Analyze the following text: ${params.text}`,
      },
    });

    return result;
  },
}

.NET

{
  ["tag"] = "analyze-text",
  ["description"] = "Analyze text using load-balanced LLM providers",
  parameters: {
    text: { ["type"] = "string", ["description"] = "Text to analyze", ["required"] = true },
  },
  handler: async (ctx, params) => {
    // Run through quota - automatically selects provider based on weight
    var result = await ctx.resilience.quota.run({
      ["tag"] = "llm-quota",
      input: {
        prompt: `Analyze the following text: ${params.text}`,
      },
    });

    return result;
  },
}

Running Through Fallbacks

TypeScript

{
  tag: 'generate-response',
  description: 'Generate response with automatic failover',
  parameters: {
    prompt: { type: 'string', description: 'User prompt', required: true },
  },
  handler: async (ctx, params) => {
    // Run through fallback chain - automatically fails over on errors
    const result = await ctx.resilience.fallback.run({
      tag: 'llm-fallback',
      input: {
        prompt: params.prompt,
      },
    });

    return result;
  },
}

Java

Map.of(
  "tag", "generate-response",
  "description", "Generate response with automatic failover",
  parameters: Map.of(
    prompt: Map.of( "type", "string", "description", "User prompt", "required", true )
  ),
  handler: async (ctx, params) => Map.of(
    // Run through fallback chain - automatically fails over on errors
    Map<String, Object> result = ctx.resilience.fallback.run(Map.of(
      "tag", "llm-fallback",
      input: Map.of(
        prompt: params.prompt
      )
    ));

    return result;
  )
)

Go

import "context"

{
  "tag": "generate-response",
  "description": "Generate response with automatic failover",
  parameters: {
    prompt: { "type": "string", "description": "User prompt", "required": true },
  },
  handler: async (ctx, params) => {
    // Run through fallback chain - automatically fails over on errors
    result := ctx.resilience.fallback.run({
      "tag": "llm-fallback",
      input: {
        prompt: params.prompt,
      },
    });

    return result;
  },
}

.NET

{
  ["tag"] = "generate-response",
  ["description"] = "Generate response with automatic failover",
  parameters: {
    prompt: { ["type"] = "string", ["description"] = "User prompt", ["required"] = true },
  },
  handler: async (ctx, params) => {
    // Run through fallback chain - automatically fails over on errors
    var result = await ctx.resilience.fallback.run({
      ["tag"] = "llm-fallback",
      input: {
        prompt: params.prompt,
      },
    });

    return result;
  },
}

---

Tool-Level Resilience

You can configure resilience at the individual tool level for fine-grained control:

TypeScript

const agent = await ductape.agents.define({
  tag: 'granular-agent',
  name: 'Granular Resilience Agent',
  model: 'default-model',
  systemPrompt: 'You are a helpful assistant.',

  tools: [
    {
      tag: 'critical-operation',
      description: 'A critical operation that needs fallback protection',
      parameters: {...},
      handler: async (ctx, params) => {...},
      // Tool-specific resilience (overrides defaults)
      resilience: {
        fallback: 'critical-fallback',
        healthcheck: 'critical-service-health',
      },
    },
    {
      tag: 'load-balanced-operation',
      description: 'An operation that should be load balanced',
      parameters: {...},
      handler: async (ctx, params) => {...},
      resilience: {
        quota: 'standard-quota',
      },
    },
  ],

  resilience: {
    // Default resilience for tools without explicit config
    defaults: {
      healthcheck: 'default-health',
      quota: 'default-quota',
    },

    // Override resilience for specific tools
    toolOverrides: {
      'special-tool': {
        fallback: 'special-fallback',
        healthcheck: 'special-health',
      },
    },
  },
});

Java

Map<String, Object> agent = ductape.agents.define(Map.of(
  "tag", "granular-agent",
  "name", "Granular Resilience Agent",
  "model", "default-model",
  "systemPrompt", "You are a helpful assistant.",

  tools: [
    Map.of(
      "tag", "critical-operation",
      "description", "A critical operation that needs fallback protection",
      parameters: Map.of(...),
      handler: async (ctx, params) => Map.of(...),
      // Tool-specific resilience (overrides defaults)
      resilience: Map.of(
        "fallback", "critical-fallback",
        "healthcheck", "critical-service-health"
      )
    ),
    Map.of(
      "tag", "load-balanced-operation",
      "description", "An operation that should be load balanced",
      parameters: Map.of(...),
      handler: async (ctx, params) => Map.of(...),
      resilience: Map.of(
        "quota", "standard-quota"
      )
    ),
  ],

  resilience: Map.of(
    // Default resilience for tools without explicit config
    defaults: Map.of(
      "healthcheck", "default-health",
      "quota", "default-quota"
    ),

    // Override resilience for specific tools
    toolOverrides: Map.of(
      'special-tool': Map.of(
        "fallback", "special-fallback",
        "healthcheck", "special-health"
      )
    )
  )
));

Go

import "context"

agent := client.agents.define({
  "tag": "granular-agent",
  "name": "Granular Resilience Agent",
  "model": "default-model",
  "systemPrompt": "You are a helpful assistant.",

  tools: [
    {
      "tag": "critical-operation",
      "description": "A critical operation that needs fallback protection",
      parameters: {...},
      handler: async (ctx, params) => {...},
      // Tool-specific resilience (overrides defaults)
      resilience: {
        "fallback": "critical-fallback",
        "healthcheck": "critical-service-health",
      },
    },
    {
      "tag": "load-balanced-operation",
      "description": "An operation that should be load balanced",
      parameters: {...},
      handler: async (ctx, params) => {...},
      resilience: {
        "quota": "standard-quota",
      },
    },
  ],

  resilience: {
    // Default resilience for tools without explicit config
    defaults: {
      "healthcheck": "default-health",
      "quota": "default-quota",
    },

    // Override resilience for specific tools
    toolOverrides: {
      'special-tool': {
        "fallback": "special-fallback",
        "healthcheck": "special-health",
      },
    },
  },
});

.NET

var agent = await ductape.agents.define({
  ["tag"] = "granular-agent",
  ["name"] = "Granular Resilience Agent",
  ["model"] = "default-model",
  ["systemPrompt"] = "You are a helpful assistant.",

  tools: [
    {
      ["tag"] = "critical-operation",
      ["description"] = "A critical operation that needs fallback protection",
      parameters: {...},
      handler: async (ctx, params) => {...},
      // Tool-specific resilience (overrides defaults)
      resilience: {
        ["fallback"] = "critical-fallback",
        ["healthcheck"] = "critical-service-health",
      },
    },
    {
      ["tag"] = "load-balanced-operation",
      ["description"] = "An operation that should be load balanced",
      parameters: {...},
      handler: async (ctx, params) => {...},
      resilience: {
        ["quota"] = "standard-quota",
      },
    },
  ],

  resilience: {
    // Default resilience for tools without explicit config
    defaults: {
      ["healthcheck"] = "default-health",
      ["quota"] = "default-quota",
    },

    // Override resilience for specific tools
    toolOverrides: {
      'special-tool': {
        ["fallback"] = "special-fallback",
        ["healthcheck"] = "special-health",
      },
    },
  },
});

---

Resilience Context API

The ctx.resilience object in tool handlers provides these namespaces:

ctx.resilience.quota

quota.run<T>(options)

Run an operation through an existing quota for weighted load distribution.

TypeScript

const result = await ctx.resilience.quota.run<ResponseType>({
  tag: 'quota-tag',               // Tag of existing quota
  input: { /* operation input */ },
  session: { tag: 'session-tag', token: 'token' }, // Optional
});

Java

Map<String, Object> result = ctx.resilience.quota.run<ResponseType>(Map.of(
  "tag", "quota-tag",               // Tag of existing quota
  input: Map.of( /* operation input */ ),
  session: Map.of( "tag", "session-tag", "token", "token" ), // Optional
));

Go

result := ctx.resilience.quota.run<ResponseType>({
  "tag": "quota-tag",               // Tag of existing quota
  input: { /* operation input */ },
  session: { "tag": "session-tag", "token": "token" }, // Optional
});

.NET

var result = await ctx.resilience.quota.run<ResponseType>({
  ["tag"] = "quota-tag",               // Tag of existing quota
  input: { /* operation input */ },
  session: { ["tag"] = "session-tag", ["token"] = "token" }, // Optional
});

quota.status(options)

Get current status and usage of an existing quota.

TypeScript

const status = await ctx.resilience.quota.status({
  tag: 'quota-tag',
});

// Returns:
// {
//   tag: 'quota-tag',
//   totalQuota: 1000,
//   usedQuota: 450,
//   remainingQuota: 550,
//   providers: [
//     { name: 'primary', weight: 70, uses: 315, status: 'available' },
//     { name: 'secondary', weight: 30, uses: 135, status: 'available' },
//   ]
// }

Java

Map<String, Object> status = ctx.resilience.quota.status(Map.of(
  "tag", "quota-tag"
));

// Returns:
// Map.of(
//   "tag", "quota-tag",
//   "totalQuota", 1000,
//   "usedQuota", 450,
//   "remainingQuota", 550,
//   providers: [
//     Map.of( "name", "primary", "weight", 70, "uses", 315, "status", "available" ),
//     Map.of( "name", "secondary", "weight", 30, "uses", 135, "status", "available" ),
//   ]
// )

Go

status := ctx.resilience.quota.status({
  "tag": "quota-tag",
});

// Returns:
// {
//   "tag": "quota-tag",
//   "totalQuota": 1000,
//   "usedQuota": 450,
//   "remainingQuota": 550,
//   providers: [
//     { "name": "primary", "weight": 70, "uses": 315, "status": "available" },
//     { "name": "secondary", "weight": 30, "uses": 135, "status": "available" },
//   ]
// }

.NET

var status = await ctx.resilience.quota.status({
  ["tag"] = "quota-tag",
});

// Returns:
// {
//   ["tag"] = "quota-tag",
//   ["totalQuota"] = 1000,
//   ["usedQuota"] = 450,
//   ["remainingQuota"] = 550,
//   providers: [
//     { ["name"] = "primary", ["weight"] = 70, ["uses"] = 315, ["status"] = "available" },
//     { ["name"] = "secondary", ["weight"] = 30, ["uses"] = 135, ["status"] = "available" },
//   ]
// }

ctx.resilience.fallback

fallback.run<T>(options)

Run an operation through an existing fallback chain for sequential failover.

TypeScript

const result = await ctx.resilience.fallback.run<ResponseType>({
  tag: 'fallback-tag',            // Tag of existing fallback
  input: { /* operation input */ },
  session: { tag: 'session-tag', token: 'token' }, // Optional
});

Java

Map<String, Object> result = ctx.resilience.fallback.run<ResponseType>(Map.of(
  "tag", "fallback-tag",            // Tag of existing fallback
  input: Map.of( /* operation input */ ),
  session: Map.of( "tag", "session-tag", "token", "token" ), // Optional
));

Go

result := ctx.resilience.fallback.run<ResponseType>({
  "tag": "fallback-tag",            // Tag of existing fallback
  input: { /* operation input */ },
  session: { "tag": "session-tag", "token": "token" }, // Optional
});

.NET

var result = await ctx.resilience.fallback.run<ResponseType>({
  ["tag"] = "fallback-tag",            // Tag of existing fallback
  input: { /* operation input */ },
  session: { ["tag"] = "session-tag", ["token"] = "token" }, // Optional
});

ctx.resilience.healthcheck

healthcheck.check(options)

Check the health status of a specific healthcheck.

TypeScript

const health = await ctx.resilience.healthcheck.check({
  tag: 'healthcheck-tag',         // Tag of existing healthcheck
  // Optional, defaults to current env
});

// Returns:
// {
//   tag: 'healthcheck-tag',
//   status: 'healthy' | 'unhealthy' | 'degraded' | 'unknown',
//   lastChecked: 1699900000000,
//   lastAvailable: 1699900000000,
//   lastLatency: 150,
//   averageLatency: 145,
//   error?: 'Connection timeout'
// }

Java

Map<String, Object> health = ctx.resilience.healthcheck.check(Map.of(
  "tag", "healthcheck-tag",         // Tag of existing healthcheck
  // Optional, defaults to current env
));

// Returns:
// Map.of(
//   "tag", "healthcheck-tag",
//   "status", "healthy" | 'unhealthy' | 'degraded' | 'unknown',
//   "lastChecked", 1699900000000,
//   "lastAvailable", 1699900000000,
//   "lastLatency", 150,
//   "averageLatency", 145,
//   error?: 'Connection timeout'
// )

Go

health := ctx.resilience.healthcheck.check({
  "tag": "healthcheck-tag",         // Tag of existing healthcheck
  // Optional, defaults to current env
});

// Returns:
// {
//   "tag": "healthcheck-tag",
//   "status": "healthy" | 'unhealthy' | 'degraded' | 'unknown',
//   "lastChecked": 1699900000000,
//   "lastAvailable": 1699900000000,
//   "lastLatency": 150,
//   "averageLatency": 145,
//   error?: 'Connection timeout'
// }

.NET

var health = await ctx.resilience.healthcheck.check({
  ["tag"] = "healthcheck-tag",         // Tag of existing healthcheck
  // Optional, defaults to current env
});

// Returns:
// {
//   ["tag"] = "healthcheck-tag",
//   ["status"] = "healthy" | 'unhealthy' | 'degraded' | 'unknown',
//   ["lastChecked"] = 1699900000000,
//   ["lastAvailable"] = 1699900000000,
//   ["lastLatency"] = 150,
//   ["averageLatency"] = 145,
//   error?: 'Connection timeout'
// }

healthcheck.status()

Get health status for all configured healthchecks referenced by the agent.

TypeScript

const allHealth = await ctx.resilience.healthcheck.status();

// Returns: Record<string, IAgentHealthcheckResult>
// {
//   'anthropic-health': { tag: '...', status: 'healthy', ... },
//   'openai-health': { tag: '...', status: 'degraded', ... },
// }

Java

Map<String, Object> allHealth = ctx.resilience.healthcheck.status();

// Returns: Record<string, IAgentHealthcheckResult>
// Map.of(
//   'anthropic-health': Map.of( "tag", "...", "status", "healthy", ... ),
//   'openai-health': Map.of( "tag", "...", "status", "degraded", ... ),
// )

Go

allHealth := ctx.resilience.healthcheck.status();

// Returns: Record<string, IAgentHealthcheckResult>
// {
//   'anthropic-health': { "tag": "...", "status": "healthy", ... },
//   'openai-health': { "tag": "...", "status": "degraded", ... },
// }

.NET

var allHealth = await ctx.resilience.healthcheck.status();

// Returns: Record<string, IAgentHealthcheckResult>
// {
//   'anthropic-health': { ["tag"] = "...", ["status"] = "healthy", ... },
//   'openai-health': { ["tag"] = "...", ["status"] = "degraded", ... },
// }

---

Complete Example

Here's a production-ready agent using existing resilience configurations:

TypeScript

import Ductape from '@ductape/sdk';

const ductape = new Ductape({...});

// Assume these resilience configs already exist at the product level:
// - healthchecks: 'anthropic-health', 'openai-health'
// - quotas: 'cost-optimized-quota'
// - fallbacks: 'llm-fallback'

const agent = await ductape.agents.define({
  tag: 'resilient-support-agent',
  name: 'Resilient Support Agent',
  model: 'claude-primary',
  systemPrompt: `You are a customer support agent with access to multiple
AI providers for reliability. Always provide helpful responses.`,

  tools: [
    {
      tag: 'answer-question',
      description: 'Answer customer questions using resilient LLM calls',
      parameters: {
        question: { type: 'string', description: 'Customer question', required: true },
        context: { type: 'string', description: 'Additional context' },
      },
      handler: async (ctx, params) => {
        // Check overall health first
        const health = await ctx.resilience.healthcheck.status();
        const healthyProviders = Object.values(health)
          .filter(h => h.status === 'healthy').length;

        ctx.log.info(`${healthyProviders} healthy providers available`);

        // Use fallback for critical customer responses
        const response = await ctx.resilience.fallback.run({
          tag: 'llm-fallback',
          input: {
            prompt: `Context: ${params.context || 'None'}

Question: ${params.question}

Please provide a helpful, accurate response.`,
          },
        });

        return response;
      },
    },
    {
      tag: 'summarize-ticket',
      description: 'Summarize support tickets using load-balanced providers',
      parameters: {
        ticketId: { type: 'string', description: 'Ticket ID', required: true },
      },
      // Use quota for non-critical operations to optimize costs
      resilience: {
        quota: 'cost-optimized-quota',
      },
      handler: async (ctx, params) => {
        const ticket = await ctx.database.query({
          database: 'tickets-db',
          event: 'get-ticket',
          params: { id: params.ticketId },
        });

        return ctx.resilience.quota.run({
          tag: 'cost-optimized-quota',
          input: {
            prompt: `Summarize this support ticket: ${JSON.stringify(ticket)}`,
          },
        });
      },
    },
  ],

  resilience: {
    defaults: {
      healthcheck: 'anthropic-health',
      fallback: 'llm-fallback',
    },
  },

  termination: {
    maxIterations: 10,
    timeout: '5m',
  },
});

// Run the agent
const result = await ductape.agents.run({
  tag: 'resilient-support-agent',
  input: {
    question: 'How do I reset my password?',
  },
});

Java

import app.ductape.sdk.Ductape;
import app.ductape.sdk.core.EnvType;
import app.ductape.sdk.core.RequestContext;

Map<String, Object> ductape = new Ductape(Map.of(...));

// Assume these resilience configs already exist at the product level:
// - "healthchecks", "anthropic-health", 'openai-health'
// - "quotas", "cost-optimized-quota"
// - "fallbacks", "llm-fallback"

Map<String, Object> agent = ductape.agents.define(Map.of(
  "tag", "resilient-support-agent",
  "name", "Resilient Support Agent",
  "model", "claude-primary",
  systemPrompt: `You are a customer support agent with access to multiple
AI providers for reliability. Always provide helpful responses.`,

  tools: [
    Map.of(
      "tag", "answer-question",
      "description", "Answer customer questions using resilient LLM calls",
      parameters: Map.of(
        question: Map.of( "type", "string", "description", "Customer question", "required", true ),
        context: Map.of( "type", "string", "description", "Additional context" )
      ),
      handler: async (ctx, params) => Map.of(
        // Check overall health first
        Map<String, Object> health = ctx.resilience.healthcheck.status();
        Map<String, Object> healthyProviders = Object.values(health)
          .filter(h => h.status === 'healthy').length;

        ctx.log.info(`$Map.of(healthyProviders) healthy providers available`);

        // Use fallback for critical customer responses
        Map<String, Object> response = ctx.resilience.fallback.run(Map.of(
          "tag", "llm-fallback",
          input: Map.of(
            prompt: `Context: $Map.of(params.context || 'None')

Question: $Map.of(params.question)

Please provide a helpful, accurate response.`
          )
        ));

        return response;
      )
    ),
    Map.of(
      "tag", "summarize-ticket",
      "description", "Summarize support tickets using load-balanced providers",
      parameters: Map.of(
        ticketId: Map.of( "type", "string", "description", "Ticket ID", "required", true )
      ),
      // Use quota for non-critical operations to optimize costs
      resilience: Map.of(
        "quota", "cost-optimized-quota"
      ),
      handler: async (ctx, params) => Map.of(
        Map<String, Object> ticket = ctx.database.query(Map.of(
          "database", "tickets-db",
          "event", "get-ticket",
          params: Map.of( id: params.ticketId )
        ));

        return ctx.resilience.quota.run(Map.of(
          "tag", "cost-optimized-quota",
          input: Map.of(
            prompt: `Summarize this support ticket: $Map.of(JSON.stringify(ticket))`
          )
        ));
      )
    ),
  ],

  resilience: Map.of(
    defaults: Map.of(
      "healthcheck", "anthropic-health",
      "fallback", "llm-fallback"
    )
  ),

  termination: Map.of(
    "maxIterations", 10,
    "timeout", "5m"
  )
));

// Run the agent
Map<String, Object> result = ductape.agents().run(Map<String, Object>.of(

  "tag", "resilient-support-agent",
  input: Map.of(
    "question", "How do I reset my password?"
  )
));

Go

import (
	"context"
	"github.com/ductape/ductape/sdk/go/core"
	ductapesdk "github.com/ductape/ductape/sdk/go/ductape"
)

ductape := new Ductape({...});

// Assume these resilience configs already exist at the product level:
// - "healthchecks": "anthropic-health", 'openai-health'
// - "quotas": "cost-optimized-quota"
// - "fallbacks": "llm-fallback"

agent := client.agents.define({
  "tag": "resilient-support-agent",
  "name": "Resilient Support Agent",
  "model": "claude-primary",
  systemPrompt: `You are a customer support agent with access to multiple
AI providers for reliability. Always provide helpful responses.`,

  tools: [
    {
      "tag": "answer-question",
      "description": "Answer customer questions using resilient LLM calls",
      parameters: {
        question: { "type": "string", "description": "Customer question", "required": true },
        context: { "type": "string", "description": "Additional context" },
      },
      handler: async (ctx, params) => {
        // Check overall health first
        health := ctx.resilience.healthcheck.status();
        healthyProviders := Object.values(health)
          .filter(h => h.status === 'healthy').length;

        ctx.log.info(`${healthyProviders} healthy providers available`);

        // Use fallback for critical customer responses
        response := ctx.resilience.fallback.run({
          "tag": "llm-fallback",
          input: {
            prompt: `Context: ${params.context || 'None'}

Question: ${params.question}

Please provide a helpful, accurate response.`,
          },
        });

        return response;
      },
    },
    {
      "tag": "summarize-ticket",
      "description": "Summarize support tickets using load-balanced providers",
      parameters: {
        ticketId: { "type": "string", "description": "Ticket ID", "required": true },
      },
      // Use quota for non-critical operations to optimize costs
      resilience: {
        "quota": "cost-optimized-quota",
      },
      handler: async (ctx, params) => {
        ticket := ctx.database.query({
          "database": "tickets-db",
          "event": "get-ticket",
          params: { id: params.ticketId },
        });

        return ctx.resilience.quota.run({
          "tag": "cost-optimized-quota",
          input: {
            prompt: `Summarize this support ticket: ${JSON.stringify(ticket)}`,
          },
        });
      },
    },
  ],

  resilience: {
    defaults: {
      "healthcheck": "anthropic-health",
      "fallback": "llm-fallback",
    },
  },

  termination: {
    "maxIterations": 10,
    "timeout": "5m",
  },
});

// Run the agent
result := client.agents.run({
  "tag": "resilient-support-agent",
  input: {
    "question": "How do I reset my password?",
  },
});

.NET

using Ductape.Sdk;
using Ductape.Sdk.Core;

var ductape = new Ductape({...});

// Assume these resilience configs already exist at the product level:
// - ["healthchecks"] = "anthropic-health", 'openai-health'
// - ["quotas"] = "cost-optimized-quota"
// - ["fallbacks"] = "llm-fallback"

var agent = await ductape.agents.define({
  ["tag"] = "resilient-support-agent",
  ["name"] = "Resilient Support Agent",
  ["model"] = "claude-primary",
  systemPrompt: `You are a customer support agent with access to multiple
AI providers for reliability. Always provide helpful responses.`,

  tools: [
    {
      ["tag"] = "answer-question",
      ["description"] = "Answer customer questions using resilient LLM calls",
      parameters: {
        question: { ["type"] = "string", ["description"] = "Customer question", ["required"] = true },
        context: { ["type"] = "string", ["description"] = "Additional context" },
      },
      handler: async (ctx, params) => {
        // Check overall health first
        var health = await ctx.resilience.healthcheck.status();
        var healthyProviders = Object.values(health)
          .filter(h => h.status === 'healthy').length;

        ctx.log.info(`${healthyProviders} healthy providers available`);

        // Use fallback for critical customer responses
        var response = await ctx.resilience.fallback.run({
          ["tag"] = "llm-fallback",
          input: {
            prompt: `Context: ${params.context || 'None'}

Question: ${params.question}

Please provide a helpful, accurate response.`,
          },
        });

        return response;
      },
    },
    {
      ["tag"] = "summarize-ticket",
      ["description"] = "Summarize support tickets using load-balanced providers",
      parameters: {
        ticketId: { ["type"] = "string", ["description"] = "Ticket ID", ["required"] = true },
      },
      // Use quota for non-critical operations to optimize costs
      resilience: {
        ["quota"] = "cost-optimized-quota",
      },
      handler: async (ctx, params) => {
        var ticket = await ctx.database.query({
          ["database"] = "tickets-db",
          ["event"] = "get-ticket",
          params: { id: params.ticketId },
        });

        return ctx.resilience.quota.run({
          ["tag"] = "cost-optimized-quota",
          input: {
            prompt: `Summarize this support ticket: ${JSON.stringify(ticket)}`,
          },
        });
      },
    },
  ],

  resilience: {
    defaults: {
      ["healthcheck"] = "anthropic-health",
      ["fallback"] = "llm-fallback",
    },
  },

  termination: {
    ["maxIterations"] = 10,
    ["timeout"] = "5m",
  },
});

// Run the agent
var result = await ductape.agents.run({
  ["tag"] = "resilient-support-agent",
  input: {
    ["question"] = "How do I reset my password?",
  },
});

---

Best Practices

1. Define Resilience at the Product Level

TypeScript

// Good: Define resilience configs once at product level
await ductape.resilience.healthcheck.create('my-product', {
  tag: 'api-health',
  // ... config
});

// Then reference in multiple agents
const agent1 = await ductape.agents.define({
  resilience: { defaults: { healthcheck: 'api-health' } },
  // ...
});

const agent2 = await ductape.agents.define({
  resilience: { defaults: { healthcheck: 'api-health' } },
  // ...
});

Java

// Good: Define resilience configs once at product level
ductape.resilience.healthcheck.create('my-product', Map.of(
  "tag", "api-health",
  // ... config
));

// Then reference in multiple agents
Map<String, Object> agent1 = ductape.agents.define(Map.of(
  resilience: Map.of( defaults: Map.of( "healthcheck", "api-health" ) ),
  // ...
));

Map<String, Object> agent2 = ductape.agents.define(Map.of(
  resilience: Map.of( defaults: Map.of( "healthcheck", "api-health" ) ),
  // ...
));

Go

// Good: Define resilience configs once at product level
client.resilience.healthcheck.create('my-product', {
  "tag": "api-health",
  // ... config
});

// Then reference in multiple agents
agent1 := client.agents.define({
  resilience: { defaults: { "healthcheck": "api-health" } },
  // ...
});

agent2 := client.agents.define({
  resilience: { defaults: { "healthcheck": "api-health" } },
  // ...
});

.NET

// Good: Define resilience configs once at product level
await ductape.resilience.healthcheck.create('my-product', {
  ["tag"] = "api-health",
  // ... config
});

// Then reference in multiple agents
var agent1 = await ductape.agents.define({
  resilience: { defaults: { ["healthcheck"] = "api-health" } },
  // ...
});

var agent2 = await ductape.agents.define({
  resilience: { defaults: { ["healthcheck"] = "api-health" } },
  // ...
});

2. Use Fallbacks for Critical Operations

TypeScript

// Critical customer-facing operations should use fallbacks
{
  tag: 'process-payment',
  resilience: { fallback: 'payment-fallback' },
  handler: async (ctx, params) => {
    return ctx.resilience.fallback.run({
      tag: 'payment-fallback',
      input: params,
    });
  },
}

Java

// Critical customer-facing operations should use fallbacks
Map.of(
  "tag", "process-payment",
  resilience: Map.of( "fallback", "payment-fallback" ),
  handler: async (ctx, params) => Map.of(
    return ctx.resilience.fallback.run(Map.of(
      "tag", "payment-fallback",
      input: params
    ));
  )
)

Go

import "context"

// Critical customer-facing operations should use fallbacks
{
  "tag": "process-payment",
  resilience: { "fallback": "payment-fallback" },
  handler: async (ctx, params) => {
    return ctx.resilience.fallback.run({
      "tag": "payment-fallback",
      input: params,
    });
  },
}

.NET

// Critical customer-facing operations should use fallbacks
{
  ["tag"] = "process-payment",
  resilience: { ["fallback"] = "payment-fallback" },
  handler: async (ctx, params) => {
    return ctx.resilience.fallback.run({
      ["tag"] = "payment-fallback",
      input: params,
    });
  },
}

3. Use Quotas for Cost Optimization

TypeScript

// Non-critical operations can use quotas to optimize costs
{
  tag: 'generate-suggestions',
  resilience: { quota: 'cost-optimized' },
  handler: async (ctx, params) => {
    return ctx.resilience.quota.run({
      tag: 'cost-optimized',
      input: params,
    });
  },
}

Java

// Non-critical operations can use quotas to optimize costs
Map.of(
  "tag", "generate-suggestions",
  resilience: Map.of( "quota", "cost-optimized" ),
  handler: async (ctx, params) => Map.of(
    return ctx.resilience.quota.run(Map.of(
      "tag", "cost-optimized",
      input: params
    ));
  )
)

Go

import "context"

// Non-critical operations can use quotas to optimize costs
{
  "tag": "generate-suggestions",
  resilience: { "quota": "cost-optimized" },
  handler: async (ctx, params) => {
    return ctx.resilience.quota.run({
      "tag": "cost-optimized",
      input: params,
    });
  },
}

.NET

// Non-critical operations can use quotas to optimize costs
{
  ["tag"] = "generate-suggestions",
  resilience: { ["quota"] = "cost-optimized" },
  handler: async (ctx, params) => {
    return ctx.resilience.quota.run({
      ["tag"] = "cost-optimized",
      input: params,
    });
  },
}

4. Check Health Before Critical Operations

TypeScript

handler: async (ctx, params) => {
  // Always check health for critical operations
  const health = await ctx.resilience.healthcheck.check({
    tag: 'primary-service',
  });

  if (health.status !== 'healthy') {
    ctx.log.warn('Primary service degraded, using fallback');
  }

  // Continue with operation...
}

Java

handler: async (ctx, params) => Map.of(
  // Always check health for critical operations
  Map<String, Object> health = ctx.resilience.healthcheck.check(Map.of(
    "tag", "primary-service"
  ));

  if (health.status !== 'healthy') Map.of(
    ctx.log.warn('Primary service degraded, using fallback');
  )

  // Continue with operation...
)

Go

import "context"

handler: async (ctx, params) => {
  // Always check health for critical operations
  health := ctx.resilience.healthcheck.check({
    "tag": "primary-service",
  });

  if (health.status !== 'healthy') {
    ctx.log.warn('Primary service degraded, using fallback');
  }

  // Continue with operation...
}

.NET

handler: async (ctx, params) => {
  // Always check health for critical operations
  var health = await ctx.resilience.healthcheck.check({
    ["tag"] = "primary-service",
  });

  if (health.status !== 'healthy') {
    ctx.log.warn('Primary service degraded, using fallback');
  }

  // Continue with operation...
}

---

Next Steps

• Resilience Overview - Learn how to define healthchecks, quotas, and fallbacks
• Agent Tools - Learn more about building agent tools
• Human in the Loop - Add approval gates
• Multi-Agent Systems - Orchestrate multiple agents