Go SDK

go get github.com/throughputs/throughputs-go

Initialize

import "github.com/throughputs/throughputs-go"

client := throughputs.New("thp_live_xxx")

baseURL defaults to https://api.throughputs.dev/v1. Override it for testing or self-hosted deployments:

client := throughputs.NewWithConfig(throughputs.Config{
    APIKey:  "thp_live_xxx",
    BaseURL: "https://api.throughputs.dev/v1",
})

Chat completions

resp, err := client.Chat.Completions.Create(ctx, &throughputs.ChatCompletionRequest{
    Model: "gpt-4o",
    Messages: []throughputs.Message{
        {Role: "user", Content: "Hello"},
    },
})
if err != nil {
    log.Fatal(err)
}

fmt.Println(resp.Choices[0].Message.Content)
fmt.Println(resp.ThroughputsCostUsd) // first-party field

Streaming

stream, err := client.Chat.Completions.CreateStream(ctx, &throughputs.ChatCompletionRequest{
    Model: "gpt-4o",
    Messages: []throughputs.Message{
        {Role: "user", Content: "Count to 5"},
    },
})
if err != nil {
    log.Fatal(err)
}
defer stream.Close()

for {
    chunk, err := stream.Recv()
    if errors.Is(err, io.EOF) {
        break
    }
    if err != nil {
        log.Fatal(err)
    }
    if chunk.Choices[0].Delta.Content != "" {
        fmt.Print(chunk.Choices[0].Delta.Content)
    }
}

Fallback chains

If the primary model is down or slow, THROUGHPUTS falls back to backups you specify:

resp, err := client.Chat.Completions.Create(ctx, &throughputs.ChatCompletionRequest{
    Model: "gpt-4o",
    Messages: msgs,
    ModelFallback: []string{"claude-3-5-sonnet", "llama-3.1-70b"},
})

fmt.Println(resp.ThroughputsModelServed) // which model actually answered

The routing layer tries each in order. The first one to succeed wins.

Context and cancellation

Every method takes a context.Context. Cancel the context to abort an in-flight request:

ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()

resp, err := client.Chat.Completions.Create(ctx, req)

Error handling

var rateLimitErr *throughputs.RateLimitError
var authErr *throughputs.AuthenticationError

resp, err := client.Chat.Completions.Create(ctx, req)
switch {
case errors.As(err, &authErr):
    // invalid or revoked key
case errors.As(err, &rateLimitErr):
    // 429 — back off and retry
    wait := rateLimitErr.RetryAfter
case err != nil:
    var throughputsErr *throughputs.ThroughputsError
    if errors.As(err, &throughputsErr) {
        log.Println(throughputsErr.StatusCode, throughputsErr.Message)
    } else {
        log.Fatal(err)
    }
}