securityvaultapi-keyssecrets

Status: ProposedAuthor: Security TeamDeciders: Core TeamCreated: Apr 19, 2026Updated: Apr 19, 2026

ADR-068: Vault API Key Storage Patterns

Status

Proposed

Context

Pattern providers and backend drivers require secure storage for API keys, database credentials, and other secrets. Current approach stores credentials in config files or environment variables, which:

Insecure: Secrets in config files (git history, logs)
No Rotation: Static credentials that never change
No Auditing: No audit trail for who accessed what
No Fine-grained Access: All services see all credentials
No Leakage Protection: Environment variables visible to all processes

Requirements:

Vault secrets engine for secure API key storage
Dynamic API key generation via Vault
Key rotation support with lease management
Audit logging for all key access
Pattern provider API key backend integration
Multi-tenant with namespace isolation

Vault Requirements:

Vault agent sidecar or direct integration
JWT authentication for Prism apps
KV2 secrets engine for dynamic keys
Audit logs for all access

Decision

We will implement API key storage using Vault KV2 secrets engine with the following patterns:

1. Vault KV2 Secret Engine Structure

# Enable KV2 secrets engine
vault secrets enable -path=secret kv-v2

# Enable cubbyhole secrets engine (for per-session secrets)
vault secrets enable -path=cubbyhole cubbyhole

# Enable authentication
vault auth enable jwt
vault auth enable aws
vault auth enable kubernetes

2. API Key Storage Path Pattern

// pkg/authz/apikey_paths.go
const (
    APIKeyBasePath = "secret/data/apikeys"
)

type APIKeyPath struct {
    Provider   string
    Environment string
    Namespace  string
    Service    string
    KeyName    string
}

func (p *APIKeyPath) String() string {
    return fmt.Sprintf("%s/%s/%s/%s/%s/%s",
        APIKeyBasePath,
        p.Provider,
        p.Environment,
        p.Namespace,
        p.Service,
        p.KeyName,
    )
}

// Example paths:
// secret/data/apikeys/redis/prod/iota-devices/api-gateway/master-key
// secret/data/apikeys/kafka/prod/iota-devices/data-ingester/producer-key
// secret/data/apikeys/postgres/prod/iota-devices/analytics-service/db-password

3. API Key Schema

// proto/prism/authz/apikey.proto
syntax = "proto3";

package prism.authz;

import "google/protobuf/timestamp.proto";

message APIKey {
  string provider = 1;          // e.g., "redis", "kafka", "postgres"
  string environment = 2;       // e.g., "prod", "staging", "dev"
  string namespace = 3;         // e.g., "iota-devices"
  string service = 4;           // e.g., "api-gateway"
  string key_name = 5;          //Human-readable name
  string key_id = 6;           //Unique ID for reference
  string key_value = 7;        //The actual secret (encrypted)
  string secret_hash = 8;      //Hash for comparison (SHA256)

  // Rotation policy
  int64 rotation_period_hours = 9;
  google.protobuf.Timestamp next_rotation = 10;
  bool auto_rotate = 11;

  // Metadata
  string created_by = 12;
  google.protobuf.Timestamp created_at = 13;
  google.protobuf.Timestamp last_rotated = 14;
  google.protobuf.Timestamp expires_at = 15;

  // Access control
  repeated string allowed_principals = 16;
  repeated string denied_principals = 17;

  // Status
  KeyStatus status = 18;

  // Type of key
  KeyType type = 19;
}

enum KeyStatus {
  ACTIVE = 0;
  INACTIVE = 1;
  REVOKED = 2;
  EXPIRED = 3;
}

enum KeyType {
  API_KEY = 0;       // Generic API key
  AWS_ACCESS_KEY = 1; // AWS IAM credentials
  DATABASE_CREDENTIAL = 2; // DB username/password
  BEARER_TOKEN = 3;  // OAuth2/Bearer token
  ENCRYPTION_KEY = 4; // Key encryption key (KEK)
}

4. Vault API Key Client

// pkg/authz/apikey_client.go
type APIKeyClient struct {
    vaultClient *VaultClient
    basePath    string
}

func NewAPIKeyClient(vaultClient *VaultClient) *APIKeyClient {
    return &APIKeyClient{
        vaultClient: vaultClient,
        basePath:    "secret/data/apikeys",
    }
}

// Create or update API key
func (c *APIKeyClient) Upsert(ctx context.Context, path *APIKeyPath, key *APIKey) error {
    // Encrypt key value before storage
    encrypted, err := c.encryptKey(key.KeyValue, path.String())
    if err != nil {
        return err
    }

    data := map[string]interface{}{
        "provider":        key.Provider,
        "environment":     key.Environment,
        "namespace":       key.Namespace,
        "service":         key.Service,
        "key_name":        key.KeyName,
        "key_value":       encrypted,
        "key_id":          key.KeyId,
        "secret_hash":     hashKey(key.KeyValue),
        "rotation_period": key.RotationPeriodHours,
        "auto_rotate":     key.AutoRotate,
        "status":          key.Status.String(),
        "type":            key.Type.String(),
    }

    return c.vaultClient.GetClient().KVv2(c.basePath).Put(ctx, path.String(), data)
}

// Read API key
func (c *APIKeyClient) Read(ctx context.Context, path *APIKeyPath) (*APIKey, error) {
    secret, err := c.vaultClient.GetClient().KVv2(c.basePath).Get(ctx, path.String())
    if err != nil {
        return nil, err
    }

    if secret == nil || secret.Data == nil {
        return nil, fmt.Errorf("key not found: %s", path.String())
    }

    // Decrypt key value
    encryptedValue := secret.Data["key_value"].(string)
    decrypted, err := c.decryptKey(encryptedValue, path.String())
    if err != nil {
        return nil, err
    }

    return &APIKey{
        Provider:        secret.Data["provider"].(string),
        Environment:     secret.Data["environment"].(string),
        Namespace:       secret.Data["namespace"].(string),
        Service:         secret.Data["service"].(string),
        KeyName:         secret.Data["key_name"].(string),
        KeyId:           secret.Data["key_id"].(string),
        KeyValue:        decrypted,
        SecretHash:      secret.Data["secret_hash"].(string),
        RotationPeriodHours: secret.Data["rotation_period"].(int64),
        AutoRotate:      secret.Data["auto_rotate"].(bool),
        Status:          KeyStatus(secret.Data["status"].(int32)),
        Type:            KeyType(secret.Data["type"].(int32)),
    }, nil
}

// Delete API key (soft delete - mark as revoked)
func (c *APIKeyClient) Delete(ctx context.Context, path *APIKeyPath) error {
    // Update status to REVOKED
    secret, err := c.vaultClient.GetClient().KVv2(c.basePath).Get(ctx, path.String())
    if err != nil {
        return err
    }

    secret.Data["status"] = KeyStatus_REVOKED
    secret.Data["revoked_at"] = time.Now().Unix()

    return c.vaultClient.GetClient().KVv2(c.basePath).Put(ctx, path.String(), secret.Data)
}

// Rotate API key
func (c *APIKeyClient) Rotate(ctx context.Context, path *APIKeyPath) (*APIKey, error) {
    // Read existing key
    existing, err := c.Read(ctx, path)
    if err != nil {
        return nil, err
    }

    // Generate new key
    newKey := GenerateSecureKey(existing.Type)

    // Update key
    existing.KeyValue = newKey
    existing.LastRotated = time.Now()
    existing.RotationPeriodHours = existing.RotationPeriodHours // Keep same
    existing.NextRotation = time.Now().Add(time.Duration(existing.RotationPeriodHours) * time.Hour)

    // Write new key
    if err := c.Upsert(ctx, path, existing); err != nil {
        return nil, err
    }

    // Audit log
    c.logAuditEvent(ctx, "rotate", path.String(), existing)

    return existing, nil
}

// List API keys for a service
func (c *APIKeyClient) List(ctx context.Context, namespace string, service string) ([]*APIKeyPath, error) {
    paths, err := c.vaultClient.GetClient().KVv2(c.basePath).List(ctx, "")
    if err != nil {
        return nil, err
    }

    var result []*APIKeyPath
    for _, path := range paths {
        if strings.Contains(path, namespace) && strings.Contains(path, service) {
            p := parsePath(path)
            result = append(result, p)
        }
    }

    return result, nil
}

5. Dynamic Credential Generation

For backends that support dynamic credentials (PostgreSQL, MySQL, MongoDB):

// pkg/authz/dynamic_credentials.go
type DynamicCredentialsConfig struct {
    Backend    string
    Role       string
    TTL        time.Duration
    Renewable  bool
}

// Generate dynamic PostgreSQL credentials
func (c *APIKeyClient) GeneratePostgresCredentials(ctx context.Context, namespace string) (*BackendCredentials, error) {
    path := fmt.Sprintf("postgres/creds/prism-role-%s", namespace)

    resp, err := c.vaultClient.GetClient().Logical().ReadWithContext(ctx, path)
    if err != nil {
        return nil, fmt.Errorf("failed to generate postgres credentials: %w", err)
    }

    if resp == nil || resp.Data == nil {
        return nil, fmt.Errorf("no credentials returned from Vault")
    }

    return &BackendCredentials{
        Username:      resp.Data["username"].(string),
        Password:      resp.Data["password"].(string),
        LeaseID:       resp.LeaseID,
        LeaseDuration: time.Duration(resp.LeaseDuration) * time.Second,
        BackendType:   BackendTypePostgres,
    }, nil
}

// Generate dynamic Kafka credentials
func (c *APIKeyClient) GenerateKafkaCredentials(ctx context.Context, namespace string) (*BackendCredentials, error) {
    path := fmt.Sprintf("kafka/creds/prism-role-%s", namespace)

    resp, err := c.vaultClient.GetClient().Logical().ReadWithContext(ctx, path)
    if err != nil {
        return nil, fmt.Errorf("failed to generate kafka credentials: %w", err)
    }

    if resp == nil || resp.Data == nil {
        return nil, fmt.Errorf("no credentials returned from Vault")
    }

    return &BackendCredentials{
       _username:      resp.Data["username"].(string),
        Password:      resp.Data["password"].(string),
        APIKey:        resp.Data["api_key"].(string),
        APISecret:     resp.Data["api_secret"].(string),
        LeaseID:       resp.LeaseID,
        LeaseDuration: time.Duration(resp.LeaseDuration) * time.Second,
        BackendType:   BackendTypeKafka,
    }, nil
}

6. Pattern Provider Integration

// patterns/producer/pkg/kafka/store.go
type KafkaStore struct {
    apiKeyClient *authz.APIKeyClient
    apiKeyPath   *authz.APIKeyPath
}

func NewKafkaStore(config Config) (*KafkaStore, error) {
    vaultClient, err := authz.NewVaultClient(config.VaultConfig)
    if err != nil {
        return nil, err
    }

    apiKeyClient := authz.NewAPIKeyClient(vaultClient)

    return &KafkaStore{
        apiKeyClient: apiKeyClient,
        apiKeyPath: &authz.APIKeyPath{
            Provider:    "kafka",
            Environment: config.Environment,
            Namespace:   config.Namespace,
            Service:     "producer",
            KeyName:     "default",
        },
    }, nil
}

func (s *KafkaStore) GetCredentials(ctx context.Context) (*authz.BackendCredentials, error) {
    // Read from Vault
    key, err := s.apiKeyClient.Read(ctx, s.apiKeyPath)
    if err != nil {
        // Fallback to config if unavailable
        return s.configToFallbackCredential()
    }

    // Check if rotation is needed
    if time.Now().After(key.NextRotation) {
        s.rotateIfNeeded(ctx)
    }

    return &authz.BackendCredentials{
        APIKey:    key.KeyId,
        APISecret: key.KeyValue,
        BackendType: authz.BackendTypeKafka,
    }, nil
}

func (s *KafkaStore) rotateIfNeeded(ctx context.Context) {
    go func() {
        _, err := s.apiKeyClient.Rotate(ctx, s.apiKeyPath)
        if err != nil {
            log.Printf("Failed to rotate Kafka credentials: %v", err)
        }
    }()
}

Rationale

Why KV2 over Other Engines?

Alternatives Considered:

KV1 (Rejected)
- Pros: Simple, backwards compatible
- Cons: ❌ No versioning, ❌ No metadata, ❌ No deletion policy
- Verdict: KV2 is better and stable
Databases Engine (Rejected for API keys)
- Pros: Automatic credential rotation
- Cons: ❌ Only supports databases, ❌ Complex setup
- Verdict: Use KV2 for API keys, databases engine for DBs
AWS Secrets Manager (Rejected)
- Pros: Native AWS integration
- Cons: ❌ Cloud-specific, ❌ No support for multi-cloud
- Verdict: Vault is multi-cloud
HashiCorp Consul (Rejected)
- Pros: Already in use
- Cons: ❌ Not designed for secrets, ❌ No audit logging
- Verdict: Use Vault for secrets
Cubbyhole Engine (Rejected for persistent keys)
- Pros: Per-token storage, automatic cleanup
- Cons: ❌ Data lost when token expires
- Verdict: Use KV2 for persistent, cubbyhole for session keys

Design Decisions

1. Separate KV2 Mount Point

Dedicated secret/data/apikeys path
Avoids mixing with other secrets
Easier RBAC policies

2. Encryption at Rest

All key values encrypted before Vault storage
Key encryption managed by Vault transit engine
Protects against Vault DB compromise

3. Soft Delete (Mark Revoked)

Keys aren't deleted, just marked revoked
Audit trail maintained
Can restore accidentally revoked keys

4. Automatic Rotation

Configurable rotation period per key
Background rotation goroutine
Graceful handling of rotation failures

5. Namespace Isolation

Keys scoped to namespace + service
No cross-namespace access
Fine-grained RBAC possible

Audit Logging

// pkg/authz/audit.go
type AuditEvent struct {
    Event      string    `json:"event"`
    Path       string    `json:"path"`
    Actor      string    `json:"actor"`
    Timestamp  time.Time `json:"timestamp"`
    Metadata   map[string]interface{} `json:"metadata"`
}

func (c *APIKeyClient) logAuditEvent(ctx context.Context, event string, path string, key *APIKey) {
    actor := extractActorFromContext(ctx)

    event := &AuditEvent{
        Event:     event,
        Path:      path,
        Actor:     actor,
        Timestamp: time.Now(),
        Metadata: map[string]interface{}{
            "service":  key.Service,
            "provider": key.Provider,
            "status":   key.Status,
        },
    }

    // Write to audit log
    c.auditLogWriter.Write(event)

    // Also log to tracing
    span := trace.SpanFromContext(ctx)
    span.AddEvent("apikey." + event, trace.WithAttributes(
        attribute.String("apikey.path", path),
        attribute.String("apikey.service", key.Service),
    ))
}

Consequences

Positive

Secure Storage: All API keys encrypted at rest and in transit
Automatic Rotation: Keys rotate on schedule without manual intervention
Audit Trail: Every access logged with actor, timestamp, metadata
Fine-grained Access: RBAC policies per key
Multi-tenant: Namespace isolation prevents cross-tenant leakage
Backwards Compatible: Can migrate existing keys
Dynamic Credentials: Support for databases that support Vault dynamic creds

Negative

Complexity: Multiple patterns (static keys, dynamic creds, etc.)
Vendor Lock-in: Vault-specific implementation
Performance: Additional Vault calls for key reads
Secret Management: Still need to manage Vault access credentials

Mitigations

Cache keys locally with TTL
Use Vault agent sidecar for local caching
Graceful degradation with warning logs
Health checks for Vault connectivity

Implementation Notes

Migration Steps

Phase 1: Setup Vault Backend (Week 1)

Enable KV2 secrets engine
Configure audit logging
Create RBAC policies
Test basic CRUD operations

Phase 2: Pattern Provider Migration (Week 2)

Implement APIKeyClient
Migrate Kafka pattern provider
Migrate Producer pattern provider
Add integration tests

Phase 3: Backend Driver Integration (Week 3)

Update Redis driver for dynamic keys
Update PostgreSQL driver for dynamic creds
Update Kafka driver for Vault secrets
Performance optimization

Phase 4: Production Rollout (Week 4)

Deploy to staging
Monitor Vault usage
Gradual rollout
Decommission old secret stores

Key Gotchas

Lease Management: Don't forget to renew leases for dynamic creds
Error Handling: Handle Vault unavailability gracefully
Race Conditions: Use CAS (Check-And-Set) for concurrent updates
Secret Handling: Never log secret values, only hashes

Code Examples

Pattern Provider: Redis

// patterns/producer/pkg/redis/store.go
func (s *RedisStore) Connect() error {
    // Get API key from Vault
    creds, err := s.apiKeyClient.Read(s.ctx, s.apiKeyPath)
    if err != nil {
        return fmt.Errorf("failed to get Redis credentials: %w", err)
    }

    // Connect to Redis
    s.client = redis.NewClient(&redis.Options{
        Addr:         s.config.RedisAddr,
        Password:     creds.Password,
        DB:           s.config.RedisDB,
        DialTimeout:  5 * time.Second,
        ReadTimeout:  3 * time.Second,
        WriteTimeout: 3 * time.Second,
    })

    return nil
}

Pattern Provider: Kafka

// patterns/producer/pkg/kafka/producer.go
func (p *Producer) produceMessage(ctx context.Context, msg *Message) error {
    // Get credentials from Vault
    creds, err := p.keyClient.Read(ctx, p.apiKeyPath)
    if err != nil {
        return fmt.Errorf("failed to get Kafka credentials: %w", err)
    }

    // Create Kafka client with credentials
    config := sarama.NewConfig()
    config.Net.SASL.Enable = true
    config.Net.SASL.User = creds.Username
    config.Net.SASL.Password = creds.Password

    producer, err := sarama.NewAsyncProducer([]string{p.broker}, config)
    if err != nil {
        return err
    }

    // Produce message...
    return nil
}

Vault CLI Examples

# Create API key
vault kv put secret/apikeys/redis/prod/iota-devices/api-gateway/master-key \
    key_value="rk_abc123def456" \
    key_id="key-001" \
    provider=redis \
    environment=prod \
    namespace=iota-devices \
    service=api-gateway \
    rotation_period_hours=168 \
    auto_rotate=true \
    status=ACTIVE

# Read API key
vault kv get -format=json secret/apikeys/redis/prod/iota-devices/api-gateway/master-key

# Rotate API key
curl -X POST \
  -H "X-Vault-Token:..." \
  http://vault:8200/v1/secret/data/apikeys/redis/prod/iota-devices/api-gateway/master-key \
  -d '{"key_value":"rk_new789xyz"}'

# List all API keys
vault kv list secret/apikeys

# Audit log example
vault audit list
# Path        Type     Description
# ----        ----     -----------
# file/       file     file path: /var/log/vault/audit.log

ADR-007: Authentication & Authorization - Initial authn/authz design
ADR-050: Topaz Policy Engine - Authorization policies
ADR-067: Vault PKI mTLS - mTLS architecture

Revision History

2026-04-19: Initial draft - Vault API key storage patterns

Status​

Context​

Decision​

1. Vault KV2 Secret Engine Structure​

2. API Key Storage Path Pattern​

3. API Key Schema​

4. Vault API Key Client​

5. Dynamic Credential Generation​

6. Pattern Provider Integration​

Rationale​

Why KV2 over Other Engines?​

Design Decisions​

Audit Logging​

Consequences​

Positive​

Negative​

Mitigations​

Implementation Notes​

Migration Steps​

Key Gotchas​

Code Examples​

Vault CLI Examples​

Related Documents​

Revision History​