Enterprise Semantic Caching with Redis & Vector Embeddings: Slashing LLM Costs by 55% in .NET Microservices

   ┌────────────────────────────────────────────────────────────────────────┐
   │                  Inbound Microservice Request / Copilot API            │
   └───────────────────────────────────┬────────────────────────────────────┘
                                       │
                                       ▼
   ┌────────────────────────────────────────────────────────────────────────┐
   │             Prompt Normalization & Dynamic Masking Pipeline            │
   │      - Strip greetings, dynamic timestamps, session identifiers        │
   │      - Extract canonical intent string                                 │
   └───────────────────────────────────┬────────────────────────────────────┘
                                       │
                                       ▼
   ┌────────────────────────────────────────────────────────────────────────┐
   │         Lightweight Dense Embedding Model (Local ONNX / Text-3-Small)  │
   │         Produces normalized float[384] or float[1536] dense vector     │
   └───────────────────────────────────┬────────────────────────────────────┘
                                       │
                                       ▼
   ┌────────────────────────────────────────────────────────────────────────┐
   │             L1 In-Memory Semantic Cache (Local RAM Fast Path)          │
   │             - Sub-microsecond lookup for hottest semantic keys         │
   └───────────────────┬────────────────────────────────┬───────────────────┘
                       │ Hit                            │ Miss
                       ▼                                ▼
              [Return Cached JSON]         ┌────────────────────────────────┐
                                           │  L2 Redis Vector Search        │
                                           │  FT.SEARCH with KNN & HNSW     │
                                           └────────────┬───────────────────┘
                                                        │
                         ┌──────────────────────────────┴──────────────────────────────┐
                         │ Match found with Cosine Sim >= 0.94                         │ Miss (Sim < 0.94)
                         ▼                                                             ▼
             ┌────────────────────────┐                                   ┌─────────────────────────┐
             │ Hydrate L1 Cache &     │                                   │ SingleFlight Dispatch   │
             │ Return Redis Response  │                                   │ to Azure OpenAI LLM     │
             │ (Latency: ~4ms - 8ms)  │                                   └────────────┬────────────┘
             └────────────────────────┘                                                │
                                                                                       ▼
                                                                          ┌─────────────────────────┐
                                                                          │ Stream Response to User │
                                                                          │ + Ingest into Redis L2  │
                                                                          └─────────────────────────┘

// Core Contract for Enterprise Vector Semantic Caching in .NET 10
using System;
using System.Threading;
using System.Threading.Tasks;

namespace Centralogic.AI.SemanticCache;

public readonly record struct SemanticCacheMatch<T>(
    T Payload,
    float SimilarityScore,
    string MatchedPrompt,
    string CacheKey,
    TimeSpan Age
);

public interface ISemanticCacheManager<TResponse>
{
    /// <summary>
    /// Searches the vector space for semantically equivalent cached prompts.
    /// </summary>
    ValueTask<SemanticCacheMatch<TResponse>?> FindSimilarAsync(
        ReadOnlyMemory<float> embeddingVector,
        string domainNamespace,
        float minimumSimilarity = 0.94f,
        CancellationToken ct = default);

    /// <summary>
    /// Ingests a new prompt, vector embedding, and LLM response into the cache.
    /// </summary>
    ValueTask StoreAsync(
        string normalizedPrompt,
        ReadOnlyMemory<float> embeddingVector,
        TResponse response,
        string domainNamespace,
        TimeSpan ttl,
        CancellationToken ct = default);

    /// <summary>
    /// Invalidates all cached vectors matching a specific domain tag namespace.
    /// </summary>
    ValueTask InvalidateNamespaceAsync(string domainNamespace, CancellationToken ct = default);
}

using System;
using System.Collections.Generic;
using System.Runtime.InteropServices;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using NRedisStack;
using NRedisStack.Search;
using NRedisStack.Search.Literals.Enums;
using StackExchange.Redis;

namespace Centralogic.AI.SemanticCache;

public sealed class RedisVectorCacheStore<TResponse> : ISemanticCacheManager<TResponse>
{
    private readonly IConnectionMultiplexer _redis;
    private readonly IDatabase _db;
    private readonly ISearchCommands _search;
    private readonly ILogger<RedisVectorCacheStore<TResponse>> _logger;
    private const string IndexName = "idx:semantic_cache:v1";
    private const string KeyPrefix = "semcache:";

    public RedisVectorCacheStore(
        IConnectionMultiplexer redis,
        ILogger<RedisVectorCacheStore<TResponse>> logger)
    {
        _redis = redis ?? throw new ArgumentNullException(nameof(redis));
        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
        _db = _redis.GetDatabase();
        _search = _db.FT();
        
        EnsureIndexCreated();
    }

    private void EnsureIndexCreated()
    {
        try
        {
            _search.Info(IndexName);
        }
        catch (RedisServerException)
        {
            // Index does not exist, initialize HNSW vector index schema
            _logger.LogInformation("Creating Redis HNSW Vector Search Index: {Index}", IndexName);
            
            var schema = new Schema()
                .AddTagField("namespace")
                .AddTextField("prompt", 1.0)
                .AddVectorField("vector", Schema.VectorField.VectorAlgo.HNSW, new Dictionary<string, object>
                {
                    ["TYPE"] = "FLOAT32",
                    ["DIM"] = 1536, // Azure OpenAI text-embedding-3-small or large
                    ["DISTANCE_METRIC"] = "COSINE",
                    ["M"] = 16,
                    ["EF_CONSTRUCTION"] = 200
                });

            _search.Create(IndexName, new FTCreateParams().On(IndexDataType.HASH).Prefix(KeyPrefix), schema);
        }
    }

    public async ValueTask<SemanticCacheMatch<TResponse>?> FindSimilarAsync(
        ReadOnlyMemory<float> embeddingVector,
        string domainNamespace,
        float minimumSimilarity = 0.94f,
        CancellationToken ct = default)
    {
        // Zero-allocation conversion from float[] to raw byte[] representation
        byte[] vectorBytes = MemoryMarshal.AsBytes(embeddingVector.Span).ToArray();

        // KNN Query syntax: filter by namespace tag and extract nearest neighbor vector score
        var query = new Query($"(@namespace:{{{domainNamespace}}})=>[KNN 1 @vector $query_vec AS vector_score]")
            .AddParam("query_vec", vectorBytes)
            .SetSortBy("vector_score")
            .ReturnFields("prompt", "response_json", "vector_score", "created_at")
            .Dialect(2);

        SearchResult result = await _search.SearchAsync(IndexName, query).ConfigureAwait(false);

        if (result.TotalResults == 0 || result.Documents.Count == 0)
        {
            return null;
        }

        var doc = result.Documents[0];
        
        // Redis returns Cosine Distance: Distance = 1.0 - CosineSimilarity
        if (!double.TryParse(doc["vector_score"], out double distance))
        {
            return null;
        }

        float similarity = (float)(1.0 - distance);

        if (similarity < minimumSimilarity)
        {
            _logger.LogDebug("Closest semantic match rejected. Similarity {Sim:F3} < Threshold {Min:F3}", similarity, minimumSimilarity);
            return null;
        }

        string responseJson = doc["response_json"];
        var payload = JsonSerializer.Deserialize<TResponse>(responseJson);
        string matchedPrompt = doc["prompt"];

        long createdAtTicks = long.Parse(doc["created_at"]);
        var age = DateTime.UtcNow - new DateTime(createdAtTicks, DateTimeKind.Utc);

        return new SemanticCacheMatch<TResponse>(
            Payload: payload!,
            SimilarityScore: similarity,
            MatchedPrompt: matchedPrompt,
            CacheKey: doc.Id,
            Age: age
        );
    }

    public async ValueTask StoreAsync(
        string normalizedPrompt,
        ReadOnlyMemory<float> embeddingVector,
        TResponse response,
        string domainNamespace,
        TimeSpan ttl,
        CancellationToken ct = default)
    {
        string hashKey = $"{KeyPrefix}{Guid.NewGuid():N}";
        byte[] vectorBytes = MemoryMarshal.AsBytes(embeddingVector.Span).ToArray();
        string responseJson = JsonSerializer.Serialize(response);

        var entries = new HashEntry[]
        {
            new("namespace", domainNamespace),
            new("prompt", normalizedPrompt),
            new("response_json", responseJson),
            new("vector", vectorBytes),
            new("created_at", DateTime.UtcNow.Ticks)
        };

        var batch = _db.CreateBatch();
        var hashTask = batch.HashSetAsync(hashKey, entries);
        var expireTask = batch.KeyExpireAsync(hashKey, ttl);
        batch.Execute();

        await Task.WhenAll(hashTask, expireTask).ConfigureAwait(false);
        _logger.LogInformation("Ingested semantic vector cache key {Key} for namespace {Ns} with TTL {TTL}", hashKey, domainNamespace, ttl);
    }

    public async ValueTask InvalidateNamespaceAsync(string domainNamespace, CancellationToken ct = default)
    {
        var query = new Query($"@namespace:{{{domainNamespace}}}")
            .ReturnFields("id")
            .Limit(0, 1000);

        var results = await _search.SearchAsync(IndexName, query).ConfigureAwait(false);
        if (results.TotalResults > 0)
        {
            var keys = new RedisKey[results.Documents.Count];
            for (int i = 0; i < results.Documents.Count; i++)
            {
                keys[i] = results.Documents[i].Id;
            }

            await _db.KeyDeleteAsync(keys).ConfigureAwait(false);
            _logger.LogWarning("Purged {Count} semantic cache records for namespace {Ns}", keys.Length, domainNamespace);
        }
    }
}

using System;
using System.Collections.Concurrent;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;

namespace Centralogic.AI.SemanticCache;

public sealed class TwoTierSemanticCacheDecorator<TResponse>
{
    private readonly IMemoryCache _l1Cache;
    private readonly ISemanticCacheManager<TResponse> _l2Redis;
    private readonly ILogger<TwoTierSemanticCacheDecorator<TResponse>> _logger;
    private readonly ConcurrentDictionary<string, SemaphoreSlim> _locks = new();

    public TwoTierSemanticCacheDecorator(
        IMemoryCache l1Cache,
        ISemanticCacheManager<TResponse> l2Redis,
        ILogger<TwoTierSemanticCacheDecorator<TResponse>> logger)
    {
        _l1Cache = l1Cache;
        _l2Redis = l2Redis;
        _logger = logger;
    }

    public async ValueTask<TResponse> GetOrComputeAsync(
        string normalizedPrompt,
        ReadOnlyMemory<float> embeddingVector,
        string domainNamespace,
        Func<CancellationToken, Task<TResponse>> factory,
        TimeSpan? l2Ttl = null,
        CancellationToken ct = default)
    {
        string l1Key = $"l1:{domainNamespace}:{normalizedPrompt}";

        // 1. Fast Path: Check L1 Local In-Memory Cache
        if (_l1Cache.TryGetValue(l1Key, out TResponse? localHit) && localHit is not null)
        {
            _logger.LogDebug("L1 In-Memory Semantic Cache Hit for prompt: '{Prompt}'", normalizedPrompt);
            return localHit;
        }

        // 2. Check L2 Redis Vector Store with HNSW Cosine Search
        var match = await _l2Redis.FindSimilarAsync(embeddingVector, domainNamespace, 0.94f, ct).ConfigureAwait(false);
        if (match.HasValue)
        {
            _logger.LogInformation(
                "L2 Redis Vector Cache Hit! Matched: '{Matched}' (Sim: {Score:F3}) for query '{Query}'",
                match.Value.MatchedPrompt, match.Value.SimilarityScore, normalizedPrompt);

            // Populate L1 cache with shorter TTL (5 minutes) to shield Redis from hot queries
            _l1Cache.Set(l1Key, match.Value.Payload, TimeSpan.FromMinutes(5));
            return match.Value.Payload;
        }

        // 3. Stampede Prevention Guard: Lock around exact intent execution
        var semaphore = _locks.GetOrAdd(normalizedPrompt, _ => new SemaphoreSlim(1, 1));
        await semaphore.WaitAsync(ct).ConfigureAwait(false);

        try
        {
            // Re-verify L2 after acquiring lock to catch in-flight completions
            var doubleCheck = await _l2Redis.FindSimilarAsync(embeddingVector, domainNamespace, 0.94f, ct).ConfigureAwait(false);
            if (doubleCheck.HasValue)
            {
                _l1Cache.Set(l1Key, doubleCheck.Value.Payload, TimeSpan.FromMinutes(5));
                return doubleCheck.Value.Payload;
            }

            // Execute expensive LLM inference via factory
            _logger.LogInformation("Cache Miss. Invoking Azure OpenAI LLM for prompt: '{Prompt}'", normalizedPrompt);
            TResponse computed = await factory(ct).ConfigureAwait(false);

            // Store in L2 Redis and L1 Memory
            var ttl = l2Ttl ?? TimeSpan.FromHours(24);
            await _l2Redis.StoreAsync(normalizedPrompt, embeddingVector, computed, domainNamespace, ttl, ct).ConfigureAwait(false);
            _l1Cache.Set(l1Key, computed, TimeSpan.FromMinutes(5));

            return computed;
        }
        finally
        {
            semaphore.Release();
            if (semaphore.CurrentCount == 1)
            {
                _locks.TryRemove(normalizedPrompt, out _);
            }
        }
    }
}