S&Box LLM Proof of Concept
Released
Proof of concept for running a real LLM entirely inside s&box using a custom C# GPT-Neo inference engine. No APIS, fully local
Llm/ILanguageModelService.cs
Interface and supporting types for a local language model service. Defines service states, a LanguageModelGenerationResult record with timing and token info, and ILanguageModelService with properties and async Initialize/Generate methods.
namespace LlmPoc.Llm;
public enum LanguageModelServiceState
{
NotLoaded,
Loading,
Ready,
Generating,
Error
}
public sealed class LanguageModelGenerationResult
{
public string Prompt { get; init; }
public int[] InputTokenIds { get; init; }
public int[] GeneratedTokenIds { get; init; }
public string GeneratedText { get; init; }
public string StopReason { get; init; }
public bool EosReached { get; init; }
public int MaximumNewTokens { get; init; }
public double TokenizationMilliseconds { get; init; }
public double FirstTokenMilliseconds { get; init; }
public double TotalForwardMilliseconds { get; init; }
public double TotalGenerationMilliseconds { get; init; }
public double TotalRequestMilliseconds { get; init; }
public double TokensPerSecond => TotalGenerationMilliseconds > 0
? GeneratedTokenIds.Length * 1000.0 / TotalGenerationMilliseconds
: 0;
}
public interface ILanguageModelService
{
string DisplayName { get; }
bool IsRealInference { get; }
LanguageModelServiceState State { get; }
string ErrorMessage { get; }
bool CanGenerate { get; }
int DefaultMaxNewTokens { get; }
double InitializationMilliseconds { get; }
LanguageModelGenerationResult LastResult { get; }
Task InitializeAsync();
Task<LanguageModelGenerationResult> GenerateAsync(
string prompt,
int? maxNewTokens = null );
}