S&Box LLM Proof of Concept
Released
Proof of concept for running a real LLM entirely inside s&box using a custom C# GPT-Neo inference engine. No APIS, fully local
Llm/TinyStoriesForwardStages.cs
Utility class implementing forward-pass stages for a TinyStories GPT-Neo style model. It provides tensor helpers: embedding combination, layer norms, linear projections, GELU-new activation, attention head split/merge, scaled attention score computation, masking, softmax, context computation and various diagnostic/description helpers.
namespace LlmPoc.Llm;
public static class TinyStoriesForwardStages
{
public const string TokenEmbeddingName = "transformer.wte.weight";
public const string PositionEmbeddingName = "transformer.wpe.weight";
public const string Layer0Ln1WeightName = "transformer.h.0.ln_1.weight";
public const string Layer0Ln1BiasName = "transformer.h.0.ln_1.bias";
public const string Layer0QWeightName = "transformer.h.0.attn.attention.q_proj.weight";
public const string Layer0KWeightName = "transformer.h.0.attn.attention.k_proj.weight";
public const string Layer0VWeightName = "transformer.h.0.attn.attention.v_proj.weight";
public const string Layer0AttentionOutProjectionWeightName =
"transformer.h.0.attn.attention.out_proj.weight";
public const string Layer0AttentionOutProjectionBiasName =
"transformer.h.0.attn.attention.out_proj.bias";
public const string Layer0Ln2WeightName = "transformer.h.0.ln_2.weight";
public const string Layer0Ln2BiasName = "transformer.h.0.ln_2.bias";
public const string Layer0MlpFcWeightName = "transformer.h.0.mlp.c_fc.weight";
public const string Layer0MlpFcBiasName = "transformer.h.0.mlp.c_fc.bias";
public const string Layer0MlpProjWeightName = "transformer.h.0.mlp.c_proj.weight";
public const string Layer0MlpProjBiasName = "transformer.h.0.mlp.c_proj.bias";
public const float GeluNewTanhCoefficient = 0.7978845608028654f;
public const float GeluNewCubicCoefficient = 0.044715f;
public const string GeluNewFormula =
"0.5*x*(1.0+tanh(sqrt(2.0/pi)*(x+0.044715*pow(x,3.0))))";
public static Tensor CombineEmbeddings(
SboxLlmModel model,
TinyStoriesConfig config,
IReadOnlyList<int> tokenIds,
bool logDiagnostics = true )
{
if ( model is null )
{
throw new ArgumentNullException( nameof( model ) );
}
if ( config is null )
{
throw new ArgumentNullException( nameof( config ) );
}
if ( tokenIds is null || tokenIds.Count == 0 )
{
throw new ArgumentException( "[LLM:ERROR] Embedding input token IDs cannot be empty." );
}
if ( tokenIds.Count > config.MaximumPositions )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Embedding sequence length {tokenIds.Count} exceeds " +
$"maximum positions {config.MaximumPositions}." );
}
Tensor tokenEmbedding = model.GetRequiredTensor( TokenEmbeddingName );
Tensor positionEmbedding = model.GetRequiredTensor( PositionEmbeddingName );
tokenEmbedding.RequireShape( config.VocabularySize, config.HiddenSize );
positionEmbedding.RequireShape( config.MaximumPositions, config.HiddenSize );
if ( tokenIds.Count > int.MaxValue / config.HiddenSize )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Embedding output shape [{tokenIds.Count},{config.HiddenSize}] " +
"exceeds the managed array limit." );
}
if ( logDiagnostics )
{
LlmLog.Info(
"EMBED",
$"input shape=[{tokenIds.Count}] tokens=[{string.Join( ",", tokenIds )}] " +
$"positions=[0..{tokenIds.Count - 1}] token_tensor={TokenEmbeddingName}{tokenEmbedding.ShapeText} " +
$"position_tensor={PositionEmbeddingName}{positionEmbedding.ShapeText}" );
}
int hiddenSize = config.HiddenSize;
float[] output = new float[tokenIds.Count * hiddenSize];
for ( int position = 0; position < tokenIds.Count; position++ )
{
int tokenId = tokenIds[position];
if ( tokenId < 0 || tokenId >= config.VocabularySize )
{
throw new IndexOutOfRangeException(
$"[LLM:ERROR] Embedding token ID {tokenId} at sequence index {position} " +
$"is outside [0,{config.VocabularySize})." );
}
int tokenRow = tokenId * hiddenSize;
int positionRow = position * hiddenSize;
int outputRow = position * hiddenSize;
for ( int hidden = 0; hidden < hiddenSize; hidden++ )
{
output[outputRow + hidden] =
tokenEmbedding.Data[tokenRow + hidden] +
positionEmbedding.Data[positionRow + hidden];
}
}
return new Tensor(
"forward.combined_embedding",
new[] { tokenIds.Count, hiddenSize },
output );
}
public static Tensor ApplyLayer0Ln1(
SboxLlmModel model,
TinyStoriesConfig config,
Tensor input )
{
return ApplyLayerNorm(
model,
config,
input,
Layer0Ln1WeightName,
Layer0Ln1BiasName,
"layer=0 ln=1",
"Layer 0 ln_1",
"forward.layer0.ln_1" );
}
public static Tensor ApplyLayer0Ln2(
SboxLlmModel model,
TinyStoriesConfig config,
Tensor input )
{
return ApplyLayerNorm(
model,
config,
input,
Layer0Ln2WeightName,
Layer0Ln2BiasName,
"layer=0 ln=2",
"Layer 0 ln_2",
"forward.layer0.ln_2" );
}
public static Tensor ApplyLayerNorm(
SboxLlmModel model,
TinyStoriesConfig config,
Tensor input,
int layerIndex,
int normIndex,
string outputName,
bool logDiagnostics = true )
{
if ( layerIndex < 0 || layerIndex >= config.LayerCount )
{
throw new IndexOutOfRangeException(
$"[LLM:ERROR] LayerNorm layer index {layerIndex} is outside " +
$"[0,{config.LayerCount})." );
}
if ( normIndex != 1 && normIndex != 2 )
{
throw new ArgumentOutOfRangeException(
nameof( normIndex ), normIndex,
"[LLM:ERROR] GPT-Neo transformer LayerNorm index must be 1 or 2." );
}
string prefix = $"transformer.h.{layerIndex}.ln_{normIndex}";
return ApplyLayerNorm(
model,
config,
input,
$"{prefix}.weight",
$"{prefix}.bias",
$"layer={layerIndex} ln={normIndex}",
$"Layer {layerIndex} ln_{normIndex}",
outputName,
logDiagnostics );
}
public static Tensor ApplyFinalLayerNorm(
SboxLlmModel model,
TinyStoriesConfig config,
Tensor input,
string outputName,
bool logDiagnostics = true )
{
return ApplyLayerNorm(
model,
config,
input,
"transformer.ln_f.weight",
"transformer.ln_f.bias",
"stage=final_ln",
"Final model LayerNorm",
outputName,
logDiagnostics );
}
private static Tensor ApplyLayerNorm(
SboxLlmModel model,
TinyStoriesConfig config,
Tensor input,
string weightName,
string biasName,
string logLabel,
string errorLabel,
string outputName,
bool logDiagnostics = true )
{
if ( model is null ) throw new ArgumentNullException( nameof( model ) );
if ( config is null ) throw new ArgumentNullException( nameof( config ) );
if ( input is null ) throw new ArgumentNullException( nameof( input ) );
if ( input.Rank != 2 || input.Shape[0] <= 0 || input.Shape[1] != config.HiddenSize )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {errorLabel} input expected shape [sequence,{config.HiddenSize}] " +
$"with a positive sequence length, found {input.ShapeText}." );
}
Tensor weight = model.GetRequiredTensor( weightName );
Tensor bias = model.GetRequiredTensor( biasName );
weight.RequireShape( config.HiddenSize );
bias.RequireShape( config.HiddenSize );
if ( !(config.LayerNormEpsilon > 0) || !float.IsFinite( config.LayerNormEpsilon ) )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {errorLabel} epsilon must be positive and finite, " +
$"found {config.LayerNormEpsilon:G9}." );
}
int sequenceLength = input.Shape[0];
int hiddenSize = config.HiddenSize;
float[] output = new float[input.Data.Length];
if ( logDiagnostics )
{
LlmLog.Info(
"LN",
$"{logLabel} input_shape={input.ShapeText} expected_shape=[{sequenceLength},{hiddenSize}] " +
$"weight={weightName}{weight.ShapeText} bias={biasName}{bias.ShapeText} " +
$"epsilon={config.LayerNormEpsilon:G9} variance=population(unbiased=false)" );
}
for ( int token = 0; token < sequenceLength; token++ )
{
int row = token * hiddenSize;
// PyTorch 2.9.1's installed AVX2 LayerNorm kernel uses RowwiseMoments:
// eight FP32 Welford lanes over this model's 64 hidden values, followed
// by a left-to-right cascade of the lane moments. This scalar spelling
// reproduces that numerical reduction order without introducing SIMD.
(float mean, float variance) = ComputeAvx2RowwiseMoments64( input.Data, row );
float denominator = MathF.Sqrt( variance + config.LayerNormEpsilon );
if ( !(denominator > 0) || !float.IsFinite( denominator ) )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {errorLabel} token {token} produced invalid denominator " +
$"{denominator:G9} from mean={mean:G9}, variance={variance:G9}, " +
$"epsilon={config.LayerNormEpsilon:G9}." );
}
float inverseStandardDeviation = 1.0f / denominator;
for ( int hidden = 0; hidden < hiddenSize; hidden++ )
{
float normalized = (input.Data[row + hidden] - mean) * inverseStandardDeviation;
output[row + hidden] = normalized * weight.Data[hidden] + bias.Data[hidden];
}
if ( logDiagnostics && token == 0 )
{
LlmLog.Trace(
"LN",
$"{logLabel} token=0 mean={mean:G12} variance={variance:G12} " +
$"denominator={denominator:G12} inverse_std={inverseStandardDeviation:G12} " +
$"output_first=[{output[row]:G9},{output[row + 1]:G9}," +
$"{output[row + 2]:G9},{output[row + 3]:G9}]" );
}
}
return new Tensor(
outputName,
new[] { sequenceLength, hiddenSize },
output );
}
private static (float Mean, float Variance) ComputeAvx2RowwiseMoments64(
float[] values,
int rowOffset )
{
const int laneCount = 8;
const int valuesPerLane = 8;
float[] laneMeans = new float[laneCount];
float[] laneMoment2 = new float[laneCount];
for ( int item = 0; item < valuesPerLane; item++ )
{
float reciprocalCount = 1.0f / (item + 1);
int itemOffset = rowOffset + item * laneCount;
for ( int lane = 0; lane < laneCount; lane++ )
{
float value = values[itemOffset + lane];
float delta = value - laneMeans[lane];
float meanIncrement = delta * reciprocalCount;
laneMeans[lane] += meanIncrement;
float remainingDelta = value - laneMeans[lane];
float momentIncrement = delta * remainingDelta;
laneMoment2[lane] += momentIncrement;
}
}
int accumulatedCount = 0;
float mean = 0.0f;
float moment2 = 0.0f;
for ( int lane = 0; lane < laneCount; lane++ )
{
int combinedCount = accumulatedCount + valuesPerLane;
float contribution = (float)valuesPerLane / combinedCount;
float delta = laneMeans[lane] - mean;
float meanIncrement = contribution * delta;
mean += meanIncrement;
float deltaSquared = delta * delta;
float weightedDelta = deltaSquared * contribution;
weightedDelta *= accumulatedCount;
float combinedMoment = laneMoment2[lane] + weightedDelta;
moment2 += combinedMoment;
accumulatedCount = combinedCount;
}
return (mean, moment2 / 64.0f);
}
public static Tensor LinearNoBias( Tensor input, Tensor weight, string outputName )
{
return Linear( input, weight, null, outputName );
}
public static Tensor LinearWithBias(
Tensor input,
Tensor weight,
Tensor bias,
string outputName )
{
if ( bias is null )
{
throw new ArgumentNullException( nameof( bias ) );
}
return Linear( input, weight, bias, outputName );
}
private static Tensor Linear(
Tensor input,
Tensor weight,
Tensor bias,
string outputName )
{
if ( input is null )
{
throw new ArgumentNullException( nameof( input ) );
}
if ( weight is null )
{
throw new ArgumentNullException( nameof( weight ) );
}
if ( string.IsNullOrWhiteSpace( outputName ) )
{
throw new ArgumentException(
"[LLM:ERROR] Linear projection output name cannot be empty.",
nameof( outputName ) );
}
if ( input.Rank != 2 )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} linear input expected rank 2 " +
$"[sequence,input_size], found {input.ShapeText}." );
}
if ( weight.Rank != 2 )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} weight '{weight.Name}' expected rank 2 " +
$"[output_size,input_size], found {weight.ShapeText}." );
}
int sequenceLength = input.Shape[0];
int inputSize = input.Shape[1];
int outputSize = weight.Shape[0];
if ( weight.Shape[1] != inputSize )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} incompatible linear shapes: input={input.ShapeText}, " +
$"weight={weight.ShapeText}; weight input axis expected {inputSize}, " +
$"found {weight.Shape[1]}." );
}
if ( bias is not null )
{
if ( bias.Rank != 1 || bias.Shape[0] != outputSize )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} linear bias '{bias.Name}' expected shape " +
$"[{outputSize}], found {bias.ShapeText}." );
}
}
if ( sequenceLength > int.MaxValue / outputSize )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} output shape [{sequenceLength},{outputSize}] " +
"exceeds the managed array limit." );
}
float[] output = new float[sequenceLength * outputSize];
for ( int token = 0; token < sequenceLength; token++ )
{
int inputRow = token * inputSize;
int outputRow = token * outputSize;
for ( int outputFeature = 0; outputFeature < outputSize; outputFeature++ )
{
int weightRow = outputFeature * inputSize;
// The installed PyTorch 2.9.1 CPU Linear path was independently
// checked against the live module outputs. For this model it is
// bit-identical to a left-to-right FP32 fused multiply-add reduction.
// The small attention matmuls intentionally retain their separately
// validated non-fused multiply/add loops.
float sum = 0.0f;
for ( int inputFeature = 0; inputFeature < inputSize; inputFeature++ )
{
sum = MathF.FusedMultiplyAdd(
input.Data[inputRow + inputFeature],
weight.Data[weightRow + inputFeature],
sum );
}
output[outputRow + outputFeature] =
bias is null ? sum : sum + bias.Data[outputFeature];
}
}
return new Tensor(
outputName,
new[] { sequenceLength, outputSize },
output );
}
public static string DescribeLinearDotProduct(
Tensor input,
Tensor weight,
int token,
int outputFeature,
float expected,
float actual,
Tensor bias = null )
{
if ( input is null || weight is null || input.Rank != 2 || weight.Rank != 2 ||
weight.Shape[1] != input.Shape[1] )
{
throw new InvalidOperationException(
"[LLM:ERROR] Dot-product diagnostic requires compatible rank-2 input and weight tensors." );
}
if ( bias is not null && (bias.Rank != 1 || bias.Shape[0] != weight.Shape[0]) )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Dot-product diagnostic bias '{bias.Name}' expected shape " +
$"[{weight.Shape[0]}], found {bias.ShapeText}." );
}
if ( token < 0 || token >= input.Shape[0] ||
outputFeature < 0 || outputFeature >= weight.Shape[0] )
{
throw new IndexOutOfRangeException(
$"[LLM:ERROR] Dot-product diagnostic index token={token}, " +
$"output_feature={outputFeature} is outside input={input.ShapeText}, " +
$"weight={weight.ShapeText}." );
}
int inputSize = input.Shape[1];
int inputRow = token * inputSize;
int weightRow = outputFeature * inputSize;
float sum = 0;
float product0 = input.Data[inputRow] * weight.Data[weightRow];
float product1 = input.Data[inputRow + 1] * weight.Data[weightRow + 1];
float product2 = input.Data[inputRow + 2] * weight.Data[weightRow + 2];
float product3 = input.Data[inputRow + 3] * weight.Data[weightRow + 3];
for ( int inputFeature = 0; inputFeature < inputSize; inputFeature++ )
{
sum = MathF.FusedMultiplyAdd(
input.Data[inputRow + inputFeature],
weight.Data[weightRow + inputFeature],
sum );
}
float biasValue = bias is null ? 0 : bias.Data[outputFeature];
float recomputed = bias is null ? sum : sum + biasValue;
string formula = bias is null
? "sum(input[token,i]*weight[out,i])"
: "sum(input[token,i]*weight[out,i])+bias[out]";
return $"token={token} output_feature={outputFeature} input_length={inputSize} " +
$"formula={formula} expected={expected:G9} actual={actual:G9} " +
$"dot={sum:G9} bias={biasValue:G9} recomputed={recomputed:G9} " +
$"first_products=[{product0:G9},{product1:G9},{product2:G9},{product3:G9}]";
}
public static Tensor ApplyGeluNew( Tensor input, string outputName )
{
if ( input is null )
{
throw new ArgumentNullException( nameof( input ) );
}
if ( string.IsNullOrWhiteSpace( outputName ) )
{
throw new ArgumentException(
"[LLM:ERROR] GELU-new output name cannot be empty.",
nameof( outputName ) );
}
if ( input.Rank != 2 || input.Shape[0] <= 0 || input.Shape[1] <= 0 )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} GELU-new input expected a positive rank-2 " +
$"[sequence,feature] tensor, found {input.ShapeText}." );
}
float[] output = new float[input.Data.Length];
for ( int index = 0; index < input.Data.Length; index++ )
{
float value = input.Data[index];
if ( !float.IsFinite( value ) )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} GELU-new input contains non-finite " +
$"value at flat index {index}: {value}." );
}
float cube = MathF.Pow( value, 3.0f );
float inner = value + GeluNewCubicCoefficient * cube;
float tanhArgument = GeluNewTanhCoefficient * inner;
float tanhValue = MathF.Tanh( tanhArgument );
float activated = (0.5f * value) * (1.0f + tanhValue);
if ( !float.IsFinite( activated ) )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} GELU-new produced non-finite output " +
$"at flat index {index}: input={value:G9}, cube={cube:G9}, " +
$"tanh_argument={tanhArgument:G9}, output={activated}." );
}
output[index] = activated;
}
return new Tensor(
outputName,
new[] { input.Shape[0], input.Shape[1] },
output );
}
public static string DescribeGeluNewElement(
Tensor input,
Tensor output,
int token,
int feature,
float expected )
{
if ( input is null || output is null || input.Rank != 2 || output.Rank != 2 ||
input.Shape[0] != output.Shape[0] || input.Shape[1] != output.Shape[1] )
{
throw new InvalidOperationException(
"[LLM:ERROR] GELU-new diagnostic requires matching rank-2 tensors." );
}
if ( token < 0 || token >= input.Shape[0] || feature < 0 || feature >= input.Shape[1] )
{
throw new IndexOutOfRangeException(
$"[LLM:ERROR] GELU-new diagnostic index [{token},{feature}] is " +
$"outside {input.ShapeText}." );
}
int index = token * input.Shape[1] + feature;
float value = input.Data[index];
float cube = MathF.Pow( value, 3.0f );
float inner = value + GeluNewCubicCoefficient * cube;
float tanhArgument = GeluNewTanhCoefficient * inner;
float tanhValue = MathF.Tanh( tanhArgument );
float recomputed = (0.5f * value) * (1.0f + tanhValue);
float actual = output.Data[index];
return $"token={token} feature={feature} input={value:G9} cube={cube:G9} " +
$"cubic_coefficient={GeluNewCubicCoefficient:G9} inner={inner:G9} " +
$"tanh_coefficient={GeluNewTanhCoefficient:G9} " +
$"tanh_argument={tanhArgument:G9} tanh={tanhValue:G9} " +
$"recomputed={recomputed:G9} actual={actual:G9} python={expected:G9} " +
$"abs_diff={MathF.Abs( actual - expected ):G12}";
}
public static Tensor AddMlpResidual(
Tensor residualSource,
Tensor mlpBranch,
string outputName )
{
if ( residualSource is null )
{
throw new ArgumentNullException( nameof( residualSource ) );
}
if ( mlpBranch is null )
{
throw new ArgumentNullException( nameof( mlpBranch ) );
}
if ( string.IsNullOrWhiteSpace( outputName ) )
{
throw new ArgumentException(
"[LLM:ERROR] MLP residual output name cannot be empty.",
nameof( outputName ) );
}
if ( residualSource.Rank != 2 || mlpBranch.Rank != 2 ||
residualSource.Shape[0] != mlpBranch.Shape[0] ||
residualSource.Shape[1] != mlpBranch.Shape[1] ||
residualSource.Data.Length != mlpBranch.Data.Length )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} MLP residual addition requires matching " +
$"rank-2 tensors, found residual={residualSource.ShapeText}, " +
$"MLP={mlpBranch.ShapeText}." );
}
float[] output = new float[residualSource.Data.Length];
for ( int index = 0; index < output.Length; index++ )
{
// Match GPTNeoBlock.forward: residual + feed_forward_hidden_states.
output[index] = residualSource.Data[index] + mlpBranch.Data[index];
}
return new Tensor(
outputName,
new[] { residualSource.Shape[0], residualSource.Shape[1] },
output );
}
public static string DescribeMlpResidualAddition(
Tensor residualSource,
Tensor mlpBranch,
Tensor output,
int token,
int feature,
float expected )
{
if ( residualSource is null || mlpBranch is null || output is null ||
residualSource.Rank != 2 || mlpBranch.Rank != 2 || output.Rank != 2 ||
residualSource.Shape[0] != mlpBranch.Shape[0] ||
residualSource.Shape[1] != mlpBranch.Shape[1] ||
residualSource.Shape[0] != output.Shape[0] ||
residualSource.Shape[1] != output.Shape[1] )
{
throw new InvalidOperationException(
"[LLM:ERROR] MLP residual diagnostic requires matching rank-2 tensors." );
}
if ( token < 0 || token >= output.Shape[0] || feature < 0 || feature >= output.Shape[1] )
{
throw new IndexOutOfRangeException(
$"[LLM:ERROR] MLP residual diagnostic index [{token},{feature}] is " +
$"outside {output.ShapeText}." );
}
int index = token * output.Shape[1] + feature;
float residual = residualSource.Data[index];
float branch = mlpBranch.Data[index];
float actual = output.Data[index];
return $"token={token} feature={feature} residual_source={residual:G9} " +
$"mlp_branch={branch:G9} sum={actual:G9} python={expected:G9} " +
$"abs_diff={MathF.Abs( expected - actual ):G12}";
}
public static Tensor AddResidual(
Tensor attentionBranch,
Tensor residualSource,
string outputName )
{
if ( attentionBranch is null )
{
throw new ArgumentNullException( nameof( attentionBranch ) );
}
if ( residualSource is null )
{
throw new ArgumentNullException( nameof( residualSource ) );
}
if ( string.IsNullOrWhiteSpace( outputName ) )
{
throw new ArgumentException(
"[LLM:ERROR] Residual-add output name cannot be empty.",
nameof( outputName ) );
}
if ( attentionBranch.Rank != 2 || residualSource.Rank != 2 ||
attentionBranch.Shape[0] != residualSource.Shape[0] ||
attentionBranch.Shape[1] != residualSource.Shape[1] ||
attentionBranch.Data.Length != residualSource.Data.Length )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} residual addition requires matching rank-2 " +
$"tensors, found attention={attentionBranch.ShapeText}, " +
$"residual={residualSource.ShapeText}." );
}
float[] output = new float[attentionBranch.Data.Length];
for ( int index = 0; index < output.Length; index++ )
{
output[index] = attentionBranch.Data[index] + residualSource.Data[index];
}
return new Tensor(
outputName,
new[] { attentionBranch.Shape[0], attentionBranch.Shape[1] },
output );
}
public static string DescribeResidualAddition(
Tensor attentionBranch,
Tensor residualSource,
Tensor output,
int token,
int feature,
float expected )
{
if ( attentionBranch is null || residualSource is null || output is null ||
attentionBranch.Rank != 2 || residualSource.Rank != 2 || output.Rank != 2 ||
attentionBranch.Shape[0] != residualSource.Shape[0] ||
attentionBranch.Shape[1] != residualSource.Shape[1] ||
attentionBranch.Shape[0] != output.Shape[0] ||
attentionBranch.Shape[1] != output.Shape[1] )
{
throw new InvalidOperationException(
"[LLM:ERROR] Residual diagnostic requires compatible rank-2 attention, " +
"residual, and output tensors." );
}
if ( token < 0 || token >= output.Shape[0] ||
feature < 0 || feature >= output.Shape[1] )
{
throw new IndexOutOfRangeException(
$"[LLM:ERROR] Residual diagnostic index [{token},{feature}] is " +
$"outside output shape {output.ShapeText}." );
}
int index = token * output.Shape[1] + feature;
float branch = attentionBranch.Data[index];
float residual = residualSource.Data[index];
float actual = output.Data[index];
return $"token={token} feature={feature} residual_source={residual:G9} " +
$"attention_branch={branch:G9} sum={actual:G9} python={expected:G9} " +
$"abs_diff={MathF.Abs( expected - actual ):G12}";
}
public static Tensor SplitHeads( Tensor projection, int headCount, string outputName )
{
if ( projection is null )
{
throw new ArgumentNullException( nameof( projection ) );
}
if ( string.IsNullOrWhiteSpace( outputName ) )
{
throw new ArgumentException(
"[LLM:ERROR] Head-split output name cannot be empty.",
nameof( outputName ) );
}
if ( projection.Rank != 2 || projection.Shape[0] <= 0 || projection.Shape[1] <= 0 )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} head split expected projection shape " +
$"[sequence,hidden] with positive dimensions, found {projection.ShapeText}." );
}
if ( headCount <= 0 || projection.Shape[1] % headCount != 0 )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} hidden size {projection.Shape[1]} must be divisible " +
$"by positive head count {headCount}." );
}
int sequenceLength = projection.Shape[0];
int hiddenSize = projection.Shape[1];
int headDimension = hiddenSize / headCount;
float[] output = new float[projection.Data.Length];
for ( int head = 0; head < headCount; head++ )
{
for ( int token = 0; token < sequenceLength; token++ )
{
int sourceRow = token * hiddenSize;
int destinationRow = (head * sequenceLength + token) * headDimension;
for ( int component = 0; component < headDimension; component++ )
{
int sourceFeature = head * headDimension + component;
output[destinationRow + component] = projection.Data[sourceRow + sourceFeature];
}
}
}
return new Tensor(
outputName,
new[] { headCount, sequenceLength, headDimension },
output );
}
public static string DescribeHeadMapping(
Tensor projection,
Tensor heads,
int head,
int token,
int component )
{
if ( projection is null || heads is null || projection.Rank != 2 || heads.Rank != 3 )
{
throw new InvalidOperationException(
"[LLM:ERROR] Head mapping diagnostic requires rank-2 projection and rank-3 heads." );
}
int headCount = heads.Shape[0];
int sequenceLength = heads.Shape[1];
int headDimension = heads.Shape[2];
if ( projection.Shape[0] != sequenceLength ||
projection.Shape[1] != headCount * headDimension )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Head mapping diagnostic incompatible shapes: " +
$"projection={projection.ShapeText}, heads={heads.ShapeText}." );
}
if ( head < 0 || head >= headCount || token < 0 || token >= sequenceLength ||
component < 0 || component >= headDimension )
{
throw new IndexOutOfRangeException(
$"[LLM:ERROR] Head mapping diagnostic index [{head},{token},{component}] " +
$"is outside {heads.ShapeText}." );
}
int sourceFeature = head * headDimension + component;
int sourceIndex = token * projection.Shape[1] + sourceFeature;
int destinationIndex = (head * sequenceLength + token) * headDimension + component;
float source = projection.Data[sourceIndex];
float destination = heads.Data[destinationIndex];
return $"head={head} token={token} component={component} source_feature={sourceFeature} " +
$"source_index={sourceIndex} destination_index={destinationIndex} " +
$"source={source:G9} destination={destination:G9} " +
$"source_bits={BitConverter.SingleToInt32Bits( source ):X8} " +
$"destination_bits={BitConverter.SingleToInt32Bits( destination ):X8}";
}
public static Tensor MergeHeads( Tensor contextHeads, string outputName )
{
if ( contextHeads is null )
{
throw new ArgumentNullException( nameof( contextHeads ) );
}
if ( string.IsNullOrWhiteSpace( outputName ) )
{
throw new ArgumentException(
"[LLM:ERROR] Head-merge output name cannot be empty.",
nameof( outputName ) );
}
if ( contextHeads.Rank != 3 || contextHeads.Shape[0] <= 0 ||
contextHeads.Shape[1] <= 0 || contextHeads.Shape[2] <= 0 )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} head merge expected positive shape " +
$"[head,sequence,head_dimension], found {contextHeads.ShapeText}." );
}
int headCount = contextHeads.Shape[0];
int sequenceLength = contextHeads.Shape[1];
int headDimension = contextHeads.Shape[2];
if ( headCount > int.MaxValue / headDimension )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} hidden size {headCount}*{headDimension} " +
"exceeds the managed array limit." );
}
int hiddenSize = headCount * headDimension;
if ( sequenceLength > int.MaxValue / hiddenSize )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} output shape [{sequenceLength},{hiddenSize}] " +
"exceeds the managed array limit." );
}
float[] output = new float[sequenceLength * hiddenSize];
for ( int token = 0; token < sequenceLength; token++ )
{
int destinationRow = token * hiddenSize;
for ( int head = 0; head < headCount; head++ )
{
int sourceRow = (head * sequenceLength + token) * headDimension;
int destinationHead = destinationRow + head * headDimension;
for ( int component = 0; component < headDimension; component++ )
{
output[destinationHead + component] =
contextHeads.Data[sourceRow + component];
}
}
}
return new Tensor(
outputName,
new[] { sequenceLength, hiddenSize },
output );
}
public static string DescribeHeadMergeMapping(
Tensor contextHeads,
Tensor merged,
int token,
int feature )
{
if ( contextHeads is null || merged is null ||
contextHeads.Rank != 3 || merged.Rank != 2 )
{
throw new InvalidOperationException(
"[LLM:ERROR] Head-merge diagnostic requires rank-3 context heads and " +
"rank-2 merged context." );
}
int headCount = contextHeads.Shape[0];
int sequenceLength = contextHeads.Shape[1];
int headDimension = contextHeads.Shape[2];
int hiddenSize = headCount * headDimension;
if ( merged.Shape[0] != sequenceLength || merged.Shape[1] != hiddenSize )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Head-merge diagnostic incompatible shapes: " +
$"context={contextHeads.ShapeText}, merged={merged.ShapeText}." );
}
if ( token < 0 || token >= sequenceLength || feature < 0 || feature >= hiddenSize )
{
throw new IndexOutOfRangeException(
$"[LLM:ERROR] Head-merge diagnostic index token={token}, feature={feature} " +
$"is outside merged shape {merged.ShapeText}." );
}
int head = feature / headDimension;
int component = feature % headDimension;
int sourceIndex = (head * sequenceLength + token) * headDimension + component;
int destinationIndex = token * hiddenSize + feature;
float source = contextHeads.Data[sourceIndex];
float destination = merged.Data[destinationIndex];
return $"token={token} feature={feature} head={head} component={component} " +
$"source_index={sourceIndex} destination_index={destinationIndex} " +
$"source={source:G9} merged={destination:G9} " +
$"source_bits={BitConverter.SingleToInt32Bits( source ):X8} " +
$"merged_bits={BitConverter.SingleToInt32Bits( destination ):X8}";
}
public static Tensor ComputeScaledUnmaskedAttentionScores(
Tensor queryHeads,
Tensor keyHeads,
float scale,
string outputName )
{
if ( queryHeads is null )
{
throw new ArgumentNullException( nameof( queryHeads ) );
}
if ( keyHeads is null )
{
throw new ArgumentNullException( nameof( keyHeads ) );
}
if ( string.IsNullOrWhiteSpace( outputName ) )
{
throw new ArgumentException(
"[LLM:ERROR] Attention-score output name cannot be empty.",
nameof( outputName ) );
}
if ( queryHeads.Rank != 3 || keyHeads.Rank != 3 )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} expected Q/K heads rank 3 " +
$"[head,sequence,component], found Q={queryHeads.ShapeText}, " +
$"K={keyHeads.ShapeText}." );
}
if ( queryHeads.Shape[0] != keyHeads.Shape[0] ||
queryHeads.Shape[2] != keyHeads.Shape[2] )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} Q/K head count and component dimensions must match; " +
$"found Q={queryHeads.ShapeText}, K={keyHeads.ShapeText}." );
}
if ( !float.IsFinite( scale ) || !(scale > 0) )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} attention scale must be positive and finite, " +
$"found {scale:G9}." );
}
int headCount = queryHeads.Shape[0];
int queryLength = queryHeads.Shape[1];
int keyLength = keyHeads.Shape[1];
int headDimension = queryHeads.Shape[2];
if ( headCount > int.MaxValue / queryLength ||
headCount * queryLength > int.MaxValue / keyLength )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} shape [{headCount},{queryLength},{keyLength}] " +
"exceeds the managed array limit." );
}
float[] output = new float[headCount * queryLength * keyLength];
for ( int head = 0; head < headCount; head++ )
{
for ( int query = 0; query < queryLength; query++ )
{
int queryRow = (head * queryLength + query) * headDimension;
for ( int key = 0; key < keyLength; key++ )
{
int keyRow = (head * keyLength + key) * headDimension;
float rawSum = 0;
for ( int component = 0; component < headDimension; component++ )
{
rawSum += queryHeads.Data[queryRow + component] *
keyHeads.Data[keyRow + component];
}
// Transformers 5.15 GPT-Neo performs no inverse-sqrt scaling.
// Avoid adding an operation when the exact model-equivalent factor is 1.
output[(head * queryLength + query) * keyLength + key] =
scale == 1.0f ? rawSum : rawSum * scale;
}
}
}
return new Tensor(
outputName,
new[] { headCount, queryLength, keyLength },
output );
}
public static bool IsLayer0AttentionAllowed(
string attentionType,
int query,
int key,
int queryLength,
int keyLength )
{
return IsAttentionAllowed(
attentionType,
windowSize: 0,
query,
key,
queryLength,
keyLength );
}
public static bool IsAttentionAllowed(
string attentionType,
int windowSize,
int query,
int key,
int queryLength,
int keyLength )
{
if ( attentionType != "global" && attentionType != "local" )
{
throw new InvalidOperationException(
$"[LLM:ERROR] GPT-Neo attention type must be global or local, " +
$"found '{attentionType}'." );
}
if ( attentionType == "local" && windowSize <= 0 )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Local GPT-Neo attention window must be positive, " +
$"found {windowSize}." );
}
if ( queryLength <= 0 || keyLength <= 0 || queryLength > keyLength ||
query < 0 || query >= queryLength || key < 0 || key >= keyLength )
{
throw new IndexOutOfRangeException(
$"[LLM:ERROR] Attention mask index query={query}, key={key} is invalid " +
$"for query_length={queryLength}, key_length={keyLength}. " +
"The verified source slice requires 0 < query_length <= key_length." );
}
int absoluteQuery = keyLength - queryLength + query;
bool causal = key <= absoluteQuery;
return attentionType == "global"
? causal
: causal && absoluteQuery - key < windowSize;
}
public static Tensor ApplyLayer0AttentionMask(
Tensor unmaskedScores,
string attentionType,
float maskedSentinel,
string outputName )
{
return ApplyAttentionMask(
unmaskedScores,
attentionType,
windowSize: 0,
maskedSentinel,
outputName );
}
public static Tensor ApplyAttentionMask(
Tensor unmaskedScores,
string attentionType,
int windowSize,
float maskedSentinel,
string outputName )
{
if ( unmaskedScores is null )
{
throw new ArgumentNullException( nameof( unmaskedScores ) );
}
if ( string.IsNullOrWhiteSpace( outputName ) )
{
throw new ArgumentException(
"[LLM:ERROR] Masked attention-score output name cannot be empty.",
nameof( outputName ) );
}
if ( unmaskedScores.Rank != 3 || unmaskedScores.Shape[0] <= 0 ||
unmaskedScores.Shape[1] <= 0 || unmaskedScores.Shape[2] <= 0 ||
unmaskedScores.Shape[1] > unmaskedScores.Shape[2] )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} expected unmasked scores shape " +
$"[head,query,key] with 0 < query <= key, found " +
$"{unmaskedScores.ShapeText}." );
}
if ( !float.IsFinite( maskedSentinel ) || !(maskedSentinel < 0) )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} mask sentinel must be finite and negative, " +
$"found {maskedSentinel:G9}." );
}
int headCount = unmaskedScores.Shape[0];
int queryLength = unmaskedScores.Shape[1];
int keyLength = unmaskedScores.Shape[2];
float[] output = new float[unmaskedScores.Data.Length];
for ( int head = 0; head < headCount; head++ )
{
for ( int query = 0; query < queryLength; query++ )
{
for ( int key = 0; key < keyLength; key++ )
{
int index = (head * queryLength + query) * keyLength + key;
bool allowed = IsAttentionAllowed(
attentionType,
windowSize,
query,
key,
queryLength,
keyLength );
output[index] = allowed ? unmaskedScores.Data[index] : maskedSentinel;
}
}
}
return new Tensor(
outputName,
new[] { headCount, queryLength, keyLength },
output );
}
public static Tensor ComputeAttentionProbabilities(
Tensor maskedScores,
string outputName )
{
if ( maskedScores is null )
{
throw new ArgumentNullException( nameof( maskedScores ) );
}
if ( string.IsNullOrWhiteSpace( outputName ) )
{
throw new ArgumentException(
"[LLM:ERROR] Attention-probability output name cannot be empty.",
nameof( outputName ) );
}
if ( maskedScores.Rank != 3 || maskedScores.Shape[0] <= 0 ||
maskedScores.Shape[1] <= 0 || maskedScores.Shape[2] <= 0 )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} expected masked scores shape " +
$"[head,query,key] with positive dimensions, found " +
$"{maskedScores.ShapeText}." );
}
int headCount = maskedScores.Shape[0];
int queryLength = maskedScores.Shape[1];
int keyLength = maskedScores.Shape[2];
float[] output = new float[maskedScores.Data.Length];
for ( int head = 0; head < headCount; head++ )
{
for ( int query = 0; query < queryLength; query++ )
{
int row = (head * queryLength + query) * keyLength;
float rowMaximum = float.NegativeInfinity;
for ( int key = 0; key < keyLength; key++ )
{
float value = maskedScores.Data[row + key];
if ( !float.IsFinite( value ) )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} input contains a non-finite score " +
$"at [{head},{query},{key}]: {value}." );
}
rowMaximum = MathF.Max( rowMaximum, value );
}
float exponentialSum = 0;
for ( int key = 0; key < keyLength; key++ )
{
float shifted = maskedScores.Data[row + key] - rowMaximum;
float exponential = MathF.Exp( shifted );
if ( !float.IsFinite( exponential ) || exponential < 0 )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} produced invalid exp at " +
$"[{head},{query},{key}]: input={maskedScores.Data[row + key]:G9}, " +
$"maximum={rowMaximum:G9}, shifted={shifted:G9}, " +
$"exp={exponential:G9}." );
}
output[row + key] = exponential;
exponentialSum += exponential;
}
if ( !(exponentialSum > 0) || !float.IsFinite( exponentialSum ) )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} row [{head},{query}] produced invalid " +
$"exponential sum {exponentialSum:G9}." );
}
float inverseSum = 1.0f / exponentialSum;
for ( int key = 0; key < keyLength; key++ )
{
output[row + key] *= inverseSum;
}
}
}
return new Tensor(
outputName,
new[] { headCount, queryLength, keyLength },
output );
}
public static Tensor ComputeAttentionContextHeads(
Tensor probabilities,
Tensor valueHeads,
string outputName )
{
if ( probabilities is null )
{
throw new ArgumentNullException( nameof( probabilities ) );
}
if ( valueHeads is null )
{
throw new ArgumentNullException( nameof( valueHeads ) );
}
if ( string.IsNullOrWhiteSpace( outputName ) )
{
throw new ArgumentException(
"[LLM:ERROR] Attention-context output name cannot be empty.",
nameof( outputName ) );
}
if ( probabilities.Rank != 3 || probabilities.Shape[0] <= 0 ||
probabilities.Shape[1] <= 0 || probabilities.Shape[2] <= 0 )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} expected probabilities shape " +
$"[head,query,key] with positive dimensions, found " +
$"{probabilities.ShapeText}." );
}
if ( valueHeads.Rank != 3 || valueHeads.Shape[0] <= 0 ||
valueHeads.Shape[1] <= 0 || valueHeads.Shape[2] <= 0 )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} expected value shape " +
$"[head,key,component] with positive dimensions, found " +
$"{valueHeads.ShapeText}." );
}
if ( probabilities.Shape[0] != valueHeads.Shape[0] ||
probabilities.Shape[2] != valueHeads.Shape[1] )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} incompatible probabilities/value shapes: " +
$"probabilities={probabilities.ShapeText} [head,query,key], " +
$"V={valueHeads.ShapeText} [head,key,component]." );
}
int headCount = probabilities.Shape[0];
int queryLength = probabilities.Shape[1];
int keyLength = probabilities.Shape[2];
int headDimension = valueHeads.Shape[2];
float[] output = new float[headCount * queryLength * headDimension];
for ( int head = 0; head < headCount; head++ )
{
for ( int query = 0; query < queryLength; query++ )
{
int probabilityRow = (head * queryLength + query) * keyLength;
for ( int component = 0; component < headDimension; component++ )
{
float sum = 0;
for ( int key = 0; key < keyLength; key++ )
{
float probability = probabilities.Data[probabilityRow + key];
int valueIndex = (head * keyLength + key) * headDimension + component;
float value = valueHeads.Data[valueIndex];
if ( !float.IsFinite( probability ) || !float.IsFinite( value ) )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} received non-finite input at " +
$"head={head}, query={query}, key={key}, component={component}: " +
$"probability={probability:G9}, V={value:G9}." );
}
sum += probability * value;
}
if ( !float.IsFinite( sum ) )
{
throw new InvalidOperationException(
$"[LLM:ERROR] {outputName} produced non-finite context at " +
$"[{head},{query},{component}]: {sum:G9}." );
}
int outputIndex =
(head * queryLength + query) * headDimension + component;
output[outputIndex] = sum;
}
}
}
return new Tensor(
outputName,
new[] { headCount, queryLength, headDimension },
output );
}
public static string DescribeAttentionContextElement(
Tensor probabilities,
Tensor valueHeads,
Tensor context,
int head,
int query,
int component,
float expected )
{
if ( probabilities is null || valueHeads is null || context is null ||
probabilities.Rank != 3 || valueHeads.Rank != 3 || context.Rank != 3 ||
probabilities.Shape[0] != valueHeads.Shape[0] ||
probabilities.Shape[0] != context.Shape[0] ||
probabilities.Shape[1] != context.Shape[1] ||
probabilities.Shape[2] != valueHeads.Shape[1] ||
valueHeads.Shape[2] != context.Shape[2] )
{
throw new InvalidOperationException(
"[LLM:ERROR] Attention-context diagnostic requires compatible " +
"probability [head,query,key], V [head,key,component], and context " +
"[head,query,component] tensors." );
}
if ( head < 0 || head >= context.Shape[0] ||
query < 0 || query >= context.Shape[1] ||
component < 0 || component >= context.Shape[2] )
{
throw new IndexOutOfRangeException(
$"[LLM:ERROR] Attention-context diagnostic index " +
$"[{head},{query},{component}] is outside {context.ShapeText}." );
}
int keyLength = probabilities.Shape[2];
int headDimension = valueHeads.Shape[2];
int probabilityRow = (head * probabilities.Shape[1] + query) * keyLength;
float[] probabilityValues = new float[keyLength];
float[] values = new float[keyLength];
float[] products = new float[keyLength];
float sum = 0;
for ( int key = 0; key < keyLength; key++ )
{
probabilityValues[key] = probabilities.Data[probabilityRow + key];
int valueIndex = (head * keyLength + key) * headDimension + component;
values[key] = valueHeads.Data[valueIndex];
products[key] = probabilityValues[key] * values[key];
sum += products[key];
}
int contextIndex =
(head * context.Shape[1] + query) * context.Shape[2] + component;
float actual = context.Data[contextIndex];
if ( BitConverter.SingleToInt32Bits( sum ) !=
BitConverter.SingleToInt32Bits( actual ) )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Attention-context diagnostic recomputation differs at " +
$"[{head},{query},{component}]: recomputed={sum:G9}, actual={actual:G9}." );
}
return $"head={head} query={query} component={component} " +
$"probabilities={FormatFloatList( probabilityValues )} " +
$"V={FormatFloatList( values )} products={FormatFloatList( products )} " +
$"accumulated={actual:G9} python={expected:G9} " +
$"abs_diff={MathF.Abs( actual - expected ):G12}";
}
public static string DescribeAttentionSoftmaxRow(
Tensor maskedScores,
Tensor probabilities,
Tensor expectedProbabilities,
int head,
int query )
{
if ( maskedScores is null || probabilities is null || expectedProbabilities is null ||
maskedScores.Rank != 3 || probabilities.Rank != 3 || expectedProbabilities.Rank != 3 ||
maskedScores.Shape[0] != probabilities.Shape[0] ||
maskedScores.Shape[1] != probabilities.Shape[1] ||
maskedScores.Shape[2] != probabilities.Shape[2] ||
maskedScores.Shape[0] != expectedProbabilities.Shape[0] ||
maskedScores.Shape[1] != expectedProbabilities.Shape[1] ||
maskedScores.Shape[2] != expectedProbabilities.Shape[2] )
{
throw new InvalidOperationException(
"[LLM:ERROR] Softmax row diagnostic requires matching rank-3 tensors." );
}
if ( head < 0 || head >= maskedScores.Shape[0] ||
query < 0 || query >= maskedScores.Shape[1] )
{
throw new IndexOutOfRangeException(
$"[LLM:ERROR] Softmax row diagnostic index [{head},{query}] is outside " +
$"{maskedScores.ShapeText}." );
}
int keyLength = maskedScores.Shape[2];
int row = (head * maskedScores.Shape[1] + query) * keyLength;
float rowMaximum = float.NegativeInfinity;
for ( int key = 0; key < keyLength; key++ )
{
rowMaximum = MathF.Max( rowMaximum, maskedScores.Data[row + key] );
}
float[] inputs = new float[keyLength];
float[] shifted = new float[keyLength];
float[] exponentials = new float[keyLength];
float[] actual = new float[keyLength];
float[] expected = new float[keyLength];
float exponentialSum = 0;
float probabilitySum = 0;
for ( int key = 0; key < keyLength; key++ )
{
inputs[key] = maskedScores.Data[row + key];
shifted[key] = inputs[key] - rowMaximum;
exponentials[key] = MathF.Exp( shifted[key] );
exponentialSum += exponentials[key];
actual[key] = probabilities.Data[row + key];
expected[key] = expectedProbabilities.Data[row + key];
probabilitySum += actual[key];
}
return $"head={head} query={query} input={FormatFloatList( inputs )} " +
$"max={rowMaximum:G9} shifted={FormatFloatList( shifted )} " +
$"exp={FormatFloatList( exponentials )} exp_sum={exponentialSum:G9} " +
$"actual={FormatFloatList( actual )} python={FormatFloatList( expected )} " +
$"row_sum={probabilitySum:G9}";
}
public static string DescribeAttentionMaskApplication(
Tensor unmaskedScores,
Tensor maskedScores,
string attentionType,
float maskedSentinel,
int head,
int query,
int key )
{
if ( unmaskedScores is null || maskedScores is null ||
unmaskedScores.Rank != 3 || maskedScores.Rank != 3 ||
unmaskedScores.Shape[0] != maskedScores.Shape[0] ||
unmaskedScores.Shape[1] != maskedScores.Shape[1] ||
unmaskedScores.Shape[2] != maskedScores.Shape[2] )
{
throw new InvalidOperationException(
"[LLM:ERROR] Mask diagnostic requires matching rank-3 unmasked/masked tensors." );
}
int queryLength = unmaskedScores.Shape[1];
int keyLength = unmaskedScores.Shape[2];
if ( head < 0 || head >= unmaskedScores.Shape[0] )
{
throw new IndexOutOfRangeException(
$"[LLM:ERROR] Mask diagnostic head {head} is outside " +
$"{unmaskedScores.ShapeText}." );
}
bool allowed = IsLayer0AttentionAllowed(
attentionType,
query,
key,
queryLength,
keyLength );
int index = (head * queryLength + query) * keyLength + key;
float unmasked = unmaskedScores.Data[index];
float masked = maskedScores.Data[index];
return $"head={head} query={query} key={key} allowed={allowed} " +
$"unmasked={unmasked:G9} masked={masked:G9} " +
$"masked_bits=0x{BitConverter.SingleToInt32Bits( masked ):X8} " +
$"expected_sentinel={maskedSentinel:G9} " +
$"sentinel_bits=0x{BitConverter.SingleToInt32Bits( maskedSentinel ):X8}";
}
public static string DescribeAttentionScore(
Tensor queryHeads,
Tensor keyHeads,
Tensor scores,
int head,
int query,
int key,
float scale,
float expected )
{
if ( queryHeads is null || keyHeads is null || scores is null ||
queryHeads.Rank != 3 || keyHeads.Rank != 3 || scores.Rank != 3 )
{
throw new InvalidOperationException(
"[LLM:ERROR] Attention-score diagnostic requires rank-3 Q, K, and score tensors." );
}
int headDimension = queryHeads.Shape[2];
if ( headDimension != 4 || keyHeads.Shape[2] != headDimension ||
queryHeads.Shape[0] != keyHeads.Shape[0] ||
scores.Shape[0] != queryHeads.Shape[0] ||
scores.Shape[1] != queryHeads.Shape[1] ||
scores.Shape[2] != keyHeads.Shape[1] )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Attention-score diagnostic expected compatible Q/K/scores with " +
$"head dimension 4, found Q={queryHeads.ShapeText}, K={keyHeads.ShapeText}, " +
$"scores={scores.ShapeText}." );
}
if ( head < 0 || head >= scores.Shape[0] || query < 0 || query >= scores.Shape[1] ||
key < 0 || key >= scores.Shape[2] )
{
throw new IndexOutOfRangeException(
$"[LLM:ERROR] Attention-score diagnostic index [{head},{query},{key}] " +
$"is outside {scores.ShapeText}." );
}
int queryRow = (head * queryHeads.Shape[1] + query) * headDimension;
int keyRow = (head * keyHeads.Shape[1] + key) * headDimension;
float q0 = queryHeads.Data[queryRow];
float q1 = queryHeads.Data[queryRow + 1];
float q2 = queryHeads.Data[queryRow + 2];
float q3 = queryHeads.Data[queryRow + 3];
float k0 = keyHeads.Data[keyRow];
float k1 = keyHeads.Data[keyRow + 1];
float k2 = keyHeads.Data[keyRow + 2];
float k3 = keyHeads.Data[keyRow + 3];
float p0 = q0 * k0;
float p1 = q1 * k1;
float p2 = q2 * k2;
float p3 = q3 * k3;
float rawSum = ((p0 + p1) + p2) + p3;
float recomputed = scale == 1.0f ? rawSum : rawSum * scale;
int scoreIndex = (head * scores.Shape[1] + query) * scores.Shape[2] + key;
float actual = scores.Data[scoreIndex];
if ( BitConverter.SingleToInt32Bits( recomputed ) !=
BitConverter.SingleToInt32Bits( actual ) )
{
throw new InvalidOperationException(
$"[LLM:ERROR] Attention-score diagnostic recomputation differs at " +
$"[{head},{query},{key}]: recomputed={recomputed:G9}, actual={actual:G9}." );
}
return $"head={head} query={query} key={key} " +
$"q=[{q0:G9},{q1:G9},{q2:G9},{q3:G9}] " +
$"k=[{k0:G9},{k1:G9},{k2:G9},{k3:G9}] " +
$"products=[{p0:G9},{p1:G9},{p2:G9},{p3:G9}] " +
$"raw_sum={rawSum:G9} scale={scale:G9} final={actual:G9} expected={expected:G9}";
}
private static string FormatFloatList( IReadOnlyList<float> values )
{
string[] formatted = new string[values.Count];
for ( int index = 0; index < values.Count; index++ )
{
formatted[index] = values[index].ToString( "G9" );
}
return $"[{string.Join( ",", formatted )}]";
}
}