using Sandbox.Diagnostics;

namespace LlmPoc.Llm;

[Title( "LLM Foundation Probe" )]
[Category( "LLM" )]
public sealed class LlmFoundationProbe : Component
{
	[Property]
	public bool RunOnStart { get; set; } = true;

	[Property]
	public bool EnableTrace { get; set; }

	[Property]
	public bool LogRepresentativeTensors { get; set; } = true;

	public string Status => LlmFoundationState.Status;
	public string Error => LlmFoundationState.Error;

	protected override void OnStart()
	{
		if ( RunOnStart )
		{
			_ = ValidateAsync();
		}
	}

	public async Task ValidateAsync()
	{
		if ( LlmFoundationState.IsRunning )
		{
			return;
		}

		LlmFoundationState.IsRunning = true;
		LlmFoundationState.IsReady = false;
		LlmFoundationState.Error = "";
		LlmFoundationState.Status = "IMPLEMENTED BUT NOT YET VALIDATED: loading model";
		LlmLog.TraceEnabled = EnableTrace;
		LlmLog.Info(
			"INIT",
			"Starting in-process foundation, full-forward, and greedy-generation validation." );
		FastTimer timer = FastTimer.StartNew();

		try
		{
			FoundationArtifacts artifacts = await Task.RunInThreadAsync( LoadAndValidate );
			LlmFoundationState.Model = artifacts.Model;
			LlmFoundationState.Tokenizer = artifacts.Tokenizer;
			LlmFoundationState.LastValidationMilliseconds = timer.ElapsedMilliSeconds;
			LlmFoundationState.Status =
				"IMPLEMENTED AND VALIDATED: deterministic greedy autoregressive generation PASS";
			LlmFoundationState.IsReady = true;
			LlmLog.Info(
				"INIT",
				$"Foundation and generation validation PASS in {timer.ElapsedMilliSeconds:N2} ms. " +
				"The reusable scalar GPT-Neo path passes all eight layer-output gates, " +
				"final LayerNorm, all 50,257 last-position logits, exact first-token " +
				"argmax, the independent 67,600-coordinate local-mask regression, and " +
				"12-token greedy autoregressive parity with full context recomputation. " +
				"Runtime chat integration is validated separately through the " +
				"production language-model service." );
		}
		catch ( Exception error )
		{
			LlmFoundationState.Error = error.Message;
			LlmFoundationState.Status = "IMPLEMENTED BUT FAILING: foundation validation failed";
			LlmLog.Error( $"Foundation validation failed: {error}" );
		}
		finally
		{
			LlmFoundationState.IsRunning = false;
		}
	}

	private FoundationArtifacts LoadAndValidate()
	{
		TinyStoriesRuntimeArtifacts runtime =
			TinyStoriesRuntimeArtifactLoader.LoadFromResourceLibrary(
				LlmPaths.RuntimeModelResource );
		TinyStoriesConfig config = runtime.Config;
		ValidateKnownArchitecture( config );

		SboxLlmModel model = runtime.Model;
		TensorManifestDocument manifest = ReadRequiredJson<TensorManifestDocument>( LlmPaths.Manifest );
		ModelManifestValidator.Validate( model, manifest );
		ValidateRequiredTensorShapes( model, config );
		LlmLog.Info(
			"PARITY",
			$"Binary/manifest agreement tensor_count={model.TensorCount} " +
			$"parameters={model.TotalParameterCount} PASS" );

		if ( LogRepresentativeTensors )
		{
			LogTensorSummaries( model );
		}

		NumericComparison tiedComparison = TensorDiagnostics.Compare(
			model.GetRequiredTensor( "transformer.wte.weight" ).Data,
			model.GetRequiredTensor( "lm_head.weight" ).Data,
			absoluteTolerance: 0,
			relativeTolerance: 0 );
		LlmLog.Info( "PARITY", $"Tied embedding/LM head {tiedComparison}" );
		if ( !tiedComparison.Passed )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] Config declares tied word embeddings, but exported " +
				"transformer.wte.weight and lm_head.weight differ." );
		}

		LlmReferenceData reference = ReadRequiredJson<LlmReferenceData>( LlmPaths.Reference );
		ValidateReference( reference, config );
		ValidateReferenceLogits( reference, config );

		Gpt2ByteBpeTokenizer tokenizer = runtime.Tokenizer;
		if ( tokenizer.VocabularySize != config.VocabularySize )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Tokenizer vocabulary has {tokenizer.VocabularySize} entries, " +
				$"model config expects {config.VocabularySize}." );
		}
		ValidateTokenizerParity( tokenizer, reference );
		ForwardReferenceDocument forwardReference = ReadRequiredJson<ForwardReferenceDocument>(
			LlmPaths.ReferenceIntermediates );
		ValidateForwardReferenceDocument( forwardReference, config, reference );
		FastTimer completeValidationChainTimer = FastTimer.StartNew();
		Tensor combinedEmbedding = ValidateEmbeddingParity(
			model,
			config,
			reference,
			forwardReference,
			out double embeddingMilliseconds );
		GptNeoLayerForwardResult layer0 = GptNeoTransformerLayer.Forward(
			model, config, combinedEmbedding, layerIndex: 0 );
		GptNeoLayerParityResult layer0Parity = GptNeoLayerParity.Validate(
			layer0, forwardReference, config );
		GptNeoLayerForwardResult layer1 = GptNeoTransformerLayer.Forward(
			model, config, layer0.Output, layerIndex: 1 );
		GptNeoLayerParityResult layer1Parity = GptNeoLayerParity.Validate(
			layer1, forwardReference, config );
		LocalAttentionMaskReferenceDocument localMask =
			ReadRequiredJson<LocalAttentionMaskReferenceDocument>(
				LlmPaths.ReferenceLayer1LocalMaskLen260Metadata );
		LocalMaskParityResult localMaskParity =
			GptNeoLayerParity.ValidateLocalMaskReference( localMask, config );
		CompleteModelParityResult completeModel = GptNeoModelParity.ValidateFromLayer2(
			model,
			config,
			reference,
			forwardReference,
			tokenizer,
			layer1.Output );
		double allEightLayersMilliseconds = layer0.ElapsedMilliseconds +
			layer1.ElapsedMilliseconds + completeModel.LaterLayersMilliseconds;
		double singleForwardComputeMilliseconds = embeddingMilliseconds +
			allEightLayersMilliseconds + completeModel.FinalLayerNormMilliseconds +
			completeModel.LmHeadMilliseconds;
		LlmLog.Info(
			"PERF",
			$"stage=complete_single_forward output={completeModel.Logits.ShapeText} " +
			$"embedding_ms={embeddingMilliseconds:N4} " +
			$"layer0_forward_ms={layer0.ElapsedMilliseconds:N4} " +
			$"layer0_validate_ms={layer0Parity.ValidationMilliseconds:N4} " +
			$"layer1_forward_ms={layer1.ElapsedMilliseconds:N4} " +
			$"layer1_validate_ms={layer1Parity.ValidationMilliseconds:N4} " +
			$"layer2_forward_ms={completeModel.LaterLayers[0].ElapsedMilliseconds:N4} " +
			$"layer3_forward_ms={completeModel.LaterLayers[1].ElapsedMilliseconds:N4} " +
			$"layer4_forward_ms={completeModel.LaterLayers[2].ElapsedMilliseconds:N4} " +
			$"layer5_forward_ms={completeModel.LaterLayers[3].ElapsedMilliseconds:N4} " +
			$"layer6_forward_ms={completeModel.LaterLayers[4].ElapsedMilliseconds:N4} " +
			$"layer7_forward_ms={completeModel.LaterLayers[5].ElapsedMilliseconds:N4} " +
			$"eight_layers_ms={allEightLayersMilliseconds:N4} " +
			$"final_ln_ms={completeModel.FinalLayerNormMilliseconds:N4} " +
			$"lm_head_ms={completeModel.LmHeadMilliseconds:N4} " +
			$"single_forward_compute_ms={singleForwardComputeMilliseconds:N4} " +
			$"local_mask_len260_ms={localMaskParity.ElapsedMilliseconds:N4} " +
			$"complete_validation_chain_ms={completeValidationChainTimer.ElapsedMilliSeconds:N4}" );

		GreedyGenerationReferenceDocument generationReference =
			ReadRequiredJson<GreedyGenerationReferenceDocument>(
				LlmPaths.ReferenceGreedyGeneration );
		GreedyGenerationParityResult generationParity = GreedyGenerationParity.Validate(
			model,
			config,
			tokenizer,
			reference,
			generationReference );
		LlmLog.Info(
			"PERF",
			$"stage=greedy_generation_validation generated_tokens=" +
			$"{generationParity.Generation.GeneratedTokenIds.Length} " +
			$"raw_forward_ms={generationParity.Generation.TotalForwardMilliseconds:N4} " +
			$"raw_generation_ms={generationParity.Generation.TotalGenerationMilliseconds:N4} " +
			$"validation_harness_ms={generationParity.ValidationHarnessMilliseconds:N4} " +
			"worker_thread=true kv_cache=false" );
		return new FoundationArtifacts( model, tokenizer );
	}

	private static T ReadRequiredJson<T>( string path )
	{
		if ( !FileSystem.Mounted.FileExists( path ) )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Required JSON file '{path}' is missing from FileSystem.Mounted." );
		}

		T document = FileSystem.Mounted.ReadJson<T>( path );
		if ( document is null )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Required JSON file '{path}' deserialized to null." );
		}
		return document;
	}

	private static void ValidateKnownArchitecture( TinyStoriesConfig config )
	{
		RequireConfig( "vocab_size", 50_257, config.VocabularySize );
		RequireConfig( "max_position_embeddings", 2_048, config.MaximumPositions );
		RequireConfig( "hidden_size", 64, config.HiddenSize );
		RequireConfig( "num_layers", 8, config.LayerCount );
		RequireConfig( "num_heads", 16, config.AttentionHeadCount );
		RequireConfig( "window_size", 256, config.AttentionWindowSize );
		RequireConfig( "bos_token_id", 50_256, config.BosTokenId );
		RequireConfig( "eos_token_id", 50_256, config.EosTokenId );
		if ( config.ActivationFunction != "gelu_new" )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] activation_function expected gelu_new, found {config.ActivationFunction}." );
		}
		if ( Math.Abs( config.LayerNormEpsilon - 1e-5f ) > 1e-10f )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] layer_norm_epsilon expected 1e-5, found {config.LayerNormEpsilon:G9}." );
		}
		if ( config.EmbeddingDropout != 0 )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] embed_dropout expected 0 for deterministic eval parity, " +
				$"found {config.EmbeddingDropout:G9}." );
		}
		if ( !config.TieWordEmbeddings )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] tie_word_embeddings expected true, found false." );
		}

		for ( int layer = 0; layer < config.LayerCount; layer++ )
		{
			string expected = layer % 2 == 0 ? "global" : "local";
			if ( config.AttentionLayers[layer] != expected )
			{
				throw new InvalidOperationException(
					$"[LLM:ERROR] attention_layers[{layer}] expected {expected}, " +
					$"found {config.AttentionLayers[layer]}." );
			}
		}

		LlmLog.Info(
			"INIT",
			$"Config validated: vocab={config.VocabularySize}, positions={config.MaximumPositions}, " +
			$"hidden={config.HiddenSize}, layers={config.LayerCount}, heads={config.AttentionHeadCount}, " +
			$"head_dim={config.HiddenSize / config.AttentionHeadCount}, " +
			$"window={config.AttentionWindowSize}, activation={config.ActivationFunction}." );
	}

	private static void ValidateRequiredTensorShapes( SboxLlmModel model, TinyStoriesConfig config )
	{
		int hidden = config.HiddenSize;
		int intermediate = hidden * 4;
		model.GetRequiredTensor( "transformer.wte.weight" ).RequireShape( config.VocabularySize, hidden );
		model.GetRequiredTensor( "transformer.wpe.weight" ).RequireShape( config.MaximumPositions, hidden );
		model.GetRequiredTensor( "transformer.ln_f.weight" ).RequireShape( hidden );
		model.GetRequiredTensor( "transformer.ln_f.bias" ).RequireShape( hidden );
		model.GetRequiredTensor( "lm_head.weight" ).RequireShape( config.VocabularySize, hidden );

		for ( int layer = 0; layer < config.LayerCount; layer++ )
		{
			string prefix = $"transformer.h.{layer}";
			model.GetRequiredTensor( $"{prefix}.attn.attention.k_proj.weight" ).RequireShape( hidden, hidden );
			model.GetRequiredTensor( $"{prefix}.attn.attention.q_proj.weight" ).RequireShape( hidden, hidden );
			model.GetRequiredTensor( $"{prefix}.attn.attention.v_proj.weight" ).RequireShape( hidden, hidden );
			model.GetRequiredTensor( $"{prefix}.attn.attention.out_proj.weight" ).RequireShape( hidden, hidden );
			model.GetRequiredTensor( $"{prefix}.attn.attention.out_proj.bias" ).RequireShape( hidden );
			model.GetRequiredTensor( $"{prefix}.ln_1.weight" ).RequireShape( hidden );
			model.GetRequiredTensor( $"{prefix}.ln_1.bias" ).RequireShape( hidden );
			model.GetRequiredTensor( $"{prefix}.ln_2.weight" ).RequireShape( hidden );
			model.GetRequiredTensor( $"{prefix}.ln_2.bias" ).RequireShape( hidden );
			model.GetRequiredTensor( $"{prefix}.mlp.c_fc.weight" ).RequireShape( intermediate, hidden );
			model.GetRequiredTensor( $"{prefix}.mlp.c_fc.bias" ).RequireShape( intermediate );
			model.GetRequiredTensor( $"{prefix}.mlp.c_proj.weight" ).RequireShape( hidden, intermediate );
			model.GetRequiredTensor( $"{prefix}.mlp.c_proj.bias" ).RequireShape( hidden );
		}
	}

	private static void LogTensorSummaries( SboxLlmModel model )
	{
		string[] names =
		{
			"transformer.wte.weight",
			"transformer.wpe.weight",
			"transformer.h.0.ln_1.weight",
			"transformer.h.0.attn.attention.q_proj.weight",
			"transformer.h.0.mlp.c_fc.weight",
			"transformer.ln_f.weight",
			"lm_head.weight"
		};

		foreach ( string name in names )
		{
			TensorSummary summary = TensorDiagnostics.Summarize( model.GetRequiredTensor( name ) );
			LlmLog.Info( "TENSOR", summary.ToString() );
			if ( !summary.IsFinite )
			{
				throw new InvalidOperationException(
					$"[LLM:ERROR] Representative tensor {name} contains non-finite values." );
			}
		}
	}

	private static void ValidateReference( LlmReferenceData reference, TinyStoriesConfig config )
	{
		if ( reference.InputTokenIds is null || reference.InputTokenIds.Length == 0 )
		{
			throw new InvalidOperationException( "[LLM:ERROR] Python reference input token IDs are empty." );
		}
		for ( int index = 0; index < reference.InputTokenIds.Length; index++ )
		{
			int tokenId = reference.InputTokenIds[index];
			if ( tokenId < 0 || tokenId >= config.VocabularySize )
			{
				throw new InvalidOperationException(
					$"[LLM:ERROR] Python reference token ID {tokenId} at input index {index} " +
					$"is outside [0,{config.VocabularySize})." );
			}
		}
		RequireConfig( "reference bos_token_id", config.BosTokenId, reference.BosTokenId );
		RequireConfig( "reference eos_token_id", config.EosTokenId, reference.EosTokenId );
	}

	private static void ValidateReferenceLogits( LlmReferenceData reference, TinyStoriesConfig config )
	{
		float[] logits = ReferenceFloatData.LoadFromMounted(
			LlmPaths.ReferenceLogits,
			config.VocabularySize );
		TensorSummary summary = TensorDiagnostics.Summarize(
			"python.reference_logits",
			$"[{config.VocabularySize}]",
			logits );
		LlmLog.Info( "LOGITS", summary.ToString() );
		if ( !summary.IsFinite )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] Python reference logits contain NaN or Infinity." );
		}

		int actualArgmax = ReferenceFloatData.ArgmaxFinite( logits, "python.reference_logits" );
		bool passed = actualArgmax == reference.GreedyNextTokenId;
		LlmLog.Info(
			"PARITY",
			$"Reference logits argmax expected={reference.GreedyNextTokenId} " +
			$"actual={actualArgmax} {(passed ? "PASS" : "FAIL")}" );
		if ( !passed )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Reference logits argmax mismatch: metadata expects " +
				$"{reference.GreedyNextTokenId}, binary logits produce {actualArgmax}." );
		}
	}

	private static void ValidateTokenizerParity(
		Gpt2ByteBpeTokenizer tokenizer,
		LlmReferenceData reference )
	{
		FastTimer timer = FastTimer.StartNew();
		int[] actual = tokenizer.Encode( reference.Prompt );
		int mismatch = FirstMismatch( reference.InputTokenIds, actual );
		if ( mismatch >= 0 )
		{
			string expectedValue = mismatch < reference.InputTokenIds.Length
				? reference.InputTokenIds[mismatch].ToString()
				: "<end>";
			string actualValue = mismatch < actual.Length ? actual[mismatch].ToString() : "<end>";
			LlmLog.Error(
				$"Tokenizer mismatch at index {mismatch}: expected={expectedValue} actual={actualValue} FAIL" );
			throw new InvalidOperationException(
				$"[LLM:ERROR] Tokenizer parity failed at token index {mismatch}." );
		}

		string decodedPrompt = tokenizer.Decode( actual );
		if ( decodedPrompt != reference.Prompt )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Tokenizer decode parity failed: expected '{reference.Prompt}', " +
				$"actual '{decodedPrompt}'." );
		}

		string decodedGenerated = tokenizer.Decode( reference.GeneratedTokenIds );
		if ( decodedGenerated != reference.GeneratedText )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Generated-token decode parity failed: expected " +
				$"'{reference.GeneratedText}', actual '{decodedGenerated}'." );
		}

		LlmLog.Info(
			"PARITY",
			$"Tokenizer expected=[{string.Join( ",", reference.InputTokenIds )}] " +
			$"actual=[{string.Join( ",", actual )}] encode/decode PASS " +
			$"({timer.ElapsedMilliSeconds:N3} ms)." );
	}

	private static Tensor ValidateEmbeddingParity(
		SboxLlmModel model,
		TinyStoriesConfig config,
		LlmReferenceData reference,
		ForwardReferenceDocument document,
		out double elapsedMilliseconds )
	{
		ForwardReferenceStage stage = document.GetRequiredStage( "combined_embedding" );
		ValidateForwardReferenceStage(
			stage,
			"reference_embedding.f32",
			document.SequenceLength,
			document.HiddenSize );

		float[] expected = ReferenceFloatData.LoadFromMounted(
			LlmPaths.ReferenceEmbedding,
			stage.Elements );
		TensorSummary expectedSummary = TensorDiagnostics.Summarize(
			"python.combined_embedding",
			$"[{document.SequenceLength},{document.HiddenSize}]",
			expected );
		if ( !expectedSummary.IsFinite )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] Python combined-embedding reference contains NaN or Infinity." );
		}

		FastTimer timer = FastTimer.StartNew();
		Tensor actual = TinyStoriesForwardStages.CombineEmbeddings(
			model,
			config,
			reference.InputTokenIds );
		elapsedMilliseconds = timer.ElapsedMilliSeconds;
		actual.RequireShape( document.SequenceLength, document.HiddenSize );
		TensorSummary actualSummary = TensorDiagnostics.Summarize( actual );
		LlmLog.Info( "EMBED", actualSummary.ToString() );
		if ( !actualSummary.IsFinite )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] C# combined embedding contains NaN or Infinity." );
		}

		NumericComparison comparison = TensorDiagnostics.Compare(
			expected,
			actual.Data,
			absoluteTolerance: 1e-6,
			relativeTolerance: 1e-6 );
		LlmLog.Info( "PARITY", $"embedding {comparison}" );
		LlmLog.Info(
			"PERF",
			$"stage=combined_embedding shape={actual.ShapeText} elapsed_ms={elapsedMilliseconds:N4}" );
		if ( !comparison.Passed )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Combined embedding parity failed: {comparison}" );
		}
		return actual;
	}

	private static Tensor ValidateLayer0Ln1Parity(
		SboxLlmModel model,
		TinyStoriesConfig config,
		ForwardReferenceDocument document,
		Tensor combinedEmbedding )
	{
		ForwardReferenceStage stage = document.GetRequiredStage( "layer0_ln_1" );
		ValidateForwardReferenceStage(
			stage,
			"reference_layer0_ln1.f32",
			document.SequenceLength,
			document.HiddenSize );

		float[] expected = ReferenceFloatData.LoadFromMounted(
			LlmPaths.ReferenceLayer0Ln1,
			stage.Elements );
		TensorSummary expectedSummary = TensorDiagnostics.Summarize(
			"python.layer0.ln_1",
			$"[{document.SequenceLength},{document.HiddenSize}]",
			expected );
		if ( !expectedSummary.IsFinite )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] Python layer 0 ln_1 reference contains NaN or Infinity." );
		}

		FastTimer timer = FastTimer.StartNew();
		Tensor actual = TinyStoriesForwardStages.ApplyLayer0Ln1(
			model,
			config,
			combinedEmbedding );
		double elapsedMilliseconds = timer.ElapsedMilliSeconds;
		actual.RequireShape( document.SequenceLength, document.HiddenSize );
		TensorSummary actualSummary = TensorDiagnostics.Summarize( actual );
		LlmLog.Info( "LN", $"layer=0 ln=1 {actualSummary}" );
		if ( !actualSummary.IsFinite )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] C# layer 0 ln_1 output contains NaN or Infinity." );
		}

		NumericComparison comparison = TensorDiagnostics.Compare(
			expected,
			actual.Data,
			absoluteTolerance: 1e-5,
			relativeTolerance: 1e-5 );
		LlmLog.Info( "PARITY", $"layer0.ln_1 {comparison}" );
		LlmLog.Info(
			"PERF",
			$"stage=layer0.ln_1 shape={actual.ShapeText} elapsed_ms={elapsedMilliseconds:N4}" );
		if ( !comparison.Passed )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 ln_1 parity failed: {comparison}" );
		}
		return actual;
	}

	private static Tensor ValidateLayer0QParity(
		SboxLlmModel model,
		TinyStoriesConfig config,
		ForwardReferenceDocument document,
		Tensor layer0Ln1 )
	{
		ForwardReferenceStage stage = document.GetRequiredStage( "layer0_q" );
		ValidateForwardReferenceStage(
			stage,
			"reference_layer0_q.f32",
			document.SequenceLength,
			document.HiddenSize );

		Tensor weight = model.GetRequiredTensor( TinyStoriesForwardStages.Layer0QWeightName );
		weight.RequireShape( config.HiddenSize, config.HiddenSize );
		float[] expected = ReferenceFloatData.LoadFromMounted(
			LlmPaths.ReferenceLayer0Q,
			stage.Elements );
		TensorSummary expectedSummary = TensorDiagnostics.Summarize(
			"python.layer0.q",
			$"[{document.SequenceLength},{document.HiddenSize}]",
			expected );
		if ( !expectedSummary.IsFinite )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] Python layer 0 Q reference contains NaN or Infinity." );
		}

		LlmLog.Info(
			"QKV",
			$"layer=0 projection=Q input={layer0Ln1.ShapeText} weight={weight.ShapeText} " +
			$"weight_layout=[out,input] output=[{document.SequenceLength},{document.HiddenSize}] bias=none" );
		FastTimer timer = FastTimer.StartNew();
		Tensor actual = TinyStoriesForwardStages.LinearNoBias(
			layer0Ln1,
			weight,
			"forward.layer0.q" );
		double elapsedMilliseconds = timer.ElapsedMilliSeconds;
		actual.RequireShape( document.SequenceLength, document.HiddenSize );
		TensorSummary actualSummary = TensorDiagnostics.Summarize( actual );
		LlmLog.Info( "QKV", $"layer=0 projection=Q {actualSummary}" );
		if ( !actualSummary.IsFinite )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] C# layer 0 Q projection contains NaN or Infinity." );
		}

		NumericComparison comparison = TensorDiagnostics.Compare(
			expected,
			actual.Data,
			absoluteTolerance: 1e-5,
			relativeTolerance: 1e-5 );
		int worstToken = comparison.MaximumErrorIndex / document.HiddenSize;
		int worstFeature = comparison.MaximumErrorIndex % document.HiddenSize;
		LlmLog.Info(
			"PARITY",
			$"layer0.q {comparison} worst=[{worstToken},{worstFeature}]" );
		LlmLog.Info(
			"PERF",
			$"stage=layer0.q shape={actual.ShapeText} elapsed_ms={elapsedMilliseconds:N4}" );
		string dotDiagnostic = TinyStoriesForwardStages.DescribeLinearDotProduct(
			layer0Ln1,
			weight,
			worstToken,
			worstFeature,
			comparison.ExpectedAtMaximumError,
			comparison.ActualAtMaximumError );
		if ( !comparison.Passed )
		{
			LlmLog.Warning( "QKV", $"layer=0 projection=Q mismatch_dot {dotDiagnostic}" );
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 Q projection parity failed: {comparison}" );
		}
		LlmLog.Trace( "QKV", $"layer=0 projection=Q worst_dot {dotDiagnostic}" );
		return actual;
	}

	private static Tensor ValidateLayer0KParity(
		SboxLlmModel model,
		TinyStoriesConfig config,
		ForwardReferenceDocument document,
		Tensor layer0Ln1 )
	{
		ForwardReferenceStage stage = document.GetRequiredStage( "layer0_k" );
		ValidateForwardReferenceStage(
			stage,
			"reference_layer0_k.f32",
			document.SequenceLength,
			document.HiddenSize );

		Tensor weight = model.GetRequiredTensor( TinyStoriesForwardStages.Layer0KWeightName );
		weight.RequireShape( config.HiddenSize, config.HiddenSize );
		float[] expected = ReferenceFloatData.LoadFromMounted(
			LlmPaths.ReferenceLayer0K,
			stage.Elements );
		TensorSummary expectedSummary = TensorDiagnostics.Summarize(
			"python.layer0.k",
			$"[{document.SequenceLength},{document.HiddenSize}]",
			expected );
		if ( !expectedSummary.IsFinite )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] Python layer 0 K reference contains NaN or Infinity." );
		}

		LlmLog.Info(
			"QKV",
			$"layer=0 projection=K input={layer0Ln1.ShapeText} weight={weight.ShapeText} " +
			$"weight_layout=[out,input] output=[{document.SequenceLength},{document.HiddenSize}] bias=none" );
		FastTimer timer = FastTimer.StartNew();
		Tensor actual = TinyStoriesForwardStages.LinearNoBias(
			layer0Ln1,
			weight,
			"forward.layer0.k" );
		double elapsedMilliseconds = timer.ElapsedMilliSeconds;
		actual.RequireShape( document.SequenceLength, document.HiddenSize );
		TensorSummary actualSummary = TensorDiagnostics.Summarize( actual );
		LlmLog.Info( "QKV", $"layer=0 projection=K {actualSummary}" );
		if ( !actualSummary.IsFinite )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] C# layer 0 K projection contains NaN or Infinity." );
		}

		NumericComparison comparison = TensorDiagnostics.Compare(
			expected,
			actual.Data,
			absoluteTolerance: 1e-5,
			relativeTolerance: 1e-5 );
		int worstToken = comparison.MaximumErrorIndex / document.HiddenSize;
		int worstFeature = comparison.MaximumErrorIndex % document.HiddenSize;
		LlmLog.Info(
			"PARITY",
			$"layer0.k {comparison} worst=[{worstToken},{worstFeature}]" );
		LlmLog.Info(
			"PERF",
			$"stage=layer0.k shape={actual.ShapeText} elapsed_ms={elapsedMilliseconds:N4}" );
		string dotDiagnostic = TinyStoriesForwardStages.DescribeLinearDotProduct(
			layer0Ln1,
			weight,
			worstToken,
			worstFeature,
			comparison.ExpectedAtMaximumError,
			comparison.ActualAtMaximumError );
		if ( !comparison.Passed )
		{
			LlmLog.Warning( "QKV", $"layer=0 projection=K mismatch_dot {dotDiagnostic}" );
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 K projection parity failed: {comparison}" );
		}
		LlmLog.Trace( "QKV", $"layer=0 projection=K worst_dot {dotDiagnostic}" );
		return actual;
	}

	private static Tensor ValidateLayer0VParity(
		SboxLlmModel model,
		TinyStoriesConfig config,
		ForwardReferenceDocument document,
		Tensor layer0Ln1 )
	{
		ForwardReferenceStage stage = document.GetRequiredStage( "layer0_v" );
		ValidateForwardReferenceStage(
			stage,
			"reference_layer0_v.f32",
			document.SequenceLength,
			document.HiddenSize );

		Tensor weight = model.GetRequiredTensor( TinyStoriesForwardStages.Layer0VWeightName );
		weight.RequireShape( config.HiddenSize, config.HiddenSize );
		float[] expected = ReferenceFloatData.LoadFromMounted(
			LlmPaths.ReferenceLayer0V,
			stage.Elements );
		TensorSummary expectedSummary = TensorDiagnostics.Summarize(
			"python.layer0.v",
			$"[{document.SequenceLength},{document.HiddenSize}]",
			expected );
		if ( !expectedSummary.IsFinite )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] Python layer 0 V reference contains NaN or Infinity." );
		}

		LlmLog.Info(
			"QKV",
			$"layer=0 projection=V input={layer0Ln1.ShapeText} weight={weight.ShapeText} " +
			$"weight_layout=[out,input] output=[{document.SequenceLength},{document.HiddenSize}] bias=none" );
		FastTimer timer = FastTimer.StartNew();
		Tensor actual = TinyStoriesForwardStages.LinearNoBias(
			layer0Ln1,
			weight,
			"forward.layer0.v" );
		double elapsedMilliseconds = timer.ElapsedMilliSeconds;
		actual.RequireShape( document.SequenceLength, document.HiddenSize );
		TensorSummary actualSummary = TensorDiagnostics.Summarize( actual );
		LlmLog.Info( "QKV", $"layer=0 projection=V {actualSummary}" );
		if ( !actualSummary.IsFinite )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] C# layer 0 V projection contains NaN or Infinity." );
		}

		NumericComparison comparison = TensorDiagnostics.Compare(
			expected,
			actual.Data,
			absoluteTolerance: 1e-5,
			relativeTolerance: 1e-5 );
		int worstToken = comparison.MaximumErrorIndex / document.HiddenSize;
		int worstFeature = comparison.MaximumErrorIndex % document.HiddenSize;
		LlmLog.Info(
			"PARITY",
			$"layer0.v {comparison} worst=[{worstToken},{worstFeature}]" );
		LlmLog.Info(
			"PERF",
			$"stage=layer0.v shape={actual.ShapeText} elapsed_ms={elapsedMilliseconds:N4}" );
		string dotDiagnostic = TinyStoriesForwardStages.DescribeLinearDotProduct(
			layer0Ln1,
			weight,
			worstToken,
			worstFeature,
			comparison.ExpectedAtMaximumError,
			comparison.ActualAtMaximumError );
		if ( !comparison.Passed )
		{
			LlmLog.Warning( "QKV", $"layer=0 projection=V mismatch_dot {dotDiagnostic}" );
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 V projection parity failed: {comparison}" );
		}
		LlmLog.Trace( "QKV", $"layer=0 projection=V worst_dot {dotDiagnostic}" );
		return actual;
	}

	private static Tensor ValidateLayer0HeadSplitParity(
		string projectionLabel,
		string stageName,
		string expectedFile,
		string mountedPath,
		Tensor projection,
		TinyStoriesConfig config,
		ForwardReferenceDocument document )
	{
		ForwardReferenceStage stage = document.GetRequiredStage( stageName );
		int headDimension = config.HiddenSize / config.AttentionHeadCount;
		ValidateHeadReferenceStage(
			stage,
			expectedFile,
			projectionLabel,
			document.SequenceLength,
			config.HiddenSize,
			config.AttentionHeadCount,
			headDimension );

		float[] expected = ReferenceFloatData.LoadFromMounted( mountedPath, stage.Elements );
		TensorSummary expectedSummary = TensorDiagnostics.Summarize(
			$"python.layer0.{projectionLabel.ToLowerInvariant()}.heads",
			$"[{config.AttentionHeadCount},{document.SequenceLength},{headDimension}]",
			expected );
		if ( !expectedSummary.IsFinite )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Python layer 0 {projectionLabel} heads reference contains NaN or Infinity." );
		}

		LlmLog.Info(
			"ATTN",
			$"layer=0 stage={projectionLabel}_HEADS input={projection.ShapeText} " +
			$"reshape=[{document.SequenceLength},{config.AttentionHeadCount},{headDimension}] " +
			$"permute=[head,sequence,component] output=" +
			$"[{config.AttentionHeadCount},{document.SequenceLength},{headDimension}] " +
			$"feature=head*{headDimension}+component" );
		FastTimer timer = FastTimer.StartNew();
		Tensor actual = TinyStoriesForwardStages.SplitHeads(
			projection,
			config.AttentionHeadCount,
			$"forward.layer0.{projectionLabel.ToLowerInvariant()}.heads" );
		double elapsedMilliseconds = timer.ElapsedMilliSeconds;
		actual.RequireShape( config.AttentionHeadCount, document.SequenceLength, headDimension );
		ValidateHeadSplitCopiesExact( projectionLabel, projection, actual );
		TensorSummary actualSummary = TensorDiagnostics.Summarize( actual );
		LlmLog.Info( "ATTN", $"layer=0 stage={projectionLabel}_HEADS {actualSummary}" );
		if ( !actualSummary.IsFinite )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] C# layer 0 {projectionLabel} heads contain NaN or Infinity." );
		}

		NumericComparison comparison = TensorDiagnostics.Compare(
			expected,
			actual.Data,
			absoluteTolerance: 1e-5,
			relativeTolerance: 1e-5 );
		int worstComponent = comparison.MaximumErrorIndex % headDimension;
		int headToken = comparison.MaximumErrorIndex / headDimension;
		int worstToken = headToken % document.SequenceLength;
		int worstHead = headToken / document.SequenceLength;
		LlmLog.Info(
			"PARITY",
			$"layer0.{projectionLabel.ToLowerInvariant()}_heads end_to_end_from_csharp_projection " +
			$"{comparison} " +
			$"worst=[{worstHead},{worstToken},{worstComponent}]" );
		LlmLog.Info(
			"PERF",
			$"stage=layer0.{projectionLabel.ToLowerInvariant()}_heads shape={actual.ShapeText} " +
			$"elapsed_ms={elapsedMilliseconds:N4}" );
		string mapping = TinyStoriesForwardStages.DescribeHeadMapping(
			projection,
			actual,
			worstHead,
			worstToken,
			worstComponent );
		if ( !comparison.Passed )
		{
			LlmLog.Warning(
				"ATTN",
				$"layer=0 stage={projectionLabel}_HEADS mismatch_mapping {mapping}" );
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 {projectionLabel} head-split parity failed: {comparison}" );
		}
		LlmLog.Trace( "ATTN", $"layer=0 stage={projectionLabel}_HEADS worst_mapping {mapping}" );
		LlmLog.Trace(
			"ATTN",
			$"layer=0 stage={projectionLabel}_HEADS deterministic_mapping " +
			TinyStoriesForwardStages.DescribeHeadMapping( projection, actual, 7, 2, 2 ) );
		return actual;
	}

	private static void ValidateHeadSplitCopiesExact(
		string projectionLabel,
		Tensor projection,
		Tensor heads )
	{
		int headCount = heads.Shape[0];
		int sequenceLength = heads.Shape[1];
		int headDimension = heads.Shape[2];
		int hiddenSize = projection.Shape[1];
		for ( int head = 0; head < headCount; head++ )
		{
			for ( int token = 0; token < sequenceLength; token++ )
			{
				for ( int component = 0; component < headDimension; component++ )
				{
					int feature = head * headDimension + component;
					float source = projection.Data[token * hiddenSize + feature];
					float destination = heads.Data[
						(head * sequenceLength + token) * headDimension + component];
					if ( BitConverter.SingleToInt32Bits( source ) !=
						BitConverter.SingleToInt32Bits( destination ) )
					{
						throw new InvalidOperationException(
							$"[LLM:ERROR] Layer 0 {projectionLabel} head split altered bits at " +
							$"[{head},{token},{component}] from source feature {feature}: " +
							$"source={source:G9}, destination={destination:G9}." );
					}
				}
			}
		}
		LlmLog.Info(
			"PARITY",
			$"layer0.{projectionLabel.ToLowerInvariant()}_heads structural_copy " +
			$"elements={heads.ElementCount:N0} bit_exact PASS" );
	}

	private static Tensor ValidateLayer0ScaledUnmaskedScoresParity(
		Tensor queryHeads,
		Tensor keyHeads,
		TinyStoriesConfig config,
		ForwardReferenceDocument document )
	{
		ForwardReferenceStage stage = document.GetRequiredStage(
			"layer0_scores_scaled_unmasked" );
		int headDimension = config.HiddenSize / config.AttentionHeadCount;
		ValidateScoreReferenceStage(
			stage,
			"reference_layer0_scores_scaled_unmasked.f32",
			document.SequenceLength,
			config.AttentionHeadCount,
			headDimension );

		float[] expected = ReferenceFloatData.LoadFromMounted(
			LlmPaths.ReferenceLayer0ScoresScaledUnmasked,
			stage.Elements );
		TensorSummary expectedSummary = TensorDiagnostics.Summarize(
			"python.layer0.scores_scaled_unmasked",
			$"[{config.AttentionHeadCount},{document.SequenceLength},{document.SequenceLength}]",
			expected );
		if ( !expectedSummary.IsFinite )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] Python layer 0 scaled unmasked score reference " +
				"contains NaN or Infinity." );
		}

		LlmLog.Info(
			"ATTN",
			$"layer=0 stage=SCORES_UNMASKED_SCALED Q={queryHeads.ShapeText} " +
			$"K={keyHeads.ShapeText} output=" +
			$"[{config.AttentionHeadCount},{document.SequenceLength},{document.SequenceLength}] " +
			$"formula=sum_d(Q[head,query,d]*K[head,key,d]) scale={stage.Scale:G9} " +
			$"scaling=none(model_equivalent_factor_1) mask=none softmax=false" );
		FastTimer timer = FastTimer.StartNew();
		Tensor actual = TinyStoriesForwardStages.ComputeScaledUnmaskedAttentionScores(
			queryHeads,
			keyHeads,
			stage.Scale,
			"forward.layer0.scores_scaled_unmasked" );
		double elapsedMilliseconds = timer.ElapsedMilliSeconds;
		actual.RequireShape(
			config.AttentionHeadCount,
			document.SequenceLength,
			document.SequenceLength );
		TensorSummary actualSummary = TensorDiagnostics.Summarize( actual );
		LlmLog.Info( "ATTN", $"layer=0 stage=SCORES_UNMASKED_SCALED {actualSummary}" );
		if ( !actualSummary.IsFinite )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] C# layer 0 scaled unmasked scores contain NaN or Infinity." );
		}

		NumericComparison comparison = TensorDiagnostics.Compare(
			expected,
			actual.Data,
			absoluteTolerance: 1e-5,
			relativeTolerance: 1e-5 );
		int worstKey = comparison.MaximumErrorIndex % document.SequenceLength;
		int headQuery = comparison.MaximumErrorIndex / document.SequenceLength;
		int worstQuery = headQuery % document.SequenceLength;
		int worstHead = headQuery / document.SequenceLength;
		LlmLog.Info(
			"PARITY",
			$"layer0.scores_scaled_unmasked {comparison} " +
			$"worst=[{worstHead},{worstQuery},{worstKey}]" );
		LlmLog.Info(
			"PERF",
			$"stage=layer0.scores_scaled_unmasked shape={actual.ShapeText} " +
			$"elapsed_ms={elapsedMilliseconds:N4}" );

		string manual00 = TinyStoriesForwardStages.DescribeAttentionScore(
			queryHeads,
			keyHeads,
			actual,
			0,
			0,
			0,
			stage.Scale,
			expected[0] );
		LlmLog.Info( "ATTN", $"layer=0 manual_score {manual00}" );
		int nonDiagonalIndex = (0 * document.SequenceLength + 1) * document.SequenceLength + 2;
		LlmLog.Trace(
			"ATTN",
			"layer=0 manual_score " +
			TinyStoriesForwardStages.DescribeAttentionScore(
				queryHeads,
				keyHeads,
				actual,
				0,
				1,
				2,
				stage.Scale,
				expected[nonDiagonalIndex] ) );

		if ( !comparison.Passed )
		{
			string worstDiagnostic = TinyStoriesForwardStages.DescribeAttentionScore(
				queryHeads,
				keyHeads,
				actual,
				worstHead,
				worstQuery,
				worstKey,
				stage.Scale,
				comparison.ExpectedAtMaximumError );
			LlmLog.Warning(
				"ATTN",
				$"layer=0 stage=SCORES_UNMASKED_SCALED mismatch_score {worstDiagnostic}" );
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 scaled unmasked score parity failed: {comparison}" );
		}
		return actual;
	}

	private static void ValidateLayer0MaskStructure(
		TinyStoriesConfig config,
		ForwardReferenceDocument document,
		AttentionMaskReferenceDocument mask )
	{
		if ( mask.Format != "SBOXLLM_ATTENTION_MASK_REFERENCE" || mask.Version != 1 ||
			mask.Model != "roneneldan/TinyStories-Instruct-1M" )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 mask reference expected " +
				$"SBOXLLM_ATTENTION_MASK_REFERENCE/1 for " +
				$"roneneldan/TinyStories-Instruct-1M, found " +
				$"{mask.Format}/{mask.Version} for '{mask.Model}'." );
		}
		if ( mask.Layer != 0 || mask.AttentionType != "global" ||
			config.AttentionLayers[0] != mask.AttentionType ||
			mask.WindowSize != config.AttentionWindowSize ||
			mask.QueryLength != document.SequenceLength ||
			mask.KeyLength != document.SequenceLength )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 mask metadata mismatch: layer={mask.Layer}, " +
				$"type={mask.AttentionType}, config_type={config.AttentionLayers[0]}, " +
				$"window={mask.WindowSize}/{config.AttentionWindowSize}, " +
				$"query={mask.QueryLength}, key={mask.KeyLength}, " +
				$"expected_sequence={document.SequenceLength}." );
		}
		bool sourceShapeMatches = mask.SourceBiasShape is not null &&
			mask.SourceBiasShape.Length == 4 &&
			mask.SourceBiasShape[0] == 1 && mask.SourceBiasShape[1] == 1 &&
			mask.SourceBiasShape[2] == config.MaximumPositions &&
			mask.SourceBiasShape[3] == config.MaximumPositions;
		bool logicalShapeMatches = mask.Shape is not null && mask.Shape.Length == 2 &&
			mask.Shape[0] == document.SequenceLength &&
			mask.Shape[1] == document.SequenceLength;
		bool axisOrderMatches = mask.LogicalAxisOrder is not null &&
			mask.LogicalAxisOrder.Length == 2 &&
			mask.LogicalAxisOrder[0] == "query_sequence" &&
			mask.LogicalAxisOrder[1] == "key_sequence";
		if ( !sourceShapeMatches || mask.SourceBiasDtype != "bool" ||
			mask.SourceBiasPersistent ||
			mask.SourceSlice != "bias[:,:,key_length-query_length:key_length,:key_length]" ||
			!logicalShapeMatches || !axisOrderMatches ||
			mask.FlattenedStorageOrder !=
				"C-order [query_sequence,key_sequence]; " +
				"flat=query*key_sequence_length+key" ||
			mask.MaskSemantics != "global causal; allowed iff key <= query" )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] Layer 0 mask source shape/dtype/slice or logical layout " +
				"does not match the verified Transformers 5.15.0 semantics." );
		}

		int sentinelBits = BitConverter.SingleToInt32Bits( mask.MaskedSentinel );
		if ( !mask.MaskedSentinelIsFinite || !float.IsFinite( mask.MaskedSentinel ) ||
			!(mask.MaskedSentinel < 0) ||
			sentinelBits != unchecked((int)0xFF7FFFFF) ||
			mask.MaskedSentinelFloat32Bits != "0xFF7FFFFF" ||
			mask.MaskOperation != "torch.where(causal_mask, scores, mask_value)" )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 mask sentinel expected finite FP32 minimum " +
				$"bits=0xFF7FFFFF, found value={mask.MaskedSentinel:G9}, " +
				$"bits=0x{sentinelBits:X8}, metadata_bits={mask.MaskedSentinelFloat32Bits}." );
		}

		if ( mask.Allowed is null || mask.Allowed.Length != document.SequenceLength )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] Layer 0 boolean mask reference has the wrong query dimension." );
		}
		int allowed = 0;
		int masked = 0;
		for ( int query = 0; query < document.SequenceLength; query++ )
		{
			if ( mask.Allowed[query] is null ||
				mask.Allowed[query].Length != document.SequenceLength )
			{
				throw new InvalidOperationException(
					$"[LLM:ERROR] Layer 0 boolean mask reference row {query} has " +
					$"length {mask.Allowed[query]?.Length ?? 0}, expected " +
					$"{document.SequenceLength}." );
			}
			for ( int key = 0; key < document.SequenceLength; key++ )
			{
				bool expected = mask.Allowed[query][key];
				bool actual = TinyStoriesForwardStages.IsLayer0AttentionAllowed(
					mask.AttentionType,
					query,
					key,
					document.SequenceLength,
					document.SequenceLength );
				if ( expected != actual )
				{
					throw new InvalidOperationException(
						$"[LLM:ERROR] Layer 0 structural mask mismatch at " +
						$"query={query}, key={key}: Python={expected}, C#={actual}." );
				}
				if ( actual ) allowed++; else masked++;
			}
		}

		int expectedAllHeadsAllowed = allowed * config.AttentionHeadCount;
		int expectedAllHeadsMasked = masked * config.AttentionHeadCount;
		if ( allowed != mask.AllowedCountPerHead || masked != mask.MaskedCountPerHead ||
			mask.HeadCount != config.AttentionHeadCount || !mask.SharedAcrossHeads ||
			mask.AllowedCountAllHeads != expectedAllHeadsAllowed ||
			mask.MaskedCountAllHeads != expectedAllHeadsMasked ||
			mask.FirstLocalAttentionLayer != 1 || mask.Status != "PASS" )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 mask counts/pattern metadata mismatch: " +
				$"per_head={allowed}/{masked}, metadata={mask.AllowedCountPerHead}/" +
				$"{mask.MaskedCountPerHead}, all_heads={expectedAllHeadsAllowed}/" +
				$"{expectedAllHeadsMasked}, metadata={mask.AllowedCountAllHeads}/" +
				$"{mask.MaskedCountAllHeads}." );
		}

		LlmLog.Info(
			"MASK",
			$"layer=0 type={mask.AttentionType} shape=[{document.SequenceLength}," +
			$"{document.SequenceLength}] allowed={allowed} masked={masked} " +
			$"all_heads_allowed={expectedAllHeadsAllowed} " +
			$"all_heads_masked={expectedAllHeadsMasked} " +
			$"sentinel={mask.MaskedSentinel:G9} sentinel_bits=0x{sentinelBits:X8} " +
			"sentinel_finite=true structure PASS" );
	}

	private static Tensor ValidateLayer0MaskedScoresParity(
		Tensor unmaskedScores,
		TinyStoriesConfig config,
		ForwardReferenceDocument document,
		AttentionMaskReferenceDocument mask )
	{
		ForwardReferenceStage stage = document.GetRequiredStage(
			"layer0_scores_masked_pre_softmax" );
		int headDimension = config.HiddenSize / config.AttentionHeadCount;
		ValidateMaskedScoreReferenceStage(
			stage,
			document.SequenceLength,
			config.AttentionHeadCount,
			headDimension,
			mask );

		float[] expectedMasked = ReferenceFloatData.LoadFromMounted(
			LlmPaths.ReferenceLayer0ScoresMaskedPreSoftmax,
			stage.Elements );
		float[] expectedUnmasked = ReferenceFloatData.LoadFromMounted(
			LlmPaths.ReferenceLayer0ScoresScaledUnmasked,
			stage.Elements );
		TensorSummary expectedSummary = TensorDiagnostics.Summarize(
			"python.layer0.scores_masked_pre_softmax",
			$"[{config.AttentionHeadCount},{document.SequenceLength}," +
			$"{document.SequenceLength}]",
			expectedMasked );
		if ( !expectedSummary.IsFinite )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] Python layer 0 masked pre-softmax score reference " +
				"contains unexpected NaN or Infinity." );
		}

		LlmLog.Info(
			"MASK",
			$"layer=0 stage=SCORES_MASKED_PRE_SOFTMAX input={unmaskedScores.ShapeText} " +
			$"type={mask.AttentionType} predicate='key <= key_length-query_length+query' " +
			$"sentinel={mask.MaskedSentinel:G9} sentinel_bits=" +
			$"{mask.MaskedSentinelFloat32Bits} softmax=false" );
		FastTimer timer = FastTimer.StartNew();
		Tensor actual = TinyStoriesForwardStages.ApplyLayer0AttentionMask(
			unmaskedScores,
			mask.AttentionType,
			mask.MaskedSentinel,
			"forward.layer0.scores_masked_pre_softmax" );
		double elapsedMilliseconds = timer.ElapsedMilliSeconds;
		actual.RequireShape(
			config.AttentionHeadCount,
			document.SequenceLength,
			document.SequenceLength );
		TensorSummary actualSummary = TensorDiagnostics.Summarize( actual );
		LlmLog.Info(
			"MASK",
			$"layer=0 stage=SCORES_MASKED_PRE_SOFTMAX {actualSummary} " +
			$"expected_mask_sentinel_count={stage.MaskedCountAllHeads}" );
		if ( !actualSummary.IsFinite )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] C# layer 0 masked pre-softmax scores contain unexpected " +
				"NaN or Infinity; the verified sentinel is finite." );
		}

		float[] expectedAllowed = new float[stage.AllowedCountAllHeads];
		float[] actualAllowed = new float[stage.AllowedCountAllHeads];
		int[] allowedFlatIndices = new int[stage.AllowedCountAllHeads];
		int allowedCount = 0;
		int maskedCount = 0;
		int exactSentinelMatches = 0;
		int sentinelMismatches = 0;
		int unexpectedNaN = 0;
		int unexpectedPositiveInfinity = 0;
		int unexpectedNegativeInfinity = 0;
		int sentinelBits = BitConverter.SingleToInt32Bits( mask.MaskedSentinel );
		int queryLength = document.SequenceLength;
		int keyLength = document.SequenceLength;
		for ( int head = 0; head < config.AttentionHeadCount; head++ )
		{
			for ( int query = 0; query < queryLength; query++ )
			{
				for ( int key = 0; key < keyLength; key++ )
				{
					int index = (head * queryLength + query) * keyLength + key;
					float actualValue = actual.Data[index];
					if ( float.IsNaN( actualValue ) ) unexpectedNaN++;
					else if ( float.IsPositiveInfinity( actualValue ) ) unexpectedPositiveInfinity++;
					else if ( float.IsNegativeInfinity( actualValue ) ) unexpectedNegativeInfinity++;

					bool allowed = TinyStoriesForwardStages.IsLayer0AttentionAllowed(
						mask.AttentionType,
						query,
						key,
						queryLength,
						keyLength );
					if ( allowed )
					{
						if ( BitConverter.SingleToInt32Bits( actualValue ) !=
							BitConverter.SingleToInt32Bits( unmaskedScores.Data[index] ) )
						{
							throw new InvalidOperationException(
								$"[LLM:ERROR] Layer 0 mask changed allowed C# score bits at " +
								$"[{head},{query},{key}]: unmasked=" +
								$"{unmaskedScores.Data[index]:G9}, masked={actualValue:G9}." );
						}
						if ( BitConverter.SingleToInt32Bits( expectedMasked[index] ) !=
							BitConverter.SingleToInt32Bits( expectedUnmasked[index] ) )
						{
							throw new InvalidOperationException(
								$"[LLM:ERROR] Python masked reference changed allowed score bits at " +
								$"[{head},{query},{key}]." );
						}
						expectedAllowed[allowedCount] = expectedMasked[index];
						actualAllowed[allowedCount] = actualValue;
						allowedFlatIndices[allowedCount] = index;
						allowedCount++;
					}
					else
					{
						maskedCount++;
						bool expectedSentinel =
							BitConverter.SingleToInt32Bits( expectedMasked[index] ) == sentinelBits;
						bool actualSentinel =
							BitConverter.SingleToInt32Bits( actualValue ) == sentinelBits;
						if ( expectedSentinel && actualSentinel ) exactSentinelMatches++;
						else sentinelMismatches++;
					}
				}
			}
		}

		if ( allowedCount != stage.AllowedCountAllHeads ||
			maskedCount != stage.MaskedCountAllHeads )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 mask application counts expected allowed=" +
				$"{stage.AllowedCountAllHeads}, masked={stage.MaskedCountAllHeads}; " +
				$"found allowed={allowedCount}, masked={maskedCount}." );
		}

		NumericComparison allowedComparison = TensorDiagnostics.Compare(
			expectedAllowed,
			actualAllowed,
			absoluteTolerance: 1e-5,
			relativeTolerance: 1e-5 );
		int worstFlat = allowedFlatIndices[allowedComparison.MaximumErrorIndex];
		int worstKey = worstFlat % keyLength;
		int headQuery = worstFlat / keyLength;
		int worstQuery = headQuery % queryLength;
		int worstHead = headQuery / queryLength;
		LlmLog.Info(
			"PARITY",
			$"layer0.scores_masked_pre_softmax allowed_coordinates " +
			$"{allowedComparison} worst=[{worstHead},{worstQuery},{worstKey}]" );
		bool sentinelPassed = sentinelMismatches == 0 &&
			exactSentinelMatches == maskedCount && unexpectedNaN == 0 &&
			unexpectedPositiveInfinity == 0 && unexpectedNegativeInfinity == 0;
		LlmLog.Info(
			"PARITY",
			$"layer0.scores_masked_pre_softmax masked_coordinates count={maskedCount} " +
			$"exact_sentinel_matches={exactSentinelMatches} mismatches={sentinelMismatches} " +
			$"unexpected_nan={unexpectedNaN} unexpected_pos_inf=" +
			$"{unexpectedPositiveInfinity} unexpected_neg_inf={unexpectedNegativeInfinity} " +
			$"{(sentinelPassed ? "PASS" : "FAIL")}" );
		LlmLog.Info(
			"PERF",
			$"stage=layer0.scores_masked_pre_softmax shape={actual.ShapeText} " +
			$"elapsed_ms={elapsedMilliseconds:N4}" );

		LlmLog.Info(
			"MASK",
			"layer=0 allowed_example " +
			TinyStoriesForwardStages.DescribeAttentionMaskApplication(
				unmaskedScores,
				actual,
				mask.AttentionType,
				mask.MaskedSentinel,
				0,
				2,
				1 ) );
		LlmLog.Info(
			"MASK",
			"layer=0 forbidden_example " +
			TinyStoriesForwardStages.DescribeAttentionMaskApplication(
				unmaskedScores,
				actual,
				mask.AttentionType,
				mask.MaskedSentinel,
				0,
				1,
				3 ) );

		if ( !allowedComparison.Passed )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 allowed-coordinate masked score parity failed: " +
				$"{allowedComparison}, worst=[{worstHead},{worstQuery},{worstKey}]." );
		}
		if ( !sentinelPassed )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 masked sentinel parity failed: exact=" +
				$"{exactSentinelMatches}/{maskedCount}, mismatches={sentinelMismatches}, " +
				$"nan={unexpectedNaN}, +inf={unexpectedPositiveInfinity}, " +
				$"-inf={unexpectedNegativeInfinity}." );
		}

		return actual;
	}

	private static Tensor ValidateLayer0AttentionProbabilityParity(
		Tensor maskedScores,
		TinyStoriesConfig config,
		ForwardReferenceDocument document,
		AttentionMaskReferenceDocument mask )
	{
		ForwardReferenceStage stage = document.GetRequiredStage(
			"layer0_attention_probs" );
		int headDimension = config.HiddenSize / config.AttentionHeadCount;
		ValidateAttentionProbabilityReferenceStage(
			stage,
			document.SequenceLength,
			config.AttentionHeadCount,
			headDimension,
			mask );

		maskedScores.RequireShape(
			config.AttentionHeadCount,
			document.SequenceLength,
			document.SequenceLength );
		float[] expected = ReferenceFloatData.LoadFromMounted(
			LlmPaths.ReferenceLayer0AttentionProbs,
			stage.Elements );
		Tensor expectedTensor = new(
			"python.layer0.attention_probs",
			new[] {
				config.AttentionHeadCount,
				document.SequenceLength,
				document.SequenceLength
			},
			expected );
		TensorSummary expectedSummary = TensorDiagnostics.Summarize( expectedTensor );
		if ( !expectedSummary.IsFinite )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] Python layer 0 attention-probability reference " +
				"contains NaN or Infinity." );
		}

		LlmLog.Info(
			"ATTN",
			$"layer=0 stage=SOFTMAX input={maskedScores.ShapeText} output_expected=" +
			$"[{config.AttentionHeadCount},{document.SequenceLength}," +
			$"{document.SequenceLength}] axis=key_sequence(dim=-1) input_dtype=float32 " +
			"output_dtype=float32 algorithm=stable_max_subtraction exp=MathF.Exp(float) " +
			"dropout=false attention_times_value=false" );
		FastTimer timer = FastTimer.StartNew();
		Tensor actual = TinyStoriesForwardStages.ComputeAttentionProbabilities(
			maskedScores,
			"forward.layer0.attention_probs" );
		double elapsedMilliseconds = timer.ElapsedMilliSeconds;
		actual.RequireShape(
			config.AttentionHeadCount,
			document.SequenceLength,
			document.SequenceLength );
		TensorSummary actualSummary = TensorDiagnostics.Summarize( actual );
		LlmLog.Info(
			"ATTN",
			$"layer=0 stage=ATTENTION_PROBS {actualSummary}" );

		int queryLength = document.SequenceLength;
		int keyLength = document.SequenceLength;
		int finiteCount = 0;
		int nonNegativeCount = 0;
		int allowedNonzeroCount = 0;
		int maskedCount = 0;
		int exactMaskedPositiveZeros = 0;
		int maskedZeroMismatches = 0;
		float maximumRowSumDeviation = 0;
		float worstRowSum = 0;
		int worstRowHead = 0;
		int worstRowQuery = 0;
		for ( int head = 0; head < config.AttentionHeadCount; head++ )
		{
			for ( int query = 0; query < queryLength; query++ )
			{
				float rowSum = 0;
				int allowedNonzeroInRow = 0;
				for ( int key = 0; key < keyLength; key++ )
				{
					int index = (head * queryLength + query) * keyLength + key;
					float value = actual.Data[index];
					if ( float.IsFinite( value ) ) finiteCount++;
					if ( value >= 0 ) nonNegativeCount++;
					rowSum += value;

					bool allowed = TinyStoriesForwardStages.IsLayer0AttentionAllowed(
						mask.AttentionType,
						query,
						key,
						queryLength,
						keyLength );
					if ( allowed )
					{
						if ( value > 0 )
						{
							allowedNonzeroCount++;
							allowedNonzeroInRow++;
						}
					}
					else
					{
						maskedCount++;
						bool expectedPositiveZero =
							BitConverter.SingleToInt32Bits( expected[index] ) == 0;
						bool actualPositiveZero =
							BitConverter.SingleToInt32Bits( value ) == 0;
						if ( expectedPositiveZero && actualPositiveZero )
						{
							exactMaskedPositiveZeros++;
						}
						else
						{
							maskedZeroMismatches++;
						}
					}
				}

				if ( allowedNonzeroInRow == 0 )
				{
					throw new InvalidOperationException(
						$"[LLM:ERROR] Layer 0 softmax row [{head},{query}] has no " +
						"non-zero causally allowed probability." );
				}
				float rowDeviation = MathF.Abs( rowSum - 1.0f );
				if ( rowDeviation > maximumRowSumDeviation )
				{
					maximumRowSumDeviation = rowDeviation;
					worstRowSum = rowSum;
					worstRowHead = head;
					worstRowQuery = query;
				}
			}
		}

		bool invariantPassed = finiteCount == actual.Data.Length &&
			nonNegativeCount == actual.Data.Length &&
			maximumRowSumDeviation <= 1.0e-6f &&
			allowedNonzeroCount == stage.NonzeroProbabilityCount &&
			maskedCount == stage.MaskedZeroCount &&
			exactMaskedPositiveZeros == maskedCount &&
			maskedZeroMismatches == 0;
		LlmLog.Info(
			"ATTN",
			$"layer=0 stage=SOFTMAX0_INVARIANTS rows=" +
			$"{config.AttentionHeadCount * queryLength} finite={finiteCount}/" +
			$"{actual.Data.Length} non_negative={nonNegativeCount}/{actual.Data.Length} " +
			$"max_row_sum_deviation={maximumRowSumDeviation:G12} " +
			$"worst_row=[{worstRowHead},{worstRowQuery}] row_sum={worstRowSum:G12} " +
			$"allowed_nonzero={allowedNonzeroCount}/{stage.NonzeroProbabilityCount} " +
			$"masked_positive_zero={exactMaskedPositiveZeros}/{maskedCount} " +
			$"masked_zero_mismatches={maskedZeroMismatches} " +
			$"{(invariantPassed ? "PASS" : "FAIL")}" );
		if ( !invariantPassed )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 softmax invariants failed: finite=" +
				$"{finiteCount}/{actual.Data.Length}, non_negative={nonNegativeCount}/" +
				$"{actual.Data.Length}, max_row_sum_deviation=" +
				$"{maximumRowSumDeviation:G12}, allowed_nonzero={allowedNonzeroCount}/" +
				$"{stage.NonzeroProbabilityCount}, masked_positive_zero=" +
				$"{exactMaskedPositiveZeros}/{maskedCount}, masked_zero_mismatches=" +
				$"{maskedZeroMismatches}." );
		}

		NumericComparison comparison = TensorDiagnostics.Compare(
			expected,
			actual.Data,
			absoluteTolerance: 1e-6,
			relativeTolerance: 1e-5 );
		int worstKey = comparison.MaximumErrorIndex % keyLength;
		int headQuery = comparison.MaximumErrorIndex / keyLength;
		int worstQuery = headQuery % queryLength;
		int worstHead = headQuery / queryLength;
		LlmLog.Info(
			"PARITY",
			$"layer0.attention_probs {comparison} " +
			$"worst=[{worstHead},{worstQuery},{worstKey}] " +
			$"max_row_sum_deviation={maximumRowSumDeviation:G12}" );
		LlmLog.Info(
			"PARITY",
			$"layer0.attention_probs masked_coordinates count={maskedCount} " +
			$"exact_positive_zero_matches={exactMaskedPositiveZeros} " +
			$"mismatches={maskedZeroMismatches} PASS" );
		LlmLog.Info(
			"ATTN",
			"layer=0 softmax_query0 " +
			TinyStoriesForwardStages.DescribeAttentionSoftmaxRow(
				maskedScores,
				actual,
				expectedTensor,
				0,
				0 ) );
		LlmLog.Info(
			"ATTN",
			"layer=0 softmax_full_row " +
			TinyStoriesForwardStages.DescribeAttentionSoftmaxRow(
				maskedScores,
				actual,
				expectedTensor,
				0,
				3 ) );
		LlmLog.Info(
			"PERF",
			$"stage=layer0.attention_softmax shape={actual.ShapeText} " +
			$"elapsed_ms={elapsedMilliseconds:N4}" );

		if ( !comparison.Passed )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 attention-probability parity failed: " +
				$"{comparison}, worst=[{worstHead},{worstQuery},{worstKey}]." );
		}

		return actual;
	}

	private static Tensor ValidateLayer0AttentionContextParity(
		Tensor probabilities,
		Tensor valueHeads,
		TinyStoriesConfig config,
		ForwardReferenceDocument document,
		AttentionMaskReferenceDocument mask )
	{
		ForwardReferenceStage stage = document.GetRequiredStage(
			"layer0_attention_context_heads" );
		int headDimension = config.HiddenSize / config.AttentionHeadCount;
		ValidateAttentionContextReferenceStage(
			stage,
			document.SequenceLength,
			config.AttentionHeadCount,
			headDimension );

		probabilities.RequireShape(
			config.AttentionHeadCount,
			document.SequenceLength,
			document.SequenceLength );
		valueHeads.RequireShape(
			config.AttentionHeadCount,
			document.SequenceLength,
			headDimension );
		float[] expected = ReferenceFloatData.LoadFromMounted(
			LlmPaths.ReferenceLayer0AttentionContextHeads,
			stage.Elements );
		Tensor expectedTensor = new(
			"python.layer0.attention_context_heads",
			new[] {
				config.AttentionHeadCount,
				document.SequenceLength,
				headDimension
			},
			expected );
		TensorSummary expectedSummary = TensorDiagnostics.Summarize( expectedTensor );
		if ( !expectedSummary.IsFinite )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] Python layer 0 per-head attention-context reference " +
				"contains NaN or Infinity." );
		}

		float[] probabilitiesBefore = new float[probabilities.Data.Length];
		for ( int index = 0; index < probabilities.Data.Length; index++ )
		{
			probabilitiesBefore[index] = probabilities.Data[index];
		}
		float[] valuesBefore = new float[valueHeads.Data.Length];
		for ( int index = 0; index < valueHeads.Data.Length; index++ )
		{
			valuesBefore[index] = valueHeads.Data[index];
		}
		LlmLog.Info(
			"ATTN",
			$"layer=0 stage=ATTENTION_CONTEXT input_probs={probabilities.ShapeText} " +
			$"input_V={valueHeads.ShapeText} output_expected=" +
			$"[{config.AttentionHeadCount},{document.SequenceLength},{headDimension}] " +
			"formula=sum_k probability[head,query,k]*V[head,k,component] " +
			"input_dtype=float32 value_dtype=float32 output_dtype=float32 " +
			"dropout=false heads_merged=false output_projection=false" );
		FastTimer timer = FastTimer.StartNew();
		Tensor actual = TinyStoriesForwardStages.ComputeAttentionContextHeads(
			probabilities,
			valueHeads,
			"forward.layer0.attention_context_heads" );
		double elapsedMilliseconds = timer.ElapsedMilliSeconds;
		actual.RequireShape(
			config.AttentionHeadCount,
			document.SequenceLength,
			headDimension );
		TensorSummary actualSummary = TensorDiagnostics.Summarize( actual );
		LlmLog.Info(
			"ATTN",
			$"layer=0 stage=ATTENTION_CONTEXT0_HEADS {actualSummary}" );

		int probabilityMutationCount = 0;
		for ( int index = 0; index < probabilities.Data.Length; index++ )
		{
			if ( BitConverter.SingleToInt32Bits( probabilitiesBefore[index] ) !=
				BitConverter.SingleToInt32Bits( probabilities.Data[index] ) )
			{
				probabilityMutationCount++;
			}
		}
		int valueMutationCount = 0;
		for ( int index = 0; index < valueHeads.Data.Length; index++ )
		{
			if ( BitConverter.SingleToInt32Bits( valuesBefore[index] ) !=
				BitConverter.SingleToInt32Bits( valueHeads.Data[index] ) )
			{
				valueMutationCount++;
			}
		}

		int headCount = config.AttentionHeadCount;
		int queryLength = document.SequenceLength;
		int keyLength = document.SequenceLength;
		int query0ExactCount = 0;
		int query0MismatchHead = -1;
		int query0MismatchComponent = -1;
		for ( int head = 0; head < headCount; head++ )
		{
			for ( int component = 0; component < headDimension; component++ )
			{
				int contextIndex = head * queryLength * headDimension + component;
				int valueIndex = head * keyLength * headDimension + component;
				if ( BitConverter.SingleToInt32Bits( actual.Data[contextIndex] ) ==
					BitConverter.SingleToInt32Bits( valueHeads.Data[valueIndex] ) )
				{
					query0ExactCount++;
				}
				else if ( query0MismatchHead < 0 )
				{
					query0MismatchHead = head;
					query0MismatchComponent = component;
				}
			}
		}
		bool query0Passed = query0ExactCount == stage.Query0IdentityElements;
		LlmLog.Info(
			"ATTN",
			$"layer=0 stage=ATTENTION_CONTEXT0_INVARIANTS " +
			$"query0_context_equals_v0 bit_exact={query0ExactCount}/" +
			$"{stage.Query0IdentityElements} first_mismatch=" +
			$"[{query0MismatchHead},{query0MismatchComponent}] " +
			$"{(query0Passed ? "PASS" : "FAIL")}" );

		int convexChecked = 0;
		int convexViolations = 0;
		float maximumRangeExcess = 0;
		int firstConvexHead = -1;
		int firstConvexQuery = -1;
		int firstConvexComponent = -1;
		for ( int head = 0; head < headCount; head++ )
		{
			for ( int query = 0; query < queryLength; query++ )
			{
				for ( int component = 0; component < headDimension; component++ )
				{
					float minimum = float.PositiveInfinity;
					float maximum = float.NegativeInfinity;
					for ( int key = 0; key < keyLength; key++ )
					{
						if ( !TinyStoriesForwardStages.IsLayer0AttentionAllowed(
							mask.AttentionType,
							query,
							key,
							queryLength,
							keyLength ) )
						{
							continue;
						}
						int valueIndex =
							(head * keyLength + key) * headDimension + component;
						float value = valueHeads.Data[valueIndex];
						minimum = MathF.Min( minimum, value );
						maximum = MathF.Max( maximum, value );
					}

					int contextIndex =
						(head * queryLength + query) * headDimension + component;
					float contextValue = actual.Data[contextIndex];
					float below = minimum - contextValue;
					float above = contextValue - maximum;
					float excess = MathF.Max( below, above );
					convexChecked++;
					if ( excess > stage.ConvexRangeTolerance )
					{
						convexViolations++;
						maximumRangeExcess = MathF.Max( maximumRangeExcess, excess );
						if ( firstConvexHead < 0 )
						{
							firstConvexHead = head;
							firstConvexQuery = query;
							firstConvexComponent = component;
						}
					}
				}
			}
		}
		bool convexPassed = convexChecked == stage.ConvexRangeCheckedElements &&
			convexViolations == stage.ConvexRangeViolations;
		LlmLog.Info(
			"ATTN",
			$"layer=0 stage=ATTENTION_CONTEXT0_INVARIANTS convex_range " +
			$"checked={convexChecked}/{stage.ConvexRangeCheckedElements} " +
			$"violations={convexViolations} tolerance={stage.ConvexRangeTolerance:G9} " +
			$"max_excess={maximumRangeExcess:G12} first_violation=" +
			$"[{firstConvexHead},{firstConvexQuery},{firstConvexComponent}] " +
			$"{(convexPassed ? "PASS" : "FAIL")}" );

		bool invariantPassed = actualSummary.IsFinite && query0Passed && convexPassed &&
			probabilityMutationCount == 0 && valueMutationCount == 0;
		LlmLog.Info(
			"ATTN",
			$"layer=0 stage=ATTENTION_CONTEXT0_INVARIANTS finite=" +
			$"{actualSummary.FiniteCount}/{actualSummary.ElementCount} " +
			$"probability_mutations={probabilityMutationCount} " +
			$"V_mutations={valueMutationCount} {(invariantPassed ? "PASS" : "FAIL")}" );
		if ( !invariantPassed )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 attention-context invariants failed: finite=" +
				$"{actualSummary.FiniteCount}/{actualSummary.ElementCount}, query0_bit_exact=" +
				$"{query0ExactCount}/{stage.Query0IdentityElements}, convex_violations=" +
				$"{convexViolations}, probability_mutations={probabilityMutationCount}, " +
				$"V_mutations={valueMutationCount}." );
		}

		NumericComparison comparison = TensorDiagnostics.Compare(
			expected,
			actual.Data,
			absoluteTolerance: 1e-6,
			relativeTolerance: 1e-5 );
		int worstComponent = comparison.MaximumErrorIndex % headDimension;
		int headQuery = comparison.MaximumErrorIndex / headDimension;
		int worstQuery = headQuery % queryLength;
		int worstHead = headQuery / queryLength;
		LlmLog.Info(
			"PARITY",
			$"layer0.attention_context_heads {comparison} " +
			$"worst=[{worstHead},{worstQuery},{worstComponent}]" );
		LlmLog.Info(
			"ATTN",
			"layer=0 context_query0 " +
			TinyStoriesForwardStages.DescribeAttentionContextElement(
				probabilities,
				valueHeads,
				actual,
				0,
				0,
				0,
				expected[0] ) );
		int manualIndex = (3 * headDimension);
		LlmLog.Info(
			"ATTN",
			"layer=0 context_full_row " +
			TinyStoriesForwardStages.DescribeAttentionContextElement(
				probabilities,
				valueHeads,
				actual,
				0,
				3,
				0,
				expected[manualIndex] ) );
		LlmLog.Info(
			"PERF",
			$"stage=layer0.attention_probabilities_times_v shape={actual.ShapeText} " +
			$"elapsed_ms={elapsedMilliseconds:N4}" );

		if ( !comparison.Passed )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 per-head attention-context parity failed: " +
				$"{comparison}, worst=[{worstHead},{worstQuery},{worstComponent}]." );
		}

		return actual;
	}

	private static Tensor ValidateLayer0AttentionMergeParity(
		Tensor contextHeads,
		TinyStoriesConfig config,
		ForwardReferenceDocument document )
	{
		ForwardReferenceStage stage = document.GetRequiredStage(
			"layer0_attention_merged" );
		int headDimension = config.HiddenSize / config.AttentionHeadCount;
		ValidateAttentionMergedReferenceStage(
			stage,
			document.SequenceLength,
			config.HiddenSize,
			config.AttentionHeadCount,
			headDimension );

		contextHeads.RequireShape(
			config.AttentionHeadCount,
			document.SequenceLength,
			headDimension );
		float[] expected = ReferenceFloatData.LoadFromMounted(
			LlmPaths.ReferenceLayer0AttentionMerged,
			stage.Elements );
		TensorSummary expectedSummary = TensorDiagnostics.Summarize(
			"python.layer0.attention_merged",
			$"[{document.SequenceLength},{config.HiddenSize}]",
			expected );
		if ( !expectedSummary.IsFinite )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] Python layer 0 merged-attention reference contains " +
				"NaN or Infinity." );
		}

		LlmLog.Info(
			"ATTN",
			$"layer=0 stage=MERGE_HEADS input={contextHeads.ShapeText} " +
			$"output_expected=[{document.SequenceLength},{config.HiddenSize}] " +
			"mapping=merged[token,head*head_dimension+component]=" +
			"context[head,token,component] arithmetic=false" );
		FastTimer timer = FastTimer.StartNew();
		Tensor actual = TinyStoriesForwardStages.MergeHeads(
			contextHeads,
			"forward.layer0.attention_merged" );
		double elapsedMilliseconds = timer.ElapsedMilliSeconds;
		actual.RequireShape( document.SequenceLength, config.HiddenSize );
		TensorSummary actualSummary = TensorDiagnostics.Summarize( actual );
		LlmLog.Info( "ATTN", $"layer=0 stage=MERGE_HEADS0 {actualSummary}" );
		if ( !actualSummary.IsFinite )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] C# layer 0 merged attention contains NaN or Infinity." );
		}

		float[] expectedContext = ReferenceFloatData.LoadFromMounted(
			LlmPaths.ReferenceLayer0AttentionContextHeads,
			stage.Elements );
		int actualExactMappings = 0;
		int referenceExactMappings = 0;
		int firstActualMismatch = -1;
		int firstReferenceMismatch = -1;
		for ( int token = 0; token < document.SequenceLength; token++ )
		{
			for ( int head = 0; head < config.AttentionHeadCount; head++ )
			{
				for ( int component = 0; component < headDimension; component++ )
				{
					int feature = head * headDimension + component;
					int sourceIndex =
						(head * document.SequenceLength + token) * headDimension + component;
					int destinationIndex = token * config.HiddenSize + feature;
					if ( BitConverter.SingleToInt32Bits( contextHeads.Data[sourceIndex] ) ==
						BitConverter.SingleToInt32Bits( actual.Data[destinationIndex] ) )
					{
						actualExactMappings++;
					}
					else if ( firstActualMismatch < 0 )
					{
						firstActualMismatch = destinationIndex;
					}

					if ( BitConverter.SingleToInt32Bits( expectedContext[sourceIndex] ) ==
						BitConverter.SingleToInt32Bits( expected[destinationIndex] ) )
					{
						referenceExactMappings++;
					}
					else if ( firstReferenceMismatch < 0 )
					{
						firstReferenceMismatch = destinationIndex;
					}
				}
			}
		}

		bool structuralPassed = actualExactMappings == stage.MappingElements &&
			referenceExactMappings == stage.MappingElements &&
			firstActualMismatch < 0 && firstReferenceMismatch < 0;
		LlmLog.Info(
			"ATTN",
			$"layer=0 stage=MERGE_HEADS0_STRUCTURE values_checked={stage.MappingElements} " +
			$"csharp_exact={actualExactMappings} reference_exact={referenceExactMappings} " +
			$"first_csharp_mismatch={firstActualMismatch} " +
			$"first_reference_mismatch={firstReferenceMismatch} " +
			$"{(structuralPassed ? "PASS" : "FAIL")}" );
		if ( !structuralPassed )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 head-merge structural parity failed: " +
				$"C# exact={actualExactMappings}/{stage.MappingElements}, " +
				$"reference exact={referenceExactMappings}/{stage.MappingElements}, " +
				$"first C# mismatch={firstActualMismatch}, " +
				$"first reference mismatch={firstReferenceMismatch}." );
		}

		NumericComparison comparison = TensorDiagnostics.Compare(
			expected,
			actual.Data,
			absoluteTolerance: 1e-6,
			relativeTolerance: 1e-5 );
		int worstFeature = comparison.MaximumErrorIndex % config.HiddenSize;
		int worstToken = comparison.MaximumErrorIndex / config.HiddenSize;
		LlmLog.Info(
			"PARITY",
			$"layer0.attention_merged {comparison} " +
			$"worst=[{worstToken},{worstFeature}] zero_new_merge_error=true" );
		LlmLog.Info(
			"ATTN",
			"layer=0 merge_target " +
			TinyStoriesForwardStages.DescribeHeadMergeMapping(
				contextHeads,
				actual,
				2,
				29 ) );
		LlmLog.Info(
			"PERF",
			$"stage=layer0.attention_merge shape={actual.ShapeText} " +
			$"elapsed_ms={elapsedMilliseconds:N4}" );
		if ( !comparison.Passed )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 merged-attention parity failed: {comparison}, " +
				$"worst=[{worstToken},{worstFeature}]." );
		}

		return actual;
	}

	private static Tensor ValidateLayer0AttentionOutputProjectionParity(
		SboxLlmModel model,
		Tensor merged,
		TinyStoriesConfig config,
		ForwardReferenceDocument document )
	{
		ForwardReferenceStage stage = document.GetRequiredStage(
			"layer0_attention_out_proj" );
		ValidateAttentionOutputProjectionReferenceStage(
			stage,
			document.SequenceLength,
			config.HiddenSize );
		merged.RequireShape( document.SequenceLength, config.HiddenSize );

		Tensor weight = model.GetRequiredTensor(
			TinyStoriesForwardStages.Layer0AttentionOutProjectionWeightName );
		Tensor bias = model.GetRequiredTensor(
			TinyStoriesForwardStages.Layer0AttentionOutProjectionBiasName );
		weight.RequireShape( config.HiddenSize, config.HiddenSize );
		bias.RequireShape( config.HiddenSize );
		float[] expected = ReferenceFloatData.LoadFromMounted(
			LlmPaths.ReferenceLayer0AttentionOutProj,
			stage.Elements );
		TensorSummary expectedSummary = TensorDiagnostics.Summarize(
			"python.layer0.attention_out_proj",
			$"[{document.SequenceLength},{config.HiddenSize}]",
			expected );
		if ( !expectedSummary.IsFinite )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] Python layer 0 attention output-projection reference " +
				"contains NaN or Infinity." );
		}

		float[] mergedBefore = new float[merged.Data.Length];
		for ( int index = 0; index < merged.Data.Length; index++ )
		{
			mergedBefore[index] = merged.Data[index];
		}
		LlmLog.Info(
			"ATTN",
			$"layer=0 stage=ATTENTION_OUT_PROJ input={merged.ShapeText} " +
			$"weight={weight.Name}{weight.ShapeText} bias={bias.Name}{bias.ShapeText} " +
			$"output_expected=[{document.SequenceLength},{config.HiddenSize}] " +
			"weight_layout=[out,input] formula=sum_i(input[token,i]*weight[out,i])+" +
			"bias[out] dtype=float32" );
		FastTimer timer = FastTimer.StartNew();
		Tensor actual = TinyStoriesForwardStages.LinearWithBias(
			merged,
			weight,
			bias,
			"forward.layer0.attention_out_proj" );
		double elapsedMilliseconds = timer.ElapsedMilliSeconds;
		actual.RequireShape( document.SequenceLength, config.HiddenSize );
		TensorSummary actualSummary = TensorDiagnostics.Summarize( actual );
		LlmLog.Info( "ATTN", $"layer=0 stage=ATTENTION_OUT_PROJ0 {actualSummary}" );
		if ( !actualSummary.IsFinite )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] C# layer 0 attention output projection contains " +
				"NaN or Infinity." );
		}

		int inputMutations = 0;
		for ( int index = 0; index < merged.Data.Length; index++ )
		{
			if ( BitConverter.SingleToInt32Bits( mergedBefore[index] ) !=
				BitConverter.SingleToInt32Bits( merged.Data[index] ) )
			{
				inputMutations++;
			}
		}
		if ( inputMutations != 0 )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 attention output projection mutated " +
				$"{inputMutations} merged-context values." );
		}

		NumericComparison comparison = TensorDiagnostics.Compare(
			expected,
			actual.Data,
			absoluteTolerance: 1e-5,
			relativeTolerance: 1e-5 );
		int worstFeature = comparison.MaximumErrorIndex % config.HiddenSize;
		int worstToken = comparison.MaximumErrorIndex / config.HiddenSize;
		LlmLog.Info(
			"PARITY",
			$"layer0.attention_out_proj {comparison} " +
			$"worst=[{worstToken},{worstFeature}] input_mutations={inputMutations}" );
		LlmLog.Info(
			"ATTN",
			"layer=0 out_proj_target " +
			TinyStoriesForwardStages.DescribeLinearDotProduct(
				merged,
				weight,
				0,
				0,
				expected[0],
				actual.Data[0],
				bias ) );
		LlmLog.Info(
			"ATTN",
			$"layer=0 stage=ATTENTION_OUT_PROJ0_POST projection_output=" +
			$"{actual.ShapeText} residual_dropout_p={stage.ResidualDropoutProbability:G9} " +
			$"model_eval={stage.ModelEval} dropout_applied={stage.DropoutApplied} " +
			$"dropout_changed_bits={stage.DropoutChangedBits} " +
			$"additional_cast={stage.AdditionalCast} branch_bit_equal=" +
			$"{stage.AttentionBranchAfterDropoutBitEqual} PASS" );
		LlmLog.Info(
			"PERF",
			$"stage=layer0.attention_out_projection shape={actual.ShapeText} " +
			$"elapsed_ms={elapsedMilliseconds:N4}" );
		if ( !comparison.Passed )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 attention output-projection parity failed: " +
				$"{comparison}, worst=[{worstToken},{worstFeature}]." );
		}

		return actual;
	}

	private static Tensor ValidateLayer0AttentionResidualParity(
		Tensor attentionBranch,
		Tensor residualSource,
		Tensor layer0Ln1,
		TinyStoriesConfig config,
		ForwardReferenceDocument document )
	{
		ForwardReferenceStage stage = document.GetRequiredStage(
			"layer0_attention_residual" );
		ValidateAttentionResidualReferenceStage(
			stage,
			document.SequenceLength,
			config.HiddenSize );
		attentionBranch.RequireShape( document.SequenceLength, config.HiddenSize );
		residualSource.RequireShape( document.SequenceLength, config.HiddenSize );
		layer0Ln1.RequireShape( document.SequenceLength, config.HiddenSize );

		float[] expectedResidualSource = ReferenceFloatData.LoadFromMounted(
			LlmPaths.ReferenceEmbedding,
			document.SequenceLength * config.HiddenSize );
		NumericComparison sourceComparison = TensorDiagnostics.Compare(
			expectedResidualSource,
			residualSource.Data,
			absoluteTolerance: 0,
			relativeTolerance: 0 );
		int sourceLn1BitMatches = 0;
		for ( int index = 0; index < residualSource.Data.Length; index++ )
		{
			if ( BitConverter.SingleToInt32Bits( residualSource.Data[index] ) ==
				BitConverter.SingleToInt32Bits( layer0Ln1.Data[index] ) )
			{
				sourceLn1BitMatches++;
			}
		}
		bool sourcePassed = residualSource.Name == "forward.combined_embedding" &&
			sourceComparison.Passed && sourceLn1BitMatches < residualSource.Data.Length &&
			stage.ResidualSourceBitExactToBlockInput;
		TensorSummary sourceSummary = TensorDiagnostics.Summarize( residualSource );
		LlmLog.Info(
			"ATTN",
			$"layer=0 stage=ATTENTION_RESIDUAL0_SOURCE name={residualSource.Name} " +
			$"shape={residualSource.ShapeText} checksum={sourceSummary.Checksum} " +
			$"python_combined_embedding={sourceComparison} ln1_bit_matches=" +
			$"{sourceLn1BitMatches}/{residualSource.Data.Length} metadata_wrong_ln1_max_abs=" +
			$"{stage.WrongLn1ResidualMaximumAbsoluteError:G12} " +
			$"{(sourcePassed ? "PASS" : "FAIL")}" );
		if ( !sourcePassed )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 attention residual source must be the bit-exact " +
				$"combined embedding/block input before ln_1; found name={residualSource.Name}, " +
				$"source parity={sourceComparison}, ln1 bit matches=" +
				$"{sourceLn1BitMatches}/{residualSource.Data.Length}." );
		}

		float[] expected = ReferenceFloatData.LoadFromMounted(
			LlmPaths.ReferenceLayer0AttentionResidual,
			stage.Elements );
		TensorSummary expectedSummary = TensorDiagnostics.Summarize(
			"python.layer0.attention_residual",
			$"[{document.SequenceLength},{config.HiddenSize}]",
			expected );
		if ( !expectedSummary.IsFinite )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] Python layer 0 attention-residual reference contains " +
				"NaN or Infinity." );
		}

		float[] branchBefore = new float[attentionBranch.Data.Length];
		float[] sourceBefore = new float[residualSource.Data.Length];
		for ( int index = 0; index < attentionBranch.Data.Length; index++ )
		{
			branchBefore[index] = attentionBranch.Data[index];
			sourceBefore[index] = residualSource.Data[index];
		}
		LlmLog.Info(
			"ATTN",
			$"layer=0 stage=ATTENTION_RESIDUAL input_attention=" +
			$"{attentionBranch.ShapeText} residual_source={residualSource.ShapeText} " +
			$"residual_boundary='{stage.ResidualSourceBoundary}' output_expected=" +
			$"[{document.SequenceLength},{config.HiddenSize}] " +
			"formula=attention_output+residual dtype=float32 ln_2=false" );
		FastTimer timer = FastTimer.StartNew();
		Tensor actual = TinyStoriesForwardStages.AddResidual(
			attentionBranch,
			residualSource,
			"forward.layer0.attention_residual" );
		double elapsedMilliseconds = timer.ElapsedMilliSeconds;
		actual.RequireShape( document.SequenceLength, config.HiddenSize );
		TensorSummary actualSummary = TensorDiagnostics.Summarize( actual );
		LlmLog.Info( "ATTN", $"layer=0 stage=ATTENTION_RESIDUAL0 {actualSummary}" );
		if ( !actualSummary.IsFinite )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] C# layer 0 attention residual contains NaN or Infinity." );
		}

		int branchMutations = 0;
		int sourceMutations = 0;
		for ( int index = 0; index < actual.Data.Length; index++ )
		{
			if ( BitConverter.SingleToInt32Bits( branchBefore[index] ) !=
				BitConverter.SingleToInt32Bits( attentionBranch.Data[index] ) )
			{
				branchMutations++;
			}
			if ( BitConverter.SingleToInt32Bits( sourceBefore[index] ) !=
				BitConverter.SingleToInt32Bits( residualSource.Data[index] ) )
			{
				sourceMutations++;
			}
		}
		if ( branchMutations != 0 || sourceMutations != 0 )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 residual addition mutated inputs: attention=" +
				$"{branchMutations}, residual={sourceMutations}." );
		}

		NumericComparison comparison = TensorDiagnostics.Compare(
			expected,
			actual.Data,
			absoluteTolerance: 1e-5,
			relativeTolerance: 1e-5 );
		int worstFeature = comparison.MaximumErrorIndex % config.HiddenSize;
		int worstToken = comparison.MaximumErrorIndex / config.HiddenSize;
		LlmLog.Info(
			"PARITY",
			$"layer0.attention_residual {comparison} " +
			$"worst=[{worstToken},{worstFeature}] attention_mutations=" +
			$"{branchMutations} residual_mutations={sourceMutations}" );
		int targetToken = 2;
		int targetFeature = 29;
		int targetIndex = targetToken * config.HiddenSize + targetFeature;
		LlmLog.Info(
			"ATTN",
			"layer=0 residual_target " +
			TinyStoriesForwardStages.DescribeResidualAddition(
				attentionBranch,
				residualSource,
				actual,
				targetToken,
				targetFeature,
				expected[targetIndex] ) );
		LlmLog.Info(
			"PERF",
			$"stage=layer0.attention_residual shape={actual.ShapeText} " +
			$"elapsed_ms={elapsedMilliseconds:N4}" );
		if ( !comparison.Passed )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 attention-residual parity failed: " +
				$"{comparison}, worst=[{worstToken},{worstFeature}]." );
		}

		return actual;
	}

	private static Tensor ValidateLayer0Ln2Parity(
		SboxLlmModel model,
		Tensor attentionResidual,
		TinyStoriesConfig config,
		ForwardReferenceDocument document )
	{
		ForwardReferenceStage stage = document.GetRequiredStage( "layer0_ln_2" );
		ValidateLayer0Ln2ReferenceStage(
			stage,
			document.SequenceLength,
			config.HiddenSize,
			config.LayerNormEpsilon );
		attentionResidual.RequireShape( document.SequenceLength, config.HiddenSize );

		float[] expected = ReferenceFloatData.LoadFromMounted(
			LlmPaths.ReferenceLayer0Ln2,
			stage.Elements );
		TensorSummary expectedSummary = TensorDiagnostics.Summarize(
			"python.layer0.ln_2",
			$"[{document.SequenceLength},{config.HiddenSize}]",
			expected );
		if ( !expectedSummary.IsFinite )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] Python layer 0 ln_2 reference contains NaN or Infinity." );
		}

		float[] inputBefore = CopyValues( attentionResidual.Data );
		FastTimer timer = FastTimer.StartNew();
		Tensor actual = TinyStoriesForwardStages.ApplyLayer0Ln2(
			model,
			config,
			attentionResidual );
		double elapsedMilliseconds = timer.ElapsedMilliSeconds;
		actual.RequireShape( document.SequenceLength, config.HiddenSize );
		TensorSummary actualSummary = TensorDiagnostics.Summarize( actual );
		int inputMutations = CountBitMismatches( inputBefore, attentionResidual.Data );
		LlmLog.Info(
			"LN",
			$"layer=0 ln=2 stage=LN0_2 {actualSummary} input_mutations={inputMutations}" );
		if ( !actualSummary.IsFinite || inputMutations != 0 )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 ln_2 invariant failed: finite=" +
				$"{actualSummary.FiniteCount}/{actualSummary.ElementCount}, " +
				$"input_mutations={inputMutations}." );
		}

		NumericComparison comparison = TensorDiagnostics.Compare(
			expected,
			actual.Data,
			absoluteTolerance: 1e-5,
			relativeTolerance: 1e-5 );
		int worstFeature = comparison.MaximumErrorIndex % config.HiddenSize;
		int worstToken = comparison.MaximumErrorIndex / config.HiddenSize;
		LlmLog.Info(
			"PARITY",
			$"layer0.ln_2 {comparison} worst=[{worstToken},{worstFeature}]" );
		LlmLog.Trace(
			"LN",
			$"layer=0 ln=2 selected=[0,0] input={attentionResidual.Data[0]:G9} " +
			$"actual={actual.Data[0]:G9} python={expected[0]:G9} " +
			$"abs_diff={MathF.Abs( actual.Data[0] - expected[0] ):G12}" );
		LlmLog.Info(
			"PERF",
			$"stage=layer0.ln_2 shape={actual.ShapeText} " +
			$"elapsed_ms={elapsedMilliseconds:N4}" );
		if ( !comparison.Passed )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 ln_2 parity failed: {comparison}, " +
				$"worst=[{worstToken},{worstFeature}]." );
		}
		return actual;
	}

	private static Tensor ValidateLayer0MlpFcParity(
		SboxLlmModel model,
		Tensor layer0Ln2,
		TinyStoriesConfig config,
		ForwardReferenceDocument document )
	{
		int intermediateSize = config.HiddenSize * 4;
		ForwardReferenceStage stage = document.GetRequiredStage( "layer0_mlp_fc" );
		ValidateMlpLinearReferenceStage(
			stage,
			"reference_layer0_mlp_fc.f32",
			"layer0_ln_2",
			TinyStoriesForwardStages.Layer0MlpFcWeightName,
			TinyStoriesForwardStages.Layer0MlpFcBiasName,
			document.SequenceLength,
			config.HiddenSize,
			intermediateSize,
			preActivation: true,
			preDropout: false );
		layer0Ln2.RequireShape( document.SequenceLength, config.HiddenSize );
		Tensor weight = model.GetRequiredTensor( TinyStoriesForwardStages.Layer0MlpFcWeightName );
		Tensor bias = model.GetRequiredTensor( TinyStoriesForwardStages.Layer0MlpFcBiasName );
		weight.RequireShape( intermediateSize, config.HiddenSize );
		bias.RequireShape( intermediateSize );

		float[] expected = ReferenceFloatData.LoadFromMounted(
			LlmPaths.ReferenceLayer0MlpFc,
			stage.Elements );
		TensorSummary expectedSummary = TensorDiagnostics.Summarize(
			"python.layer0.mlp.c_fc",
			$"[{document.SequenceLength},{intermediateSize}]",
			expected );
		if ( !expectedSummary.IsFinite )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] Python layer 0 MLP c_fc reference contains NaN or Infinity." );
		}

		float[] inputBefore = CopyValues( layer0Ln2.Data );
		LlmLog.Info(
			"MLP",
			$"layer=0 stage=MLP_FC input={layer0Ln2.ShapeText} " +
			$"weight={weight.Name}{weight.ShapeText} bias={bias.Name}{bias.ShapeText} " +
			$"output_expected=[{document.SequenceLength},{intermediateSize}] " +
			"weight_layout=[out,input] pre_activation=true" );
		FastTimer timer = FastTimer.StartNew();
		Tensor actual = TinyStoriesForwardStages.LinearWithBias(
			layer0Ln2,
			weight,
			bias,
			"forward.layer0.mlp.c_fc" );
		double elapsedMilliseconds = timer.ElapsedMilliSeconds;
		actual.RequireShape( document.SequenceLength, intermediateSize );
		TensorSummary actualSummary = TensorDiagnostics.Summarize( actual );
		int inputMutations = CountBitMismatches( inputBefore, layer0Ln2.Data );
		LlmLog.Info(
			"MLP",
			$"layer=0 stage=MLP_FC0 {actualSummary} input_mutations={inputMutations}" );
		if ( !actualSummary.IsFinite || inputMutations != 0 )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 MLP c_fc invariant failed: finite=" +
				$"{actualSummary.FiniteCount}/{actualSummary.ElementCount}, " +
				$"input_mutations={inputMutations}." );
		}

		NumericComparison comparison = TensorDiagnostics.Compare(
			expected,
			actual.Data,
			absoluteTolerance: 1e-5,
			relativeTolerance: 1e-5 );
		int worstFeature = comparison.MaximumErrorIndex % intermediateSize;
		int worstToken = comparison.MaximumErrorIndex / intermediateSize;
		LlmLog.Info(
			"PARITY",
			$"layer0.mlp_fc {comparison} worst=[{worstToken},{worstFeature}]" );
		string targeted = TinyStoriesForwardStages.DescribeLinearDotProduct(
			layer0Ln2,
			weight,
			0,
			0,
			expected[0],
			actual.Data[0],
			bias );
		LlmLog.Info( "MLP", $"layer=0 c_fc_target {targeted}" );
		LlmLog.Info(
			"PERF",
			$"stage=layer0.mlp_fc shape={actual.ShapeText} elapsed_ms={elapsedMilliseconds:N4}" );
		if ( !comparison.Passed )
		{
			LlmLog.Warning(
				"MLP",
				"layer=0 c_fc_worst " +
				TinyStoriesForwardStages.DescribeLinearDotProduct(
					layer0Ln2,
					weight,
					worstToken,
					worstFeature,
					comparison.ExpectedAtMaximumError,
					comparison.ActualAtMaximumError,
					bias ) );
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 MLP c_fc parity failed: {comparison}, " +
				$"worst=[{worstToken},{worstFeature}]." );
		}
		return actual;
	}

	private static Tensor ValidateLayer0MlpGeluParity(
		Tensor mlpFc,
		TinyStoriesConfig config,
		ForwardReferenceDocument document )
	{
		int intermediateSize = config.HiddenSize * 4;
		ForwardReferenceStage stage = document.GetRequiredStage( "layer0_mlp_gelu" );
		ValidateGeluNewReferenceStage(
			stage,
			document.SequenceLength,
			intermediateSize );
		mlpFc.RequireShape( document.SequenceLength, intermediateSize );
		float[] expected = ReferenceFloatData.LoadFromMounted(
			LlmPaths.ReferenceLayer0MlpGelu,
			stage.Elements );
		TensorSummary expectedSummary = TensorDiagnostics.Summarize(
			"python.layer0.mlp.gelu_new",
			$"[{document.SequenceLength},{intermediateSize}]",
			expected );
		if ( !expectedSummary.IsFinite )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] Python layer 0 GELU-new reference contains NaN or Infinity." );
		}

		float[] inputBefore = CopyValues( mlpFc.Data );
		LlmLog.Info(
			"MLP",
			$"layer=0 stage=GELU_NEW input={mlpFc.ShapeText} " +
			$"formula='{TinyStoriesForwardStages.GeluNewFormula}' " +
			$"tanh_coefficient={TinyStoriesForwardStages.GeluNewTanhCoefficient:G9} " +
			$"cubic_coefficient={TinyStoriesForwardStages.GeluNewCubicCoefficient:G9} " +
			"pow_exponent=3 operation_dtype=float32" );
		FastTimer timer = FastTimer.StartNew();
		Tensor actual = TinyStoriesForwardStages.ApplyGeluNew(
			mlpFc,
			"forward.layer0.mlp.gelu_new" );
		double elapsedMilliseconds = timer.ElapsedMilliSeconds;
		actual.RequireShape( document.SequenceLength, intermediateSize );
		TensorSummary actualSummary = TensorDiagnostics.Summarize( actual );
		int inputMutations = CountBitMismatches( inputBefore, mlpFc.Data );
		LlmLog.Info(
			"MLP",
			$"layer=0 stage=GELU_NEW0 {actualSummary} input_mutations={inputMutations}" );
		if ( !actualSummary.IsFinite || inputMutations != 0 )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 GELU-new invariant failed: finite=" +
				$"{actualSummary.FiniteCount}/{actualSummary.ElementCount}, " +
				$"input_mutations={inputMutations}." );
		}

		Tensor sanityInput = new(
			"validation.gelu_new.sanity_input",
			new[] { 1, 3 },
			new[] { -3.0f, 0.0f, 3.0f } );
		Tensor sanityOutput = TinyStoriesForwardStages.ApplyGeluNew(
			sanityInput,
			"validation.gelu_new.sanity_output" );
		bool sanityPassed = sanityOutput.Data[0] < 0 &&
			BitConverter.SingleToInt32Bits( sanityOutput.Data[1] ) == 0 &&
			sanityOutput.Data[2] > 0 && MathF.Abs( sanityOutput.Data[2] - 3.0f ) < 0.01f;
		LlmLog.Info(
			"MLP",
			$"layer=0 stage=GELU_NEW0_SANITY input=[-3,0,3] output=" +
			$"[{sanityOutput.Data[0]:G9},{sanityOutput.Data[1]:G9}," +
			$"{sanityOutput.Data[2]:G9}] zero_bits=0x" +
			$"{BitConverter.SingleToInt32Bits( sanityOutput.Data[1] ):X8} " +
			$"{(sanityPassed ? "PASS" : "FAIL")}" );
		if ( !sanityPassed )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] Layer 0 GELU-new scalar sanity checks failed." );
		}

		NumericComparison comparison = TensorDiagnostics.Compare(
			expected,
			actual.Data,
			absoluteTolerance: 1e-6,
			relativeTolerance: 1e-5 );
		int worstFeature = comparison.MaximumErrorIndex % intermediateSize;
		int worstToken = comparison.MaximumErrorIndex / intermediateSize;
		LlmLog.Info(
			"PARITY",
			$"layer0.mlp_gelu_new {comparison} worst=[{worstToken},{worstFeature}] " +
			$"worst_input={mlpFc.Data[comparison.MaximumErrorIndex]:G9}" );
		LlmLog.Info(
			"MLP",
			"layer=0 gelu_target " +
			TinyStoriesForwardStages.DescribeGeluNewElement(
				mlpFc,
				actual,
				worstToken,
				worstFeature,
				comparison.ExpectedAtMaximumError ) );
		LlmLog.Info(
			"PERF",
			$"stage=layer0.mlp_gelu_new shape={actual.ShapeText} " +
			$"elapsed_ms={elapsedMilliseconds:N4}" );
		if ( !comparison.Passed )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 GELU-new parity failed: {comparison}, " +
				$"worst=[{worstToken},{worstFeature}], " +
				$"input={mlpFc.Data[comparison.MaximumErrorIndex]:G9}." );
		}
		return actual;
	}

	private static Tensor ValidateLayer0MlpProjectionParity(
		SboxLlmModel model,
		Tensor mlpGelu,
		TinyStoriesConfig config,
		ForwardReferenceDocument document )
	{
		int intermediateSize = config.HiddenSize * 4;
		ForwardReferenceStage stage = document.GetRequiredStage( "layer0_mlp_proj" );
		ValidateMlpLinearReferenceStage(
			stage,
			"reference_layer0_mlp_proj.f32",
			"layer0_mlp_gelu",
			TinyStoriesForwardStages.Layer0MlpProjWeightName,
			TinyStoriesForwardStages.Layer0MlpProjBiasName,
			document.SequenceLength,
			intermediateSize,
			config.HiddenSize,
			preActivation: false,
			preDropout: true );
		mlpGelu.RequireShape( document.SequenceLength, intermediateSize );
		Tensor weight = model.GetRequiredTensor( TinyStoriesForwardStages.Layer0MlpProjWeightName );
		Tensor bias = model.GetRequiredTensor( TinyStoriesForwardStages.Layer0MlpProjBiasName );
		weight.RequireShape( config.HiddenSize, intermediateSize );
		bias.RequireShape( config.HiddenSize );

		float[] expected = ReferenceFloatData.LoadFromMounted(
			LlmPaths.ReferenceLayer0MlpProj,
			stage.Elements );
		TensorSummary expectedSummary = TensorDiagnostics.Summarize(
			"python.layer0.mlp.c_proj",
			$"[{document.SequenceLength},{config.HiddenSize}]",
			expected );
		if ( !expectedSummary.IsFinite )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] Python layer 0 MLP c_proj reference contains NaN or Infinity." );
		}

		float[] inputBefore = CopyValues( mlpGelu.Data );
		LlmLog.Info(
			"MLP",
			$"layer=0 stage=MLP_PROJ input={mlpGelu.ShapeText} " +
			$"weight={weight.Name}{weight.ShapeText} bias={bias.Name}{bias.ShapeText} " +
			$"output_expected=[{document.SequenceLength},{config.HiddenSize}] " +
			"weight_layout=[out,input] pre_dropout=true" );
		FastTimer timer = FastTimer.StartNew();
		Tensor actual = TinyStoriesForwardStages.LinearWithBias(
			mlpGelu,
			weight,
			bias,
			"forward.layer0.mlp.c_proj" );
		double elapsedMilliseconds = timer.ElapsedMilliSeconds;
		actual.RequireShape( document.SequenceLength, config.HiddenSize );
		TensorSummary actualSummary = TensorDiagnostics.Summarize( actual );
		int inputMutations = CountBitMismatches( inputBefore, mlpGelu.Data );
		LlmLog.Info(
			"MLP",
			$"layer=0 stage=MLP_PROJ0 {actualSummary} input_mutations={inputMutations}" );
		if ( !actualSummary.IsFinite || inputMutations != 0 )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 MLP c_proj invariant failed: finite=" +
				$"{actualSummary.FiniteCount}/{actualSummary.ElementCount}, " +
				$"input_mutations={inputMutations}." );
		}

		NumericComparison comparison = TensorDiagnostics.Compare(
			expected,
			actual.Data,
			absoluteTolerance: 1e-5,
			relativeTolerance: 1e-5 );
		int worstFeature = comparison.MaximumErrorIndex % config.HiddenSize;
		int worstToken = comparison.MaximumErrorIndex / config.HiddenSize;
		LlmLog.Info(
			"PARITY",
			$"layer0.mlp_proj {comparison} worst=[{worstToken},{worstFeature}]" );
		LlmLog.Info(
			"MLP",
			"layer=0 c_proj_target " +
			TinyStoriesForwardStages.DescribeLinearDotProduct(
				mlpGelu,
				weight,
				0,
				0,
				expected[0],
				actual.Data[0],
				bias ) );
		LlmLog.Info(
			"MLP",
			$"layer=0 stage=MLP_PROJ0_POST projection_output={actual.ShapeText} " +
			$"mlp_dropout_p={stage.MlpDropoutProbability:G9} model_eval={stage.ModelEval} " +
			$"dropout_applied={stage.DropoutApplied} " +
			$"dropout_changed_bits={stage.DropoutChangedBits} " +
			$"branch_bit_equal={stage.MlpBranchAfterDropoutBitEqual} PASS" );
		LlmLog.Info(
			"PERF",
			$"stage=layer0.mlp_projection shape={actual.ShapeText} " +
			$"elapsed_ms={elapsedMilliseconds:N4}" );
		if ( !comparison.Passed )
		{
			LlmLog.Warning(
				"MLP",
				"layer=0 c_proj_worst " +
				TinyStoriesForwardStages.DescribeLinearDotProduct(
					mlpGelu,
					weight,
					worstToken,
					worstFeature,
					comparison.ExpectedAtMaximumError,
					comparison.ActualAtMaximumError,
					bias ) );
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 MLP c_proj parity failed: {comparison}, " +
				$"worst=[{worstToken},{worstFeature}]." );
		}
		return actual;
	}

	private static Tensor ValidateLayer0OutputParity(
		Tensor mlpBranch,
		Tensor residualSource,
		Tensor layer0Ln2,
		TinyStoriesConfig config,
		ForwardReferenceDocument document )
	{
		ForwardReferenceStage stage = document.GetRequiredStage( "layer0_output" );
		ValidateLayer0OutputReferenceStage(
			stage,
			document.SequenceLength,
			config.HiddenSize );
		mlpBranch.RequireShape( document.SequenceLength, config.HiddenSize );
		residualSource.RequireShape( document.SequenceLength, config.HiddenSize );
		layer0Ln2.RequireShape( document.SequenceLength, config.HiddenSize );

		float[] expectedResidual = ReferenceFloatData.LoadFromMounted(
			LlmPaths.ReferenceLayer0AttentionResidual,
			document.SequenceLength * config.HiddenSize );
		NumericComparison sourceComparison = TensorDiagnostics.Compare(
			expectedResidual,
			residualSource.Data,
			absoluteTolerance: 1e-5,
			relativeTolerance: 1e-5 );
		float[] expected = ReferenceFloatData.LoadFromMounted(
			LlmPaths.ReferenceLayer0Output,
			stage.Elements );
		TensorSummary expectedSummary = TensorDiagnostics.Summarize(
			"python.layer0.output",
			$"[{document.SequenceLength},{config.HiddenSize}]",
			expected );
		if ( !expectedSummary.IsFinite )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] Python complete layer 0 output contains NaN or Infinity." );
		}

		float wrongLn2MaximumAbsoluteError = 0;
		int ln2BitMatches = 0;
		for ( int index = 0; index < expected.Length; index++ )
		{
			float wrong = layer0Ln2.Data[index] + mlpBranch.Data[index];
			wrongLn2MaximumAbsoluteError = MathF.Max(
				wrongLn2MaximumAbsoluteError,
				MathF.Abs( expected[index] - wrong ) );
			if ( BitConverter.SingleToInt32Bits( residualSource.Data[index] ) ==
				BitConverter.SingleToInt32Bits( layer0Ln2.Data[index] ) )
			{
				ln2BitMatches++;
			}
		}
		bool sourcePassed = residualSource.Name == "forward.layer0.attention_residual" &&
			sourceComparison.Passed && ln2BitMatches < residualSource.Data.Length &&
			stage.ResidualSourceBitExactToAttentionResidual &&
			stage.WrongLn2ResidualMaximumAbsoluteError > 1.0e-3f &&
			wrongLn2MaximumAbsoluteError > 1.0e-3f;
		TensorSummary sourceSummary = TensorDiagnostics.Summarize( residualSource );
		LlmLog.Info(
			"MLP",
			$"layer=0 stage=MLP_RESIDUAL0_SOURCE name={residualSource.Name} " +
			$"shape={residualSource.ShapeText} checksum={sourceSummary.Checksum} " +
			$"python_attention_residual={sourceComparison} ln2_bit_matches=" +
			$"{ln2BitMatches}/{residualSource.Data.Length} " +
			$"wrong_ln2_residual_max_abs={wrongLn2MaximumAbsoluteError:G12} " +
			$"metadata_wrong_ln2_max_abs={stage.WrongLn2ResidualMaximumAbsoluteError:G12} " +
			$"{(sourcePassed ? "PASS" : "FAIL")}" );
		if ( !sourcePassed )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Layer 0 MLP residual source must be the validated " +
				$"attention residual before ln_2; found name={residualSource.Name}, " +
				$"source parity={sourceComparison}, ln2_bit_matches={ln2BitMatches}/" +
				$"{residualSource.Data.Length}, wrong_ln2_max_abs=" +
				$"{wrongLn2MaximumAbsoluteError:G12}." );
		}

		float[] branchBefore = CopyValues( mlpBranch.Data );
		float[] sourceBefore = CopyValues( residualSource.Data );
		LlmLog.Info(
			"MLP",
			$"layer=0 stage=MLP_RESIDUAL input_mlp={mlpBranch.ShapeText} " +
			$"residual_source={residualSource.ShapeText} residual_boundary=" +
			$"'{stage.ResidualSourceBoundary}' output_expected=" +
			$"[{document.SequenceLength},{config.HiddenSize}] " +
			"formula=residual+feed_forward_hidden_states dtype=float32 layer_complete=true" );
		FastTimer timer = FastTimer.StartNew();
		Tensor actual = TinyStoriesForwardStages.AddMlpResidual(
			residualSource,
			mlpBranch,
			"forward.layer0.output" );
		double elapsedMilliseconds = timer.ElapsedMilliSeconds;
		actual.RequireShape( document.SequenceLength, config.HiddenSize );
		TensorSummary actualSummary = TensorDiagnostics.Summarize( actual );
		int branchMutations = CountBitMismatches( branchBefore, mlpBranch.Data );
		int sourceMutations = CountBitMismatches( sourceBefore, residualSource.Data );
		LlmLog.Info(
			"LAYER",
			$"layer=0 stage=LAYER0_OUTPUT {actualSummary} mlp_mutations={branchMutations} " +
			$"residual_mutations={sourceMutations}" );
		if ( !actualSummary.IsFinite || branchMutations != 0 || sourceMutations != 0 )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Complete layer 0 output invariant failed: finite=" +
				$"{actualSummary.FiniteCount}/{actualSummary.ElementCount}, " +
				$"MLP_mutations={branchMutations}, residual_mutations={sourceMutations}." );
		}

		NumericComparison comparison = TensorDiagnostics.Compare(
			expected,
			actual.Data,
			absoluteTolerance: 1e-5,
			relativeTolerance: 1e-5 );
		int worstFeature = comparison.MaximumErrorIndex % config.HiddenSize;
		int worstToken = comparison.MaximumErrorIndex / config.HiddenSize;
		LlmLog.Info(
			"PARITY",
			$"layer0.output {comparison} worst=[{worstToken},{worstFeature}] " +
			$"mlp_mutations={branchMutations} residual_mutations={sourceMutations}" );
		int targetToken = 2;
		int targetFeature = 29;
		int targetIndex = targetToken * config.HiddenSize + targetFeature;
		LlmLog.Info(
			"MLP",
			"layer=0 final_residual_target " +
			TinyStoriesForwardStages.DescribeMlpResidualAddition(
				residualSource,
				mlpBranch,
				actual,
				targetToken,
				targetFeature,
				expected[targetIndex] ) );
		LlmLog.Info(
			"PERF",
			$"stage=layer0.final_residual shape={actual.ShapeText} " +
			$"elapsed_ms={elapsedMilliseconds:N4}" );
		if ( !comparison.Passed )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Complete layer 0 output parity failed: {comparison}, " +
				$"worst=[{worstToken},{worstFeature}]." );
		}
		return actual;
	}

	private static void ValidateForwardReferenceDocument(
		ForwardReferenceDocument document,
		TinyStoriesConfig config,
		LlmReferenceData reference )
	{
		if ( document.Format != "SBOXLLM_REFERENCE_INTERMEDIATES" || document.Version != 1 )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Intermediate reference format/version expected " +
				$"SBOXLLM_REFERENCE_INTERMEDIATES/1, found {document.Format}/{document.Version}." );
		}
		if ( document.Model != "roneneldan/TinyStories-Instruct-1M" )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Intermediate reference model expected " +
				$"roneneldan/TinyStories-Instruct-1M, found '{document.Model}'." );
		}
		if ( document.InputTokenIds is null || document.Prompt != reference.Prompt ||
			FirstMismatch( document.InputTokenIds, reference.InputTokenIds ) >= 0 )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] Intermediate reference prompt/token IDs disagree with reference.json." );
		}
		if ( document.SequenceLength != reference.InputTokenIds.Length ||
			document.HiddenSize != config.HiddenSize )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Intermediate reference expected shape " +
				$"[{reference.InputTokenIds.Length},{config.HiddenSize}], found " +
				$"[{document.SequenceLength},{document.HiddenSize}]." );
		}
		if ( document.PositionIds is null || document.PositionIds.Length != document.SequenceLength )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] Intermediate reference position IDs have the wrong length." );
		}
		for ( int position = 0; position < document.PositionIds.Length; position++ )
		{
			if ( document.PositionIds[position] != position )
			{
				throw new InvalidOperationException(
					$"[LLM:ERROR] Intermediate reference position_ids[{position}] expected " +
					$"{position}, found {document.PositionIds[position]}." );
			}
		}
		if ( document.Dtype != "float32-le" || document.ModelTraining ||
			document.EmbeddingDropout != config.EmbeddingDropout ||
			document.LayerNormVariance != "biased_population_unbiased_false" ||
			Math.Abs( document.LayerNormEpsilon - config.LayerNormEpsilon ) > 1e-10f )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] Intermediate reference dtype/eval/dropout/LayerNorm metadata " +
				"does not match the validated model configuration." );
		}
	}

	private static void ValidateForwardReferenceStage(
		ForwardReferenceStage stage,
		string expectedFile,
		int sequenceLength,
		int hiddenSize )
	{
		int expectedElements = sequenceLength * hiddenSize;
		if ( stage.File != expectedFile || stage.Dtype != "float32-le" ||
			stage.Shape is null || stage.Shape.Length != 2 ||
			stage.Shape[0] != sequenceLength || stage.Shape[1] != hiddenSize ||
			stage.Elements != expectedElements || stage.Bytes != expectedElements * sizeof( float ) )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Intermediate stage '{stage.Stage}' expected file={expectedFile}, " +
				$"dtype=float32-le, shape=[{sequenceLength},{hiddenSize}], elements={expectedElements}, " +
				$"bytes={expectedElements * sizeof( float )}; found file={stage.File}, " +
				$"dtype={stage.Dtype}, shape=[{string.Join( ",", stage.Shape ?? Array.Empty<int>() )}], " +
				$"elements={stage.Elements}, bytes={stage.Bytes}." );
		}
	}

	private static void ValidateHeadReferenceStage(
		ForwardReferenceStage stage,
		string expectedFile,
		string projectionLabel,
		int sequenceLength,
		int hiddenSize,
		int headCount,
		int headDimension )
	{
		int expectedElements = headCount * sequenceLength * headDimension;
		bool projectionShapeMatches = stage.OriginalProjectionShape is not null &&
			stage.OriginalProjectionShape.Length == 2 &&
			stage.OriginalProjectionShape[0] == sequenceLength &&
			stage.OriginalProjectionShape[1] == hiddenSize;
		bool axisOrderMatches = stage.LogicalAxisOrder is not null &&
			stage.LogicalAxisOrder.Length == 3 &&
			stage.LogicalAxisOrder[0] == "head" &&
			stage.LogicalAxisOrder[1] == "sequence" &&
			stage.LogicalAxisOrder[2] == "head_dimension";
		if ( stage.File != expectedFile || stage.Dtype != "float32-le" ||
			stage.Shape is null || stage.Shape.Length != 3 ||
			stage.Shape[0] != headCount || stage.Shape[1] != sequenceLength ||
			stage.Shape[2] != headDimension || stage.Elements != expectedElements ||
			stage.Bytes != expectedElements * sizeof( float ) ||
			!projectionShapeMatches || stage.HeadCount != headCount ||
			stage.HeadDimension != headDimension || stage.SequenceLength != sequenceLength ||
			!axisOrderMatches || stage.Projection != projectionLabel ||
			stage.SourceFeatureFormula != "feature=head*head_dimension+component" ||
			stage.FlattenedStorageOrder !=
				"C-order [head,sequence,head_dimension]; " +
				"flat=((head*sequence_length)+token)*head_dimension+component" )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Intermediate head stage '{stage.Stage}' expected " +
				$"file={expectedFile}, projection={projectionLabel}, dtype=float32-le, " +
				$"shape=[{headCount},{sequenceLength},{headDimension}], " +
				$"source_shape=[{sequenceLength},{hiddenSize}], elements={expectedElements}, " +
				$"bytes={expectedElements * sizeof( float )}, axis_order=[head,sequence,head_dimension]; " +
				$"found file={stage.File}, projection={stage.Projection}, dtype={stage.Dtype}, " +
				$"shape=[{string.Join( ",", stage.Shape ?? Array.Empty<int>() )}], " +
				$"elements={stage.Elements}, bytes={stage.Bytes}." );
		}
	}

	private static void ValidateScoreReferenceStage(
		ForwardReferenceStage stage,
		string expectedFile,
		int sequenceLength,
		int headCount,
		int headDimension )
	{
		int expectedElements = headCount * sequenceLength * sequenceLength;
		bool axisOrderMatches = stage.LogicalAxisOrder is not null &&
			stage.LogicalAxisOrder.Length == 3 &&
			stage.LogicalAxisOrder[0] == "head" &&
			stage.LogicalAxisOrder[1] == "query_sequence" &&
			stage.LogicalAxisOrder[2] == "key_sequence";
		if ( stage.File != expectedFile || stage.Dtype != "float32-le" ||
			stage.Shape is null || stage.Shape.Length != 3 ||
			stage.Shape[0] != headCount || stage.Shape[1] != sequenceLength ||
			stage.Shape[2] != sequenceLength || stage.Elements != expectedElements ||
			stage.Bytes != expectedElements * sizeof( float ) ||
			stage.HeadCount != headCount || stage.HeadDimension != headDimension ||
			stage.QuerySequenceLength != sequenceLength ||
			stage.KeySequenceLength != sequenceLength || !axisOrderMatches ||
			stage.Formula != "sum_d Q[head,query,d]*K[head,key,d]" ||
			stage.Scale != 1.0f ||
			stage.ScalingOperation != "none; exact model-equivalent factor is 1.0" ||
			stage.CausalMaskApplied || stage.LocalOrGlobalMaskApplied ||
			stage.SoftmaxApplied || stage.DropoutApplied ||
			stage.FlattenedStorageOrder !=
				"C-order [head,query_sequence,key_sequence]; " +
				"flat=((head*query_sequence_length)+query)*key_sequence_length+key" )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Intermediate score stage '{stage.Stage}' expected " +
				$"file={expectedFile}, dtype=float32-le, " +
				$"shape=[{headCount},{sequenceLength},{sequenceLength}], " +
				$"head_dim={headDimension}, scale=1, mask=false, softmax=false, " +
				$"elements={expectedElements}, bytes={expectedElements * sizeof( float )}; " +
				$"found file={stage.File}, dtype={stage.Dtype}, " +
				$"shape=[{string.Join( ",", stage.Shape ?? Array.Empty<int>() )}], " +
				$"scale={stage.Scale:G9}, elements={stage.Elements}, bytes={stage.Bytes}." );
		}
	}

	private static void ValidateMaskedScoreReferenceStage(
		ForwardReferenceStage stage,
		int sequenceLength,
		int headCount,
		int headDimension,
		AttentionMaskReferenceDocument mask )
	{
		int expectedElements = headCount * sequenceLength * sequenceLength;
		bool axisOrderMatches = stage.LogicalAxisOrder is not null &&
			stage.LogicalAxisOrder.Length == 3 &&
			stage.LogicalAxisOrder[0] == "head" &&
			stage.LogicalAxisOrder[1] == "query_sequence" &&
			stage.LogicalAxisOrder[2] == "key_sequence";
		if ( stage.File != "reference_layer0_scores_masked_pre_softmax.f32" ||
			stage.Dtype != "float32-le" ||
			stage.Shape is null || stage.Shape.Length != 3 ||
			stage.Shape[0] != headCount || stage.Shape[1] != sequenceLength ||
			stage.Shape[2] != sequenceLength || stage.Elements != expectedElements ||
			stage.Bytes != expectedElements * sizeof( float ) ||
			stage.AttentionLayer != 0 || stage.AttentionType != "global" ||
			stage.HeadCount != headCount || stage.HeadDimension != headDimension ||
			stage.QuerySequenceLength != sequenceLength ||
			stage.KeySequenceLength != sequenceLength || !axisOrderMatches ||
			stage.Formula != "sum_d Q[head,query,d]*K[head,key,d]" ||
			stage.Scale != 1.0f ||
			stage.ScalingOperation != "none; exact model-equivalent factor is 1.0" ||
			stage.MaskReferenceFile != "reference_layer0_attention_mask.json" ||
			stage.MaskSemantics != "global causal; allowed iff key <= query" ||
			BitConverter.SingleToInt32Bits( stage.MaskedSentinel ) !=
				BitConverter.SingleToInt32Bits( mask.MaskedSentinel ) ||
			stage.MaskedSentinelFloat32Bits != "0xFF7FFFFF" ||
			!stage.MaskedSentinelIsFinite ||
			stage.AllowedCountPerHead != mask.AllowedCountPerHead ||
			stage.MaskedCountPerHead != mask.MaskedCountPerHead ||
			stage.AllowedCountAllHeads != mask.AllowedCountAllHeads ||
			stage.MaskedCountAllHeads != mask.MaskedCountAllHeads ||
			!stage.CausalMaskApplied || !stage.LocalOrGlobalMaskApplied ||
			!stage.PreSoftmax || stage.SoftmaxApplied || stage.DropoutApplied ||
			stage.FlattenedStorageOrder !=
				"C-order [head,query_sequence,key_sequence]; " +
				"flat=((head*query_sequence_length)+query)*key_sequence_length+key" )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Intermediate masked-score stage '{stage.Stage}' expected " +
				$"file=reference_layer0_scores_masked_pre_softmax.f32, dtype=float32-le, " +
				$"shape=[{headCount},{sequenceLength},{sequenceLength}], layer=0, " +
				$"type=global, sentinel=0xFF7FFFFF, causal_mask=true, " +
				$"pre_softmax=true, softmax=false, elements={expectedElements}; found " +
				$"file={stage.File}, dtype={stage.Dtype}, shape=" +
				$"[{string.Join( ",", stage.Shape ?? Array.Empty<int>() )}], " +
				$"type={stage.AttentionType}, sentinel_bits=" +
				$"{stage.MaskedSentinelFloat32Bits}, elements={stage.Elements}." );
		}
	}

	private static void ValidateAttentionProbabilityReferenceStage(
		ForwardReferenceStage stage,
		int sequenceLength,
		int headCount,
		int headDimension,
		AttentionMaskReferenceDocument mask )
	{
		int expectedElements = headCount * sequenceLength * sequenceLength;
		bool axisOrderMatches = stage.LogicalAxisOrder is not null &&
			stage.LogicalAxisOrder.Length == 3 &&
			stage.LogicalAxisOrder[0] == "head" &&
			stage.LogicalAxisOrder[1] == "query_sequence" &&
			stage.LogicalAxisOrder[2] == "key_sequence";
		if ( stage.File != "reference_layer0_attention_probs.f32" ||
			stage.Dtype != "float32-le" ||
			stage.Shape is null || stage.Shape.Length != 3 ||
			stage.Shape[0] != headCount || stage.Shape[1] != sequenceLength ||
			stage.Shape[2] != sequenceLength || stage.Elements != expectedElements ||
			stage.Bytes != expectedElements * sizeof( float ) ||
			stage.SourceStage != "layer0_scores_masked_pre_softmax" ||
			stage.AttentionLayer != 0 || stage.AttentionType != "global" ||
			stage.HeadCount != headCount || stage.HeadDimension != headDimension ||
			stage.QuerySequenceLength != sequenceLength ||
			stage.KeySequenceLength != sequenceLength || !axisOrderMatches ||
			stage.SoftmaxFunction != "torch.nn.functional.softmax" ||
			stage.SoftmaxDimension != -1 || stage.SoftmaxAxis != "key_sequence" ||
			stage.InputDtype != "float32" || stage.OutputDtype != "float32" ||
			stage.SoftmaxDtypeArgument is not null ||
			stage.PostSoftmaxCast != "attn_weights.to(value.dtype)" ||
			!stage.ModelEval || stage.AttentionDropoutProbability != 0.0f ||
			BitConverter.SingleToInt32Bits( stage.MaskedProbability ) != 0 ||
			stage.MaskedProbabilityFloat32Bits != "0x00000000" ||
			stage.MaskedZeroCount != mask.MaskedCountAllHeads ||
			stage.NonzeroProbabilityCount != mask.AllowedCountAllHeads ||
			stage.MaximumRowSumDeviation > 1.0e-6f ||
			!stage.CausalMaskApplied || !stage.LocalOrGlobalMaskApplied ||
			stage.PreSoftmax || !stage.SoftmaxApplied || !stage.PostSoftmax ||
			stage.DropoutApplied || stage.AttentionTimesValueApplied ||
			stage.FlattenedStorageOrder !=
				"C-order [head,query_sequence,key_sequence]; " +
				"flat=((head*query_sequence_length)+query)*key_sequence_length+key" )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Intermediate attention-probability stage '{stage.Stage}' " +
				$"expected file=reference_layer0_attention_probs.f32, dtype=float32-le, " +
				$"shape=[{headCount},{sequenceLength},{sequenceLength}], " +
				$"source=layer0_scores_masked_pre_softmax, axis=key_sequence(dim=-1), " +
				$"input/output=float32, masked_bits=0x00000000, softmax=true, " +
				$"dropout=false, attention_times_value=false; found file={stage.File}, " +
				$"dtype={stage.Dtype}, shape=" +
				$"[{string.Join( ",", stage.Shape ?? Array.Empty<int>() )}], " +
				$"source={stage.SourceStage}, function={stage.SoftmaxFunction}, " +
				$"axis={stage.SoftmaxAxis}({stage.SoftmaxDimension}), " +
				$"masked_bits={stage.MaskedProbabilityFloat32Bits}, " +
				$"elements={stage.Elements}." );
		}
	}

	private static void ValidateAttentionContextReferenceStage(
		ForwardReferenceStage stage,
		int sequenceLength,
		int headCount,
		int headDimension )
	{
		int expectedElements = headCount * sequenceLength * headDimension;
		bool axisOrderMatches = stage.LogicalAxisOrder is not null &&
			stage.LogicalAxisOrder.Length == 3 &&
			stage.LogicalAxisOrder[0] == "head" &&
			stage.LogicalAxisOrder[1] == "query_sequence" &&
			stage.LogicalAxisOrder[2] == "head_dimension";
		if ( stage.File != "reference_layer0_attention_context_heads.f32" ||
			stage.Dtype != "float32-le" ||
			stage.Shape is null || stage.Shape.Length != 3 ||
			stage.Shape[0] != headCount || stage.Shape[1] != sequenceLength ||
			stage.Shape[2] != headDimension || stage.Elements != expectedElements ||
			stage.Bytes != expectedElements * sizeof( float ) ||
			stage.SourceProbabilityStage != "layer0_attention_probs" ||
			stage.ValueStage != "layer0_v_heads" ||
			stage.AttentionLayer != 0 || stage.AttentionType != "global" ||
			stage.HeadCount != headCount || stage.HeadDimension != headDimension ||
			stage.QuerySequenceLength != sequenceLength ||
			stage.KeySequenceLength != sequenceLength || !axisOrderMatches ||
			stage.Formula !=
				"context[head,query,component] = sum_k " +
				"probability[head,query,k] * V[head,k,component]" ||
			stage.InputDtype != "float32" || stage.ValueDtype != "float32" ||
			stage.OutputDtype != "float32" || !stage.ModelEval ||
			stage.AttentionDropoutProbability != 0.0f || stage.DropoutApplied ||
			!stage.AttentionTimesValueApplied ||
			stage.Query0IdentityElements != headCount * headDimension ||
			!stage.Query0IdentityBitExact ||
			MathF.Abs( stage.ConvexRangeTolerance - 1.0e-6f ) > 1.0e-12f ||
			stage.ConvexRangeCheckedElements != expectedElements ||
			stage.ConvexRangeViolations != 0 || stage.HeadsMerged ||
			stage.OutputProjectionApplied ||
			stage.FlattenedStorageOrder !=
				"C-order [head,query_sequence,head_dimension]; " +
				"flat=((head*query_sequence_length)+query)*head_dimension+component" )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Intermediate attention-context stage '{stage.Stage}' " +
				$"expected file=reference_layer0_attention_context_heads.f32, " +
				$"dtype=float32-le, shape=[{headCount},{sequenceLength},{headDimension}], " +
				"source_probability=layer0_attention_probs, value=layer0_v_heads, " +
				"formula=probabilities-times-V, input/value/output=float32, " +
				"dropout=false, heads_merged=false, output_projection=false; found " +
				$"file={stage.File}, dtype={stage.Dtype}, shape=" +
				$"[{string.Join( ",", stage.Shape ?? Array.Empty<int>() )}], " +
				$"source_probability={stage.SourceProbabilityStage}, " +
				$"value={stage.ValueStage}, elements={stage.Elements}." );
		}
	}

	private static void ValidateAttentionMergedReferenceStage(
		ForwardReferenceStage stage,
		int sequenceLength,
		int hiddenSize,
		int headCount,
		int headDimension )
	{
		ValidateForwardReferenceStage(
			stage,
			"reference_layer0_attention_merged.f32",
			sequenceLength,
			hiddenSize );
		bool sourceShapeMatches = stage.SourceShape is not null &&
			stage.SourceShape.Length == 3 && stage.SourceShape[0] == headCount &&
			stage.SourceShape[1] == sequenceLength &&
			stage.SourceShape[2] == headDimension;
		bool sourceAxisMatches = stage.SourceAxisOrder is not null &&
			stage.SourceAxisOrder.Length == 3 && stage.SourceAxisOrder[0] == "head" &&
			stage.SourceAxisOrder[1] == "sequence" &&
			stage.SourceAxisOrder[2] == "head_dimension";
		bool outputAxisMatches = stage.LogicalAxisOrder is not null &&
			stage.LogicalAxisOrder.Length == 2 &&
			stage.LogicalAxisOrder[0] == "sequence" &&
			stage.LogicalAxisOrder[1] == "hidden";
		bool permutationMatches = stage.Permutation is not null &&
			stage.Permutation.Length == 4 && stage.Permutation[0] == 0 &&
			stage.Permutation[1] == 2 && stage.Permutation[2] == 1 &&
			stage.Permutation[3] == 3;
		if ( stage.SourceStage != "layer0_attention_context_heads" ||
			!sourceShapeMatches || !sourceAxisMatches || !outputAxisMatches ||
			!permutationMatches || !stage.ContiguousCalled ||
			stage.HeadCount != headCount || stage.HeadDimension != headDimension ||
			stage.SequenceLength != sequenceLength || stage.InputDtype != "float32" ||
			stage.OutputDtype != "float32" ||
			stage.Mapping != "merged[token,head*head_dimension+component] = " +
				"context[head,token,component]" ||
			stage.MappingElements != sequenceLength * hiddenSize ||
			!stage.MappingBitExact || !stage.HeadsMerged ||
			stage.OutputProjectionApplied ||
			stage.FlattenedStorageOrder !=
				"C-order [sequence,hidden]; flat=token*hidden_size+feature" )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Intermediate merged-attention stage '{stage.Stage}' " +
				$"expected source=[{headCount},{sequenceLength},{headDimension}], " +
				$"destination=[{sequenceLength},{hiddenSize}], permutation=[0,2,1,3], " +
				$"exact mappings={sequenceLength * hiddenSize}, out_proj=false; found " +
				$"source=[{string.Join( ",", stage.SourceShape ?? Array.Empty<int>() )}], " +
				$"permutation=[{string.Join( ",", stage.Permutation ?? Array.Empty<int>() )}], " +
				$"mappings={stage.MappingElements}, bit_exact={stage.MappingBitExact}." );
		}
	}

	private static void ValidateAttentionOutputProjectionReferenceStage(
		ForwardReferenceStage stage,
		int sequenceLength,
		int hiddenSize )
	{
		ValidateForwardReferenceStage(
			stage,
			"reference_layer0_attention_out_proj.f32",
			sequenceLength,
			hiddenSize );
		bool weightShapeMatches = stage.WeightShape is not null &&
			stage.WeightShape.Length == 2 && stage.WeightShape[0] == hiddenSize &&
			stage.WeightShape[1] == hiddenSize;
		bool biasShapeMatches = stage.BiasShape is not null &&
			stage.BiasShape.Length == 1 && stage.BiasShape[0] == hiddenSize;
		bool axisOrderMatches = stage.LogicalAxisOrder is not null &&
			stage.LogicalAxisOrder.Length == 2 &&
			stage.LogicalAxisOrder[0] == "sequence" &&
			stage.LogicalAxisOrder[1] == "hidden";
		if ( stage.SourceStage != "layer0_attention_merged" ||
			stage.ModuleType != "torch.nn.Linear" ||
			stage.WeightTensor !=
				TinyStoriesForwardStages.Layer0AttentionOutProjectionWeightName ||
			stage.BiasTensor != TinyStoriesForwardStages.Layer0AttentionOutProjectionBiasName ||
			!weightShapeMatches || !stage.BiasExists || !biasShapeMatches ||
			stage.WeightLayout != "[out_features,in_features]" ||
			stage.Formula != "output[token,out] = sum_i input[token,i] * " +
				"weight[out,i] + bias[out]" ||
			!axisOrderMatches || stage.InputDtype != "float32" ||
			stage.WeightDtype != "float32" || stage.BiasDtype != "float32" ||
			stage.OutputDtype != "float32" || stage.AdditionalCast ||
			stage.ResidualDropoutModule != "torch.nn.Dropout" ||
			stage.ResidualDropoutProbability != 0.0f || !stage.ModelEval ||
			stage.DropoutApplied || stage.DropoutChangedBits ||
			!stage.AttentionBranchAfterDropoutBitEqual || !stage.HeadsMerged ||
			!stage.OutputProjectionApplied || stage.ResidualAdditionApplied ||
			stage.FlattenedStorageOrder !=
				"C-order [sequence,hidden]; flat=token*hidden_size+feature" )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Intermediate attention output-projection stage " +
				$"'{stage.Stage}' expected biased Linear({hiddenSize},{hiddenSize}), " +
				$"weight={TinyStoriesForwardStages.Layer0AttentionOutProjectionWeightName}, " +
				$"bias={TinyStoriesForwardStages.Layer0AttentionOutProjectionBiasName}, " +
				"FP32, eval-mode residual dropout p=0 no-op, residual=false; found " +
				$"module={stage.ModuleType}, weight={stage.WeightTensor}, " +
				$"bias={stage.BiasTensor}, bias_exists={stage.BiasExists}, " +
				$"dropout_p={stage.ResidualDropoutProbability:G9}, " +
				$"dropout_applied={stage.DropoutApplied}." );
		}
	}

	private static void ValidateAttentionResidualReferenceStage(
		ForwardReferenceStage stage,
		int sequenceLength,
		int hiddenSize )
	{
		ValidateForwardReferenceStage(
			stage,
			"reference_layer0_attention_residual.f32",
			sequenceLength,
			hiddenSize );
		bool axisOrderMatches = stage.LogicalAxisOrder is not null &&
			stage.LogicalAxisOrder.Length == 2 &&
			stage.LogicalAxisOrder[0] == "sequence" &&
			stage.LogicalAxisOrder[1] == "hidden";
		if ( stage.SourceStage != "layer0_attention_out_proj" ||
			stage.AttentionBranchStage != "layer0_attention_out_proj" ||
			!stage.AttentionBranchAfterDropoutBitEqual ||
			stage.ResidualSourceStage != "combined_embedding" ||
			stage.ResidualSourceBoundary !=
				"transformer block input before layer0 ln_1" ||
			stage.AdditionOrder != "attention_output + residual" ||
			stage.Formula != "output[token,feature] = attention_branch[token,feature] + " +
				"residual[token,feature]" ||
			!axisOrderMatches || stage.InputDtype != "float32" ||
			stage.OutputDtype != "float32" ||
			stage.ResidualDropoutProbability != 0.0f || stage.DropoutApplied ||
			!stage.ResidualSourceBitExactToBlockInput ||
			stage.WrongLn1ResidualMaximumAbsoluteError < 1.0e-3f ||
			!stage.ResidualAdditionApplied || stage.NextStage != "layer0_ln_2" ||
			stage.Layer0Ln2Applied ||
			stage.FlattenedStorageOrder !=
				"C-order [sequence,hidden]; flat=token*hidden_size+feature" )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Intermediate attention-residual stage '{stage.Stage}' " +
				"expected attention branch=layer0_attention_out_proj, residual source=" +
				"combined_embedding/block input before ln_1, FP32 addition, next=ln_2 " +
				$"with ln_2 not applied; found branch={stage.AttentionBranchStage}, " +
				$"residual={stage.ResidualSourceStage}, boundary=" +
				$"'{stage.ResidualSourceBoundary}', next={stage.NextStage}, " +
				$"ln_2={stage.Layer0Ln2Applied}." );
		}
	}

	private static void ValidateLayer0Ln2ReferenceStage(
		ForwardReferenceStage stage,
		int sequenceLength,
		int hiddenSize,
		float epsilon )
	{
		ValidateForwardReferenceStage(
			stage,
			"reference_layer0_ln2.f32",
			sequenceLength,
			hiddenSize );
		bool weightShapeMatches = stage.WeightShape is not null &&
			stage.WeightShape.Length == 1 && stage.WeightShape[0] == hiddenSize;
		bool biasShapeMatches = stage.BiasShape is not null &&
			stage.BiasShape.Length == 1 && stage.BiasShape[0] == hiddenSize;
		bool axisOrderMatches = stage.LogicalAxisOrder is not null &&
			stage.LogicalAxisOrder.Length == 2 &&
			stage.LogicalAxisOrder[0] == "sequence" &&
			stage.LogicalAxisOrder[1] == "hidden";
		if ( stage.SourceStage != "layer0_attention_residual" ||
			stage.SourceImplementation != "torch.nn.LayerNorm" ||
			stage.ModuleType != "torch.nn.LayerNorm" ||
			stage.WeightTensor != TinyStoriesForwardStages.Layer0Ln2WeightName ||
			stage.BiasTensor != TinyStoriesForwardStages.Layer0Ln2BiasName ||
			!weightShapeMatches || !stage.BiasExists || !biasShapeMatches ||
			MathF.Abs( stage.Epsilon - epsilon ) > 1.0e-10f ||
			stage.NormalizationDimension != "hidden (last dimension, size 64)" ||
			stage.Variance != "biased population variance (unbiased=False)" ||
			!axisOrderMatches || stage.InputDtype != "float32" ||
			stage.WeightDtype != "float32" || stage.BiasDtype != "float32" ||
			stage.OutputDtype != "float32" || stage.InputMutated ||
			!stage.Layer0Ln2Applied || stage.NextStage != "layer0_mlp_fc" ||
			stage.FlattenedStorageOrder !=
				"C-order [sequence,hidden]; flat=token*hidden_size+feature" )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Intermediate layer 0 ln_2 stage '{stage.Stage}' expected " +
				$"LayerNorm({hiddenSize}, eps={epsilon:G9}), weight=" +
				$"{TinyStoriesForwardStages.Layer0Ln2WeightName}, bias=" +
				$"{TinyStoriesForwardStages.Layer0Ln2BiasName}, source=" +
				$"layer0_attention_residual, next=layer0_mlp_fc; found source=" +
				$"{stage.SourceStage}, module={stage.ModuleType}, epsilon={stage.Epsilon:G9}, " +
				$"weight={stage.WeightTensor}, bias={stage.BiasTensor}, next={stage.NextStage}." );
		}
	}

	private static void ValidateMlpLinearReferenceStage(
		ForwardReferenceStage stage,
		string expectedFile,
		string expectedSourceStage,
		string expectedWeight,
		string expectedBias,
		int sequenceLength,
		int inputSize,
		int outputSize,
		bool preActivation,
		bool preDropout )
	{
		ValidateForwardReferenceStage(
			stage,
			expectedFile,
			sequenceLength,
			outputSize );
		bool weightShapeMatches = stage.WeightShape is not null &&
			stage.WeightShape.Length == 2 && stage.WeightShape[0] == outputSize &&
			stage.WeightShape[1] == inputSize;
		bool biasShapeMatches = stage.BiasShape is not null &&
			stage.BiasShape.Length == 1 && stage.BiasShape[0] == outputSize;
		string outputAxis = outputSize == 256 ? "intermediate_feature" : "hidden";
		string expectedStorage = outputSize == 256
			? "C-order [sequence,intermediate_feature]; " +
				"flat=token*intermediate_size+feature"
			: "C-order [sequence,hidden]; flat=token*hidden_size+feature";
		bool axisOrderMatches = stage.LogicalAxisOrder is not null &&
			stage.LogicalAxisOrder.Length == 2 &&
			stage.LogicalAxisOrder[0] == "sequence" &&
			stage.LogicalAxisOrder[1] == outputAxis;
		bool dropoutMetadataMatches = !preDropout ||
			(stage.MlpDropoutModule == "torch.nn.Dropout" &&
			stage.MlpDropoutProbability == 0.0f && stage.ModelEval &&
			!stage.DropoutApplied && !stage.DropoutChangedBits &&
			stage.MlpBranchAfterDropoutBitEqual);
		if ( stage.SourceStage != expectedSourceStage ||
			stage.SourceImplementation != "torch.nn.Linear" ||
			stage.ModuleType != "torch.nn.Linear" ||
			stage.WeightTensor != expectedWeight || stage.BiasTensor != expectedBias ||
			!weightShapeMatches || !stage.BiasExists || !biasShapeMatches ||
			stage.WeightLayout != "[out_features,in_features]" ||
			stage.Formula != "output[token,out] = sum_i input[token,i] * " +
				"weight[out,i] + bias[out]" ||
			!axisOrderMatches || stage.FlattenedStorageOrder != expectedStorage ||
			stage.InputDtype != "float32" || stage.WeightDtype != "float32" ||
			stage.BiasDtype != "float32" || stage.OutputDtype != "float32" ||
			stage.InputMutated || stage.IntermediateSize != 256 ||
			stage.PreActivation != preActivation ||
			(preActivation && stage.ActivationApplied) ||
			stage.PreDropout != preDropout || !dropoutMetadataMatches ||
			stage.ResidualAdditionApplied )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Intermediate MLP linear stage '{stage.Stage}' expected " +
				$"file={expectedFile}, source={expectedSourceStage}, Linear({inputSize}," +
				$"{outputSize}), weight={expectedWeight}[{outputSize},{inputSize}], " +
				$"bias={expectedBias}[{outputSize}], pre_activation={preActivation}, " +
				$"pre_dropout={preDropout}; found source={stage.SourceStage}, " +
				$"weight={stage.WeightTensor}, bias={stage.BiasTensor}, " +
				$"pre_activation={stage.PreActivation}, pre_dropout={stage.PreDropout}." );
		}
	}

	private static void ValidateGeluNewReferenceStage(
		ForwardReferenceStage stage,
		int sequenceLength,
		int intermediateSize )
	{
		ValidateForwardReferenceStage(
			stage,
			"reference_layer0_mlp_gelu.f32",
			sequenceLength,
			intermediateSize );
		bool axisOrderMatches = stage.LogicalAxisOrder is not null &&
			stage.LogicalAxisOrder.Length == 2 &&
			stage.LogicalAxisOrder[0] == "sequence" &&
			stage.LogicalAxisOrder[1] == "intermediate_feature";
		int tanhBits = BitConverter.SingleToInt32Bits(
			TinyStoriesForwardStages.GeluNewTanhCoefficient );
		int cubicBits = BitConverter.SingleToInt32Bits(
			TinyStoriesForwardStages.GeluNewCubicCoefficient );
		if ( stage.SourceStage != "layer0_mlp_fc" ||
			stage.SourceImplementation !=
				"transformers.activations.NewGELUActivation.forward" ||
			stage.ActivationFunction != "gelu_new" ||
			stage.ActivationModule != "transformers.activations.NewGELUActivation" ||
			stage.ActivationFormula != TinyStoriesForwardStages.GeluNewFormula ||
			BitConverter.SingleToInt32Bits( stage.TanhCoefficientFloat32 ) != tanhBits ||
			stage.TanhCoefficientFloat32Bits != $"0x{tanhBits:X8}" ||
			BitConverter.SingleToInt32Bits( stage.CubicCoefficientFloat32 ) != cubicBits ||
			stage.CubicCoefficientFloat32Bits != $"0x{cubicBits:X8}" ||
			stage.PowExponent != 3.0f ||
			stage.OperationDtype !=
				"float32 tensor operations; Python scalar constants cast by PyTorch" ||
			!axisOrderMatches || stage.FlattenedStorageOrder !=
				"C-order [sequence,intermediate_feature]; " +
				"flat=token*intermediate_size+feature" ||
			stage.InputDtype != "float32" || stage.OutputDtype != "float32" ||
			stage.IntermediateSize != intermediateSize || stage.PreActivation ||
			!stage.ActivationApplied || stage.InputMutated )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Intermediate GELU-new stage '{stage.Stage}' expected " +
				$"installed NewGELUActivation formula='{TinyStoriesForwardStages.GeluNewFormula}', " +
				$"tanh_bits=0x{tanhBits:X8}, cubic_bits=0x{cubicBits:X8}, " +
				$"shape=[{sequenceLength},{intermediateSize}]; found function=" +
				$"{stage.ActivationFunction}, module={stage.ActivationModule}, " +
				$"formula='{stage.ActivationFormula}', tanh_bits=" +
				$"{stage.TanhCoefficientFloat32Bits}, cubic_bits=" +
				$"{stage.CubicCoefficientFloat32Bits}." );
		}
	}

	private static void ValidateLayer0OutputReferenceStage(
		ForwardReferenceStage stage,
		int sequenceLength,
		int hiddenSize )
	{
		ValidateForwardReferenceStage(
			stage,
			"reference_layer0_output.f32",
			sequenceLength,
			hiddenSize );
		bool axisOrderMatches = stage.LogicalAxisOrder is not null &&
			stage.LogicalAxisOrder.Length == 2 &&
			stage.LogicalAxisOrder[0] == "sequence" &&
			stage.LogicalAxisOrder[1] == "hidden";
		if ( stage.SourceStage != "layer0_mlp_proj" ||
			stage.MlpBranchStage != "layer0_mlp_proj" ||
			stage.SourceImplementation !=
				"transformers.models.gpt_neo.modeling_gpt_neo.GPTNeoBlock.forward" ||
			stage.MlpDropoutModule != "torch.nn.Dropout" ||
			stage.MlpDropoutProbability != 0.0f || !stage.ModelEval ||
			stage.DropoutApplied || stage.DropoutChangedBits ||
			!stage.MlpBranchAfterDropoutBitEqual ||
			stage.ResidualSourceStage != "layer0_attention_residual" ||
			stage.ResidualSourceBoundary !=
				"block hidden state after attention residual and before layer0 ln_2" ||
			stage.AdditionOrder != "residual + feed_forward_hidden_states" ||
			stage.Formula != "output[token,feature] = residual[token,feature] + " +
				"mlp_branch[token,feature]" ||
			!axisOrderMatches || stage.FlattenedStorageOrder !=
				"C-order [sequence,hidden]; flat=token*hidden_size+feature" ||
			stage.InputDtype != "float32" || stage.OutputDtype != "float32" ||
			!stage.ResidualSourceBitExactToAttentionResidual ||
			stage.WrongLn2ResidualMaximumAbsoluteError < 1.0e-3f ||
			!stage.ResidualAdditionApplied || stage.LayerIndex != 0 ||
			stage.AttentionType != "global" || !stage.LayerComplete ||
			stage.NextStage != "layer1" )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] Intermediate complete layer 0 stage '{stage.Stage}' expected " +
				"MLP branch=layer0_mlp_proj, residual=layer0_attention_residual " +
				"before ln_2, eval-mode Dropout(p=0) no-op, residual-first FP32 addition, " +
				$"layer complete and next=layer1; found branch={stage.MlpBranchStage}, " +
				$"residual={stage.ResidualSourceStage}, dropout_p=" +
				$"{stage.MlpDropoutProbability:G9}, layer={stage.LayerIndex}, " +
				$"complete={stage.LayerComplete}, next={stage.NextStage}." );
		}
	}

	private static float[] CopyValues( float[] source )
	{
		if ( source is null )
		{
			throw new ArgumentNullException( nameof( source ) );
		}
		float[] copy = new float[source.Length];
		for ( int index = 0; index < source.Length; index++ )
		{
			copy[index] = source[index];
		}
		return copy;
	}

	private static int CountBitMismatches( float[] expected, float[] actual )
	{
		if ( expected is null || actual is null || expected.Length != actual.Length )
		{
			throw new InvalidOperationException(
				"[LLM:ERROR] Bit-mismatch validation requires equal-length arrays." );
		}
		int mismatches = 0;
		for ( int index = 0; index < expected.Length; index++ )
		{
			if ( BitConverter.SingleToInt32Bits( expected[index] ) !=
				BitConverter.SingleToInt32Bits( actual[index] ) )
			{
				mismatches++;
			}
		}
		return mismatches;
	}

	private static int FirstMismatch( IReadOnlyList<int> expected, IReadOnlyList<int> actual )
	{
		int common = Math.Min( expected.Count, actual.Count );
		for ( int index = 0; index < common; index++ )
		{
			if ( expected[index] != actual[index] )
			{
				return index;
			}
		}
		return expected.Count == actual.Count ? -1 : common;
	}

	private static void RequireConfig( string name, int expected, int actual )
	{
		if ( expected != actual )
		{
			throw new InvalidOperationException(
				$"[LLM:ERROR] {name} expected {expected}, found {actual}." );
		}
	}

	private sealed record FoundationArtifacts(
		SboxLlmModel Model,
		Gpt2ByteBpeTokenizer Tokenizer );
}