using System.Text.Json.Serialization;

namespace LlmPoc.Llm;

public sealed class Gpt2TokenizerDocument
{
	[JsonPropertyName( "version" )]
	public string Version { get; set; }

	[JsonPropertyName( "added_tokens" )]
	public List<Gpt2AddedTokenDocument> AddedTokens { get; set; }

	[JsonPropertyName( "normalizer" )]
	public object Normalizer { get; set; }

	[JsonPropertyName( "pre_tokenizer" )]
	public Gpt2ByteLevelDocument PreTokenizer { get; set; }

	[JsonPropertyName( "post_processor" )]
	public Gpt2ByteLevelDocument PostProcessor { get; set; }

	[JsonPropertyName( "decoder" )]
	public Gpt2ByteLevelDocument Decoder { get; set; }

	[JsonPropertyName( "model" )]
	public Gpt2BpeModelDocument Model { get; set; }
}

public sealed class Gpt2AddedTokenDocument
{
	[JsonPropertyName( "id" )]
	public int Id { get; set; }

	[JsonPropertyName( "content" )]
	public string Content { get; set; }

	[JsonPropertyName( "special" )]
	public bool Special { get; set; }
}

public sealed class Gpt2ByteLevelDocument
{
	[JsonPropertyName( "type" )]
	public string Type { get; set; }

	[JsonPropertyName( "add_prefix_space" )]
	public bool AddPrefixSpace { get; set; }

	[JsonPropertyName( "trim_offsets" )]
	public bool TrimOffsets { get; set; }

	[JsonPropertyName( "use_regex" )]
	public bool UseRegex { get; set; }
}

public sealed class Gpt2BpeModelDocument
{
	[JsonPropertyName( "type" )]
	public string Type { get; set; }

	[JsonPropertyName( "dropout" )]
	public float? Dropout { get; set; }

	[JsonPropertyName( "unk_token" )]
	public string UnknownToken { get; set; }

	[JsonPropertyName( "continuing_subword_prefix" )]
	public string ContinuingSubwordPrefix { get; set; }

	[JsonPropertyName( "end_of_word_suffix" )]
	public string EndOfWordSuffix { get; set; }

	[JsonPropertyName( "fuse_unk" )]
	public bool FuseUnknown { get; set; }

	[JsonPropertyName( "byte_fallback" )]
	public bool ByteFallback { get; set; }

	[JsonPropertyName( "ignore_merges" )]
	public bool IgnoreMerges { get; set; }

	[JsonPropertyName( "vocab" )]
	public Dictionary<string, int> Vocabulary { get; set; }

	[JsonPropertyName( "merges" )]
	public List<string[]> Merges { get; set; }
}