Gliner/Preprocessing/GlinerProcessor.cs
using System;
using System.Collections.Generic;
namespace GlinerPoc.Preprocessing;
/// <summary>
/// Native GLiNER classification preprocessing (Phase 3.10–3.15). Reproduces
/// the pinned official path exactly:
///
/// context + task + candidates (+ descriptions)
/// → schema word structure: ( [P] <task+descriptions> ( [L] label ... ) )
/// → combined tokens: schema words + [SEP_TEXT] + context words
/// → per combined token: GlinerTokenizer.TokenizeToken
/// → input IDs (no [CLS], no [SEP]); markers = first-subword positions of
/// the [P] slot (index 1) and every [L] slot (word indices 4, 6, ...,
/// len-3), matching the official routing; classification scoring drops
/// the first ([P]) marker.
///
/// Pure managed data in/out; safe to call from any thread (see architecture
/// doc §15). V1 rejects requests whose FINAL assembled encoded sequence
/// exceeds the plan's 256-token budget with a clear error (no truncation).
/// </summary>
public sealed class GlinerProcessor
{
public const int MaxEncodedTokens = 256;
public const string PToken = "[P]";
public const string LToken = "[L]";
public const string SepTextToken = "[SEP_TEXT]";
public const string SepStructToken = "[SEP_STRUCT]";
public const string DescriptionToken = "[DESCRIPTION]";
private readonly GlinerTokenizer _tokenizer;
public GlinerProcessor( GlinerTokenizer tokenizer )
{
_tokenizer = tokenizer ?? throw new ArgumentNullException( nameof( tokenizer ) );
}
public GlinerTokenizer Tokenizer => _tokenizer;
public GlinerEncodedRequest Encode( GlinerClassificationRequest request )
{
return Encode( request, collectDiagnostics: true );
}
public GlinerEncodedRequest Encode( GlinerClassificationRequest request, bool collectDiagnostics )
{
ValidateRequest( request );
// ---- schema word structure ------------------------------------------
var hasDescriptions = false;
foreach ( var c in request.Candidates )
{
// Official semantics: description MODE activates when any label has a
// description entry, even an empty string; only present entries are
// appended.
if ( c.Description is not null )
{
hasDescriptions = true;
break;
}
}
string prompt = request.Task;
if ( hasDescriptions )
{
var sb = new System.Text.StringBuilder( prompt );
foreach ( var c in request.Candidates )
{
if ( c.Description is null )
{
continue;
}
sb.Append( ' ' ).Append( DescriptionToken ).Append( ' ' )
.Append( c.Label ).Append( ": " ).Append( c.Description );
}
prompt = sb.ToString();
}
var schema = new List<string>( 8 + request.Candidates.Count * 2 )
{
"(", PToken, prompt, "(",
};
foreach ( var c in request.Candidates )
{
schema.Add( LToken );
schema.Add( c.Label );
}
schema.Add( ")" );
schema.Add( ")" );
// ---- context words -----------------------------------------------------
string context = GlinerTextPreprocessor.ApplyContextRule( request.Context );
var words = GlinerTextPreprocessor.SplitWords( context );
// ---- combined tokens ----------------------------------------------------
var combined = new List<string>( schema.Count + 1 + words.Count );
combined.AddRange( schema );
combined.Add( SepTextToken );
combined.AddRange( words );
// ---- marker routing (official rule) --------------------------------------
// cls_marker_indices contains ONLY the [L] slots (word indices 4, 6, ...,
// len-3). The [P] marker lives in schema_special_positions[0] on the
// Python side and is stripped when the batch routes classification
// markers; legacy decoding re-adds a zero row via embs[1:]. Native
// Phase 4 gathers these [L] positions directly.
var markerWordIndices = new List<int>( request.Candidates.Count );
for ( int i = 4; i < schema.Count - 2; i += 2 )
{
markerWordIndices.Add( i );
}
// ---- tokenize every combined token ---------------------------------------
var ids = new List<int>( 64 );
var allPieces = collectDiagnostics ? new List<string>( 64 ) : null;
var tokenPieces = collectDiagnostics ? new List<List<string>>( combined.Count ) : null;
var markerPositions = new List<int>( markerWordIndices.Count );
var tokenStarts = new int[combined.Count];
for ( int i = 0; i < combined.Count; i++ )
{
tokenStarts[i] = ids.Count;
var pieceList = collectDiagnostics ? new List<string>() : null;
int[] tokenIds = _tokenizer.TokenizeToken( combined[i], pieceList );
ids.AddRange( tokenIds );
if ( collectDiagnostics )
{
tokenPieces.Add( pieceList );
if ( pieceList is not null )
{
allPieces.AddRange( pieceList );
}
}
}
for ( int i = 0; i < combined.Count; i++ )
{
foreach ( int wordIndex in markerWordIndices )
{
if ( wordIndex == i )
{
markerPositions.Add( tokenStarts[i] );
}
}
}
if ( ids.Count > MaxEncodedTokens )
{
throw new InvalidOperationException(
$"[GLI:ERROR] Encoded request exceeds the supported native limit: " +
$"{ids.Count} encoded tokens > {MaxEncodedTokens}. Truncation is disabled in V1." );
}
var markerArray = markerPositions.ToArray();
foreach ( int m in markerArray )
{
if ( m < 0 || m >= ids.Count )
{
throw new InvalidOperationException(
$"[GLI:ERROR] Classification marker position {m} outside sequence length {ids.Count}." );
}
}
if ( markerArray.Length != request.Candidates.Count )
{
throw new InvalidOperationException(
$"[GLI:ERROR] Marker count {markerArray.Length} != candidate count " +
$"{request.Candidates.Count}." );
}
var order = new List<string>( request.Candidates.Count );
foreach ( var c in request.Candidates )
{
order.Add( c.Label );
}
var mask = new byte[ids.Count];
Array.Fill( mask, (byte)1 );
return new GlinerEncodedRequest(
ids.ToArray(),
mask,
markerArray,
order,
words.Count,
collectDiagnostics ? schema.ToArray() : null,
collectDiagnostics ? words.ToArray() : null,
collectDiagnostics ? combined.ToArray() : null,
collectDiagnostics ? allPieces.ToArray() : null );
}
private static void ValidateRequest( GlinerClassificationRequest request )
{
if ( request is null )
{
throw new InvalidOperationException( "[GLI:ERROR] Request is null." );
}
if ( string.IsNullOrEmpty( request.Task ) )
{
throw new InvalidOperationException( "[GLI:ERROR] Task is null or empty." );
}
if ( request.Candidates is null || request.Candidates.Count == 0 )
{
throw new InvalidOperationException( "[GLI:ERROR] Request has no candidates." );
}
foreach ( var c in request.Candidates )
{
if ( c is null || c.Label is null )
{
throw new InvalidOperationException( "[GLI:ERROR] Candidate or label is null." );
}
if ( c.Label.Length == 0 )
{
// Official behaviour: an empty label tokenizes to nothing but
// keeps its [L] marker. V1 parity keeps that behaviour.
continue;
}
}
}
}