AI Decision Maker (GLiNER 2.5)
Released
Native GPU-accelerated GLiNER 2.5 decision workbench for testing structured AI choices directly inside s&box.
Gliner/Preprocessing/PreprocessingFixturesV2.cs
// GENERATED by tools/gliner/export_preproc_fixtures_v2.py (Phase 3.17).
// Expected preprocessing results from the pinned Python oracle.
// Do not edit by hand; regenerate instead. (Renamed from .g.cs: Razor SDK treats *.g.cs specially.)
using System;
namespace GlinerPoc.Preprocessing;
public sealed class PreprocessingFixturesV2
{
public sealed record Fixture(
string Name, string Context, string Task, string[] Labels, string[] DescLabels, string[] DescTexts,
string[] ContextWords, string[] CombinedTokens, string[][] PerTokenPieces,
int[] InputIds, byte[] AttentionMask, int[] MarkerPositions, int[] MarkerTokenIds)
{
public int SequenceLength => InputIds.Length;
}
public static readonly Fixture[] All =
{
new Fixture(
"basic_hello", "hello world", "What is this?",
new string[]{"greeting","question"},
new string[]{},
new string[]{},
new string[]{"hello","world","."}, new string[]{"(","[P]","What is this?","(","[L]","greeting","[L]","question",")",")","[SEP_TEXT]","hello","world","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁What","▁is","▁this","?"},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁greeting"},new string[]{"[L]"},new string[]{"▁question"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁hello"},new string[]{"▁world"},new string[]{"▁."}},
new int[]{287,128003,458,269,291,302,287,128007,14344,128007,900,1263,1263,128002,12018,447,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{7,9}, new int[]{128007,128007}),
new Fixture(
"game_state", "Health: 25%. Ammo: 2/10. Player visible: yes. Heal available: yes.", "Choose the best action.",
new string[]{"attack","heal","reload","retreat"},
new string[]{},
new string[]{},
new string[]{"health",":","25","%",".","ammo",":","2","/","10",".","player","visible",":","yes",".","heal","available",":","yes","."}, new string[]{"(","[P]","Choose the best action.","(","[L]","attack","[L]","heal","[L]","reload","[L]","retreat",")",")","[SEP_TEXT]","health",":","25","%",".","ammo",":","2","/","10",".","player","visible",":","yes",".","heal","available",":","yes","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁Choose","▁the","▁best","▁action","."},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁attack"},new string[]{"[L]"},new string[]{"▁heal"},new string[]{"[L]"},new string[]{"▁reload"},new string[]{"[L]"},new string[]{"▁retreat"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁health"},new string[]{"▁:"},new string[]{"▁25"},new string[]{"▁%"},new string[]{"▁."},new string[]{"▁ammo"},new string[]{"▁:"},new string[]{"▁2"},new string[]{"▁/"},new string[]{"▁10"},new string[]{"▁."},new string[]{"▁player"},new string[]{"▁visible"},new string[]{"▁:"},new string[]{"▁yes"},new string[]{"▁."},new string[]{"▁heal"},new string[]{"▁available"},new string[]{"▁:"},new string[]{"▁yes"},new string[]{"▁."}},
new int[]{287,128003,4989,262,410,1016,260,287,128007,1815,128007,7533,128007,22452,128007,7886,1263,1263,128002,622,877,977,4639,323,22405,877,392,840,466,323,1472,3979,877,2489,323,7533,499,877,2489,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{8,10,12,14}, new int[]{128007,128007,128007,128007}),
new Fixture(
"ws_multi_space", "status is ok", "Classify status.",
new string[]{"ok","bad"},
new string[]{},
new string[]{},
new string[]{"status","is","ok","."}, new string[]{"(","[P]","Classify status.","(","[L]","ok","[L]","bad",")",")","[SEP_TEXT]","status","is","ok","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁Class","ify","▁status","."},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁ok"},new string[]{"[L]"},new string[]{"▁bad"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁status"},new string[]{"▁is"},new string[]{"▁ok"},new string[]{"▁."}},
new int[]{287,128003,3854,12336,1980,260,287,128007,5522,128007,966,1263,1263,128002,1980,269,5522,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{7,9}, new int[]{128007,128007}),
new Fixture(
"ws_tabs", "A\tB\t\tC", "Classify the line.",
new string[]{"list","prose"},
new string[]{},
new string[]{},
new string[]{"a","b","c","."}, new string[]{"(","[P]","Classify the line.","(","[L]","list","[L]","prose",")",")","[SEP_TEXT]","a","b","c","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁Class","ify","▁the","▁line","."},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁list"},new string[]{"[L]"},new string[]{"▁prose"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁a"},new string[]{"▁b"},new string[]{"▁c"},new string[]{"▁."}},
new int[]{287,128003,3854,12336,262,683,260,287,128007,686,128007,16493,1263,1263,128002,266,2165,2285,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{8,10}, new int[]{128007,128007}),
new Fixture(
"ws_newlines", "line one\nline two\r\nline three", "How many lines?",
new string[]{"two","three","four"},
new string[]{},
new string[]{},
new string[]{"line","one","line","two","line","three","."}, new string[]{"(","[P]","How many lines?","(","[L]","two","[L]","three","[L]","four",")",")","[SEP_TEXT]","line","one","line","two","line","three","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁How","▁many","▁lines","?"},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁two"},new string[]{"[L]"},new string[]{"▁three"},new string[]{"[L]"},new string[]{"▁four"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁line"},new string[]{"▁one"},new string[]{"▁line"},new string[]{"▁two"},new string[]{"▁line"},new string[]{"▁three"},new string[]{"▁."}},
new int[]{287,128003,577,386,1728,302,287,128007,375,128007,475,128007,654,1263,1263,128002,683,311,683,375,683,475,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{7,9,11}, new int[]{128007,128007,128007}),
new Fixture(
"ws_leading_trailing", " padded both sides ", "Classify padding.",
new string[]{"padded","clean"},
new string[]{},
new string[]{},
new string[]{"padded","both","sides","."}, new string[]{"(","[P]","Classify padding.","(","[L]","padded","[L]","clean",")",")","[SEP_TEXT]","padded","both","sides","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁Class","ify","▁padding","."},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁padded"},new string[]{"[L]"},new string[]{"▁clean"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁padded"},new string[]{"▁both"},new string[]{"▁sides"},new string[]{"▁."}},
new int[]{287,128003,3854,12336,20162,260,287,128007,16719,128007,1347,1263,1263,128002,16719,462,2793,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{7,9}, new int[]{128007,128007}),
new Fixture(
"num_int", "Enemies: 12", "How many enemies?",
new string[]{"few","many"},
new string[]{},
new string[]{},
new string[]{"enemies",":","12","."}, new string[]{"(","[P]","How many enemies?","(","[L]","few","[L]","many",")",")","[SEP_TEXT]","enemies",":","12","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁How","▁many","▁enemies","?"},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁few"},new string[]{"[L]"},new string[]{"▁many"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁enemies"},new string[]{"▁:"},new string[]{"▁12"},new string[]{"▁."}},
new int[]{287,128003,577,386,6253,302,287,128007,477,128007,386,1263,1263,128002,6253,877,621,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{7,9}, new int[]{128007,128007}),
new Fixture(
"num_negative", "Score: -15", "Is the score good?",
new string[]{"good","bad"},
new string[]{},
new string[]{},
new string[]{"score",":","-","15","."}, new string[]{"(","[P]","Is the score good?","(","[L]","good","[L]","bad",")",")","[SEP_TEXT]","score",":","-","15","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁I","s","▁the","▁score","▁good","?"},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁good"},new string[]{"[L]"},new string[]{"▁bad"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁score"},new string[]{"▁:"},new string[]{"▁-"},new string[]{"▁15"},new string[]{"▁."}},
new int[]{287,128003,273,268,262,2271,397,302,287,128007,397,128007,966,1263,1263,128002,2271,877,341,706,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{9,11}, new int[]{128007,128007}),
new Fixture(
"num_decimal", "Progress: 87.5 percent complete", "Is it nearly done?",
new string[]{"yes","no"},
new string[]{},
new string[]{},
new string[]{"progress",":","87",".","5","percent","complete","."}, new string[]{"(","[P]","Is it nearly done?","(","[L]","yes","[L]","no",")",")","[SEP_TEXT]","progress",":","87",".","5","percent","complete","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁I","s","▁it","▁nearly","▁done","?"},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁yes"},new string[]{"[L]"},new string[]{"▁no"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁progress"},new string[]{"▁:"},new string[]{"▁87"},new string[]{"▁."},new string[]{"▁5"},new string[]{"▁percent"},new string[]{"▁complete"},new string[]{"▁."}},
new int[]{287,128003,273,268,278,1533,619,302,287,128007,2489,128007,363,1263,1263,128002,2201,877,9204,323,456,864,850,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{9,11}, new int[]{128007,128007}),
new Fixture(
"num_percent", "Battery: 8%", "Is power low?",
new string[]{"low","high"},
new string[]{},
new string[]{},
new string[]{"battery",":","8","%","."}, new string[]{"(","[P]","Is power low?","(","[L]","low","[L]","high",")",")","[SEP_TEXT]","battery",":","8","%","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁I","s","▁power","▁low","?"},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁low"},new string[]{"[L]"},new string[]{"▁high"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁battery"},new string[]{"▁:"},new string[]{"▁8"},new string[]{"▁%"},new string[]{"▁."}},
new int[]{287,128003,273,268,617,759,302,287,128007,759,128007,459,1263,1263,128002,2643,877,578,4639,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{8,10}, new int[]{128007,128007}),
new Fixture(
"num_ratio", "Kills to deaths: 3/1", "Is the ratio good?",
new string[]{"good","poor"},
new string[]{},
new string[]{},
new string[]{"kills","to","deaths",":","3","/","1","."}, new string[]{"(","[P]","Is the ratio good?","(","[L]","good","[L]","poor",")",")","[SEP_TEXT]","kills","to","deaths",":","3","/","1","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁I","s","▁the","▁ratio","▁good","?"},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁good"},new string[]{"[L]"},new string[]{"▁poor"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁kills"},new string[]{"▁to"},new string[]{"▁deaths"},new string[]{"▁:"},new string[]{"▁3"},new string[]{"▁/"},new string[]{"▁1"},new string[]{"▁."}},
new int[]{287,128003,273,268,262,3678,397,302,287,128007,397,128007,1970,1263,1263,128002,11137,264,5876,877,404,840,376,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{9,11}, new int[]{128007,128007}),
new Fixture(
"num_large", "Distance: 1000000 meters", "Is it far?",
new string[]{"far","near"},
new string[]{},
new string[]{},
new string[]{"distance",":","1000000","meters","."}, new string[]{"(","[P]","Is it far?","(","[L]","far","[L]","near",")",")","[SEP_TEXT]","distance",":","1000000","meters","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁I","s","▁it","▁far","?"},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁far"},new string[]{"[L]"},new string[]{"▁near"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁distance"},new string[]{"▁:"},new string[]{"▁1000","000"},new string[]{"▁meters"},new string[]{"▁."}},
new int[]{287,128003,273,268,278,659,302,287,128007,659,128007,941,1263,1263,128002,2249,877,4985,528,5580,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{8,10}, new int[]{128007,128007}),
new Fixture(
"punct_mixed", "Wait... really?! Yes: 'yes', \"no\"; (maybe) [sure] {doubt} a/b c-d", "Classify tone.",
new string[]{"confident","uncertain"},
new string[]{},
new string[]{},
new string[]{"wait",".",".",".","really","?","!","yes",":","'","yes","'",",","\"","no","\"",";","(","maybe",")","[","sure","]","{","doubt","}","a","/","b","c-d","."}, new string[]{"(","[P]","Classify tone.","(","[L]","confident","[L]","uncertain",")",")","[SEP_TEXT]","wait",".",".",".","really","?","!","yes",":","'","yes","'",",","\"","no","\"",";","(","maybe",")","[","sure","]","{","doubt","}","a","/","b","c-d","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁Class","ify","▁tone","."},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁confident"},new string[]{"[L]"},new string[]{"▁uncertain"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁wait"},new string[]{"▁."},new string[]{"▁."},new string[]{"▁."},new string[]{"▁really"},new string[]{"▁?"},new string[]{"▁!"},new string[]{"▁yes"},new string[]{"▁:"},new string[]{"▁'"},new string[]{"▁yes"},new string[]{"▁'"},new string[]{"▁,"},new string[]{"▁\""},new string[]{"▁no"},new string[]{"▁\""},new string[]{"▁;"},new string[]{"▁("},new string[]{"▁maybe"},new string[]{"▁)"},new string[]{"▁["},new string[]{"▁sure"},new string[]{"▁]"},new string[]{"▁{"},new string[]{"▁doubt"},new string[]{"▁}"},new string[]{"▁a"},new string[]{"▁/"},new string[]{"▁b"},new string[]{"▁c","-","d"},new string[]{"▁."}},
new int[]{287,128003,3854,12336,4063,260,287,128007,3797,128007,10518,1263,1263,128002,1495,323,323,323,431,1102,1084,2489,877,382,2489,382,366,307,363,307,2600,287,1461,1263,647,521,6311,13856,2495,15805,266,840,2165,2285,271,407,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{7,9}, new int[]{128007,128007}),
new Fixture(
"punct_apostrophe", "The player's base isn't safe, it's exposed", "Defend the base?",
new string[]{"defend","ignore"},
new string[]{},
new string[]{},
new string[]{"the","player","'","s","base","isn","'","t","safe",",","it","'","s","exposed","."}, new string[]{"(","[P]","Defend the base?","(","[L]","defend","[L]","ignore",")",")","[SEP_TEXT]","the","player","'","s","base","isn","'","t","safe",",","it","'","s","exposed","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁Defend","▁the","▁base","?"},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁defend"},new string[]{"[L]"},new string[]{"▁ignore"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁the"},new string[]{"▁player"},new string[]{"▁'"},new string[]{"▁s"},new string[]{"▁base"},new string[]{"▁isn"},new string[]{"▁'"},new string[]{"▁t"},new string[]{"▁safe"},new string[]{"▁,"},new string[]{"▁it"},new string[]{"▁'"},new string[]{"▁s"},new string[]{"▁exposed"},new string[]{"▁."}},
new int[]{287,128003,55002,262,1436,302,287,128007,5595,128007,5690,1263,1263,128002,262,1472,382,1550,1436,928,382,1941,1295,366,278,382,1550,4062,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{7,9}, new int[]{128007,128007}),
new Fixture(
"uni_accents", "Le café proposes des gaufres à Bruxelles", "What food?",
new string[]{"waffles","soup"},
new string[]{},
new string[]{},
new string[]{"le","café","proposes","des","gaufres","à","bruxelles","."}, new string[]{"(","[P]","What food?","(","[L]","waffles","[L]","soup",")",")","[SEP_TEXT]","le","café","proposes","des","gaufres","à","bruxelles","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁What","▁food","?"},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁waffles"},new string[]{"[L]"},new string[]{"▁soup"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁le"},new string[]{"▁café"},new string[]{"▁proposes"},new string[]{"▁des"},new string[]{"▁g","auf","re","s"},new string[]{"▁à"},new string[]{"▁bru","x","elles"},new string[]{"▁."}},
new int[]{287,128003,458,645,302,287,128007,35096,128007,6121,1263,1263,128002,5352,15924,17059,5385,2905,52296,368,268,9681,39416,982,44100,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{6,8}, new int[]{128007,128007}),
new Fixture(
"uni_smart", "He said “hello” — then left…", "Did he greet?",
new string[]{"greeted","ignored"},
new string[]{},
new string[]{},
new string[]{"he","said","“","hello","”","—","then","left","…","."}, new string[]{"(","[P]","Did he greet?","(","[L]","greeted","[L]","ignored",")",")","[SEP_TEXT]","he","said","“","hello","”","—","then","left","…","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁Did","▁he","▁greet","?"},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁greeted"},new string[]{"[L]"},new string[]{"▁ignored"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁he"},new string[]{"▁said"},new string[]{"▁“"},new string[]{"▁hello"},new string[]{"▁","”"},new string[]{"▁—"},new string[]{"▁then"},new string[]{"▁left"},new string[]{"▁","…"},new string[]{"▁."}},
new int[]{287,128003,2709,313,15684,302,287,128007,12245,128007,6935,1263,1263,128002,313,357,317,12018,507,318,533,393,595,507,3,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{7,9}, new int[]{128007,128007}),
new Fixture(
"uni_cjk", "玩家血量很低，弹药充足", "Best action?",
new string[]{"attack","reload"},
new string[]{},
new string[]{},
new string[]{"玩家血量很低","，","弹药充足","."}, new string[]{"(","[P]","Best action?","(","[L]","attack","[L]","reload",")",")","[SEP_TEXT]","玩家血量很低","，","弹药充足","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁Best","▁action","?"},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁attack"},new string[]{"[L]"},new string[]{"▁reload"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁","玩","家","血","量","很","低"},new string[]{"▁","，"},new string[]{"▁","弹","药","充","足"},new string[]{"▁."}},
new int[]{287,128003,1652,1016,302,287,128007,1815,128007,22452,1263,1263,128002,507,123204,64619,121285,53954,105359,101748,507,3,507,124225,123276,122636,122703,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{6,8}, new int[]{128007,128007}),
new Fixture(
"uni_emoji", "Boss spawned 🐉🔥 run!", "What to do?",
new string[]{"fight","flee"},
new string[]{},
new string[]{},
new string[]{"boss","spawned","🐉","🔥","run","!"}, new string[]{"(","[P]","What to do?","(","[L]","fight","[L]","flee",")",")","[SEP_TEXT]","boss","spawned","🐉","🔥","run","!"}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁What","▁to","▁do","?"},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁fight"},new string[]{"[L]"},new string[]{"▁flee"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁boss"},new string[]{"▁spawned"},new string[]{"▁","🐉"},new string[]{"▁🔥"},new string[]{"▁run"},new string[]{"▁!"}},
new int[]{287,128003,458,264,333,302,287,128007,1801,128007,16021,1263,1263,128002,4861,28579,507,3,115475,684,1084}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{7,9}, new int[]{128007,128007}),
new Fixture(
"schema_like", "This text mentions [P] and [L] and [SEP_TEXT] literally", "Is this schema?",
new string[]{"schema","plain"},
new string[]{},
new string[]{},
new string[]{"this","text","mentions","[","p","]","and","[","l","]","and","[","sep_text","]","literally","."}, new string[]{"(","[P]","Is this schema?","(","[L]","schema","[L]","plain",")",")","[SEP_TEXT]","this","text","mentions","[","p","]","and","[","l","]","and","[","sep_text","]","literally","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁I","s","▁this","▁schema","?"},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁schema"},new string[]{"[L]"},new string[]{"▁plain"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁this"},new string[]{"▁text"},new string[]{"▁mentions"},new string[]{"▁["},new string[]{"▁p"},new string[]{"▁]"},new string[]{"▁and"},new string[]{"▁["},new string[]{"▁l"},new string[]{"▁]"},new string[]{"▁and"},new string[]{"▁["},new string[]{"▁sep","_","text"},new string[]{"▁]"},new string[]{"▁literally"},new string[]{"▁."}},
new int[]{287,128003,273,268,291,22941,302,287,128007,22941,128007,4827,1263,1263,128002,291,1529,12583,647,845,6311,263,647,2531,6311,263,647,41418,616,12948,6311,3839,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{8,10}, new int[]{128007,128007}),
new Fixture(
"labels_two", "The sky is dark and stars are out.", "Time of day?",
new string[]{"night","day"},
new string[]{},
new string[]{},
new string[]{"the","sky","is","dark","and","stars","are","out","."}, new string[]{"(","[P]","Time of day?","(","[L]","night","[L]","day",")",")","[SEP_TEXT]","the","sky","is","dark","and","stars","are","out","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁Time","▁of","▁day","?"},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁night"},new string[]{"[L]"},new string[]{"▁day"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁the"},new string[]{"▁sky"},new string[]{"▁is"},new string[]{"▁dark"},new string[]{"▁and"},new string[]{"▁stars"},new string[]{"▁are"},new string[]{"▁out"},new string[]{"▁."}},
new int[]{287,128003,2210,265,406,302,287,128007,661,128007,406,1263,1263,128002,262,3907,269,1777,263,2906,281,321,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{7,9}, new int[]{128007,128007}),
new Fixture(
"labels_many", "Return opened item after 30 days, no receipt, box damaged.", "Choose the best action.",
new string[]{"accept refund","offer store credit","decline refund","escalate to manager","request more information","exchange product"},
new string[]{},
new string[]{},
new string[]{"return","opened","item","after","30","days",",","no","receipt",",","box","damaged","."}, new string[]{"(","[P]","Choose the best action.","(","[L]","accept refund","[L]","offer store credit","[L]","decline refund","[L]","escalate to manager","[L]","request more information","[L]","exchange product",")",")","[SEP_TEXT]","return","opened","item","after","30","days",",","no","receipt",",","box","damaged","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁Choose","▁the","▁best","▁action","."},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁accept","▁refund"},new string[]{"[L]"},new string[]{"▁offer","▁store","▁credit"},new string[]{"[L]"},new string[]{"▁decline","▁refund"},new string[]{"[L]"},new string[]{"▁escalate","▁to","▁manager"},new string[]{"[L]"},new string[]{"▁request","▁more","▁information"},new string[]{"[L]"},new string[]{"▁exchange","▁product"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁return"},new string[]{"▁opened"},new string[]{"▁item"},new string[]{"▁after"},new string[]{"▁30"},new string[]{"▁days"},new string[]{"▁,"},new string[]{"▁no"},new string[]{"▁receipt"},new string[]{"▁,"},new string[]{"▁box"},new string[]{"▁damaged"},new string[]{"▁."}},
new int[]{287,128003,4989,262,410,1016,260,287,128007,2272,6257,128007,630,1106,1236,128007,4680,6257,128007,29738,264,2140,128007,1409,310,439,128007,2395,714,1263,1263,128002,1067,1932,1905,385,703,538,366,363,7610,366,1352,4514,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{8,11,15,18,22,26}, new int[]{128007,128007,128007,128007,128007,128007}),
new Fixture(
"labels_short", "go", "Command?",
new string[]{"go","stop"},
new string[]{},
new string[]{},
new string[]{"go","."}, new string[]{"(","[P]","Command?","(","[L]","go","[L]","stop",")",")","[SEP_TEXT]","go","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁Command","?"},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁go"},new string[]{"[L]"},new string[]{"▁stop"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁go"},new string[]{"▁."}},
new int[]{287,128003,10270,302,287,128007,424,128007,1001,1263,1263,128002,424,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{5,7}, new int[]{128007,128007}),
new Fixture(
"labels_multiword", "The enemy squad is massing at the northern bridge at dawn", "Choose the best action.",
new string[]{"attack now","wait and observe","request reinforcements"},
new string[]{},
new string[]{},
new string[]{"the","enemy","squad","is","massing","at","the","northern","bridge","at","dawn","."}, new string[]{"(","[P]","Choose the best action.","(","[L]","attack now","[L]","wait and observe","[L]","request reinforcements",")",")","[SEP_TEXT]","the","enemy","squad","is","massing","at","the","northern","bridge","at","dawn","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁Choose","▁the","▁best","▁action","."},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁attack","▁now"},new string[]{"[L]"},new string[]{"▁wait","▁and","▁observe"},new string[]{"[L]"},new string[]{"▁request","▁reinforcements"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁the"},new string[]{"▁enemy"},new string[]{"▁squad"},new string[]{"▁is"},new string[]{"▁mass","ing"},new string[]{"▁at"},new string[]{"▁the"},new string[]{"▁northern"},new string[]{"▁bridge"},new string[]{"▁at"},new string[]{"▁dawn"},new string[]{"▁."}},
new int[]{287,128003,4989,262,410,1016,260,287,128007,1815,394,128007,1495,263,6695,128007,1409,40518,1263,1263,128002,262,4648,6450,269,2675,510,288,262,4570,3579,288,12179,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{8,11,15}, new int[]{128007,128007,128007}),
new Fixture(
"labels_punct", "Convert $1,000 to EUR at 0.9", "Is this possible?",
new string[]{"yes","no"},
new string[]{},
new string[]{},
new string[]{"convert","$","1",",","000","to","eur","at","0",".","9","."}, new string[]{"(","[P]","Is this possible?","(","[L]","yes","[L]","no",")",")","[SEP_TEXT]","convert","$","1",",","000","to","eur","at","0",".","9","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁I","s","▁this","▁possible","?"},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁yes"},new string[]{"[L]"},new string[]{"▁no"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁convert"},new string[]{"▁$"},new string[]{"▁1"},new string[]{"▁,"},new string[]{"▁000"},new string[]{"▁to"},new string[]{"▁e","ur"},new string[]{"▁at"},new string[]{"▁0"},new string[]{"▁."},new string[]{"▁9"},new string[]{"▁."}},
new int[]{287,128003,273,268,291,628,302,287,128007,2489,128007,363,1263,1263,128002,5338,419,376,366,7962,264,865,4757,288,767,323,712,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{8,10}, new int[]{128007,128007}),
new Fixture(
"labels_descriptions", "A hiker is lost, night is falling, temperature is dropping.", "What should the hiker do?",
new string[]{"build shelter","keep walking"},
new string[]{"build shelter","keep walking"},
new string[]{"construct windbreak and insulate ground before dark","continue along the ridge hoping to find the trail"},
new string[]{"a","hiker","is","lost",",","night","is","falling",",","temperature","is","dropping","."}, new string[]{"(","[P]","What should the hiker do? [DESCRIPTION] build shelter: construct windbreak and insulate ground before dark [DESCRIPTION] keep walking: continue along the ridge hoping to find the trail","(","[L]","build shelter","[L]","keep walking",")",")","[SEP_TEXT]","a","hiker","is","lost",",","night","is","falling",",","temperature","is","dropping","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁What","▁should","▁the","▁hiker","▁do","?","[DESCRIPTION]","▁build","▁shelter",":","▁construct","▁wind","break","▁and","▁insulate","▁ground","▁before","▁dark","[DESCRIPTION]","▁keep","▁walking",":","▁continue","▁along","▁the","▁ridge","▁hoping","▁to","▁find","▁the","▁trail"},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁build","▁shelter"},new string[]{"[L]"},new string[]{"▁keep","▁walking"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁a"},new string[]{"▁hiker"},new string[]{"▁is"},new string[]{"▁lost"},new string[]{"▁,"},new string[]{"▁night"},new string[]{"▁is"},new string[]{"▁falling"},new string[]{"▁,"},new string[]{"▁temperature"},new string[]{"▁is"},new string[]{"▁dropping"},new string[]{"▁."}},
new int[]{287,128003,458,403,262,52313,333,302,128010,1044,6412,294,7970,2450,19051,263,43661,1353,416,1777,128010,548,2220,294,959,641,262,15056,3293,264,433,262,3707,287,128007,1044,6412,128007,548,2220,1263,1263,128002,266,52313,269,1125,366,661,269,3928,366,2247,269,7053,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{34,37}, new int[]{128007,128007}),
new Fixture(
"labels_reordered", "Health: 25%. Heal available: yes. Enemy visible: no.", "Choose the best action.",
new string[]{"retreat","heal","attack","reload"},
new string[]{},
new string[]{},
new string[]{"health",":","25","%",".","heal","available",":","yes",".","enemy","visible",":","no","."}, new string[]{"(","[P]","Choose the best action.","(","[L]","retreat","[L]","heal","[L]","attack","[L]","reload",")",")","[SEP_TEXT]","health",":","25","%",".","heal","available",":","yes",".","enemy","visible",":","no","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁Choose","▁the","▁best","▁action","."},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁retreat"},new string[]{"[L]"},new string[]{"▁heal"},new string[]{"[L]"},new string[]{"▁attack"},new string[]{"[L]"},new string[]{"▁reload"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁health"},new string[]{"▁:"},new string[]{"▁25"},new string[]{"▁%"},new string[]{"▁."},new string[]{"▁heal"},new string[]{"▁available"},new string[]{"▁:"},new string[]{"▁yes"},new string[]{"▁."},new string[]{"▁enemy"},new string[]{"▁visible"},new string[]{"▁:"},new string[]{"▁no"},new string[]{"▁."}},
new int[]{287,128003,4989,262,410,1016,260,287,128007,7886,128007,7533,128007,1815,128007,22452,1263,1263,128002,622,877,977,4639,323,7533,499,877,2489,323,4648,3979,877,363,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{8,10,12,14}, new int[]{128007,128007,128007,128007}),
new Fixture(
"edge_unknown_char", "Device status: ok", "Classify status.",
new string[]{"ok","bad"},
new string[]{},
new string[]{},
new string[]{"device","status",":","","","ok","."}, new string[]{"(","[P]","Classify status.","(","[L]","ok","[L]","bad",")",")","[SEP_TEXT]","device","status",":","","","ok","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁Class","ify","▁status","."},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁ok"},new string[]{"[L]"},new string[]{"▁bad"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁device"},new string[]{"▁status"},new string[]{"▁:"},new string[]{"▁",""},new string[]{"▁",""},new string[]{"▁ok"},new string[]{"▁."}},
new int[]{287,128003,3854,12336,1980,260,287,128007,5522,128007,966,1263,1263,128002,1332,1980,877,507,3,507,3,5522,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{7,9}, new int[]{128007,128007}),
new Fixture(
"edge_unknown_mixed", "Value 🧊 and measured", "Is it valid?",
new string[]{"valid","invalid"},
new string[]{},
new string[]{},
new string[]{"value","🧊","and","","measured","."}, new string[]{"(","[P]","Is it valid?","(","[L]","valid","[L]","invalid",")",")","[SEP_TEXT]","value","🧊","and","","measured","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁I","s","▁it","▁valid","?"},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁valid"},new string[]{"[L]"},new string[]{"▁invalid"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁value"},new string[]{"▁","🧊"},new string[]{"▁and"},new string[]{"▁",""},new string[]{"▁measured"},new string[]{"▁."}},
new int[]{287,128003,273,268,278,3816,302,287,128007,3816,128007,9191,1263,1263,128002,772,507,3,263,507,3,4836,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{8,10}, new int[]{128007,128007}),
new Fixture(
"edge_special_in_task", "Choose [P] the best action", "Choose [P] the best action",
new string[]{"attack","heal"},
new string[]{},
new string[]{},
new string[]{"choose","[","p","]","the","best","action","."}, new string[]{"(","[P]","Choose [P] the best action","(","[L]","attack","[L]","heal",")",")","[SEP_TEXT]","choose","[","p","]","the","best","action","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁Choose","[P]","▁the","▁best","▁action"},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁attack"},new string[]{"[L]"},new string[]{"▁heal"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁choose"},new string[]{"▁["},new string[]{"▁p"},new string[]{"▁]"},new string[]{"▁the"},new string[]{"▁best"},new string[]{"▁action"},new string[]{"▁."}},
new int[]{287,128003,4989,128003,262,410,1016,287,128007,1815,128007,7533,1263,1263,128002,1003,647,845,6311,262,410,1016,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{8,10}, new int[]{128007,128007}),
new Fixture(
"edge_special_in_label", "Enemy close. Health: 50%.", "Choose the best action.",
new string[]{"attack [L]","retreat"},
new string[]{},
new string[]{},
new string[]{"enemy","close",".","health",":","50","%","."}, new string[]{"(","[P]","Choose the best action.","(","[L]","attack [L]","[L]","retreat",")",")","[SEP_TEXT]","enemy","close",".","health",":","50","%","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁Choose","▁the","▁best","▁action","."},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁attack","[L]"},new string[]{"[L]"},new string[]{"▁retreat"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁enemy"},new string[]{"▁close"},new string[]{"▁."},new string[]{"▁health"},new string[]{"▁:"},new string[]{"▁50"},new string[]{"▁%"},new string[]{"▁."}},
new int[]{287,128003,4989,262,410,1016,260,287,128007,1815,128007,128007,7886,1263,1263,128002,4648,828,323,622,877,960,4639,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{8,11}, new int[]{128007,128007}),
new Fixture(
"edge_task_leading_space", "Health: 50%.", " Choose the best action.",
new string[]{"attack","heal"},
new string[]{},
new string[]{},
new string[]{"health",":","50","%","."}, new string[]{"(","[P]"," Choose the best action.","(","[L]","attack","[L]","heal",")",")","[SEP_TEXT]","health",":","50","%","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁Choose","▁the","▁best","▁action","."},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁attack"},new string[]{"[L]"},new string[]{"▁heal"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁health"},new string[]{"▁:"},new string[]{"▁50"},new string[]{"▁%"},new string[]{"▁."}},
new int[]{287,128003,4989,262,410,1016,260,287,128007,1815,128007,7533,1263,1263,128002,622,877,960,4639,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{8,10}, new int[]{128007,128007}),
new Fixture(
"edge_desc_punct", "Night falls; temperature -5C.", "What now?",
new string[]{"build shelter","keep walking"},
new string[]{"build shelter","keep walking"},
new string[]{"wind-break + ground pad; fire optional!","risk: exposure, frost-bite"},
new string[]{"night","falls",";","temperature","-","5c","."}, new string[]{"(","[P]","What now? [DESCRIPTION] build shelter: wind-break + ground pad; fire optional! [DESCRIPTION] keep walking: risk: exposure, frost-bite","(","[L]","build shelter","[L]","keep walking",")",")","[SEP_TEXT]","night","falls",";","temperature","-","5c","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁What","▁now","?","[DESCRIPTION]","▁build","▁shelter",":","▁wind","-","break","▁+","▁ground","▁pad",";","▁fire","▁optional","!","[DESCRIPTION]","▁keep","▁walking",":","▁risk",":","▁exposure",",","▁frost","-","bite"},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁build","▁shelter"},new string[]{"[L]"},new string[]{"▁keep","▁walking"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁night"},new string[]{"▁falls"},new string[]{"▁;"},new string[]{"▁temperature"},new string[]{"▁-"},new string[]{"▁5","c"},new string[]{"▁."}},
new int[]{287,128003,458,394,302,128010,1044,6412,294,2450,271,19051,1566,1353,7061,346,1351,6862,300,128010,548,2220,294,1051,294,3534,261,17911,271,36153,287,128007,1044,6412,128007,548,2220,1263,1263,128002,661,4737,2600,2247,341,456,1207,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{31,34}, new int[]{128007,128007}),
new Fixture(
"edge_empty_desc", "Lost in fog on a ridge.", "What now?",
new string[]{"build shelter","stay put"},
new string[]{"build shelter","stay put"},
new string[]{"","wait for daylight"},
new string[]{"lost","in","fog","on","a","ridge","."}, new string[]{"(","[P]","What now? [DESCRIPTION] build shelter: [DESCRIPTION] stay put: wait for daylight","(","[L]","build shelter","[L]","stay put",")",")","[SEP_TEXT]","lost","in","fog","on","a","ridge","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁What","▁now","?","[DESCRIPTION]","▁build","▁shelter",":","[DESCRIPTION]","▁stay","▁put",":","▁wait","▁for","▁daylight"},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁build","▁shelter"},new string[]{"[L]"},new string[]{"▁stay","▁put"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁lost"},new string[]{"▁in"},new string[]{"▁fog"},new string[]{"▁on"},new string[]{"▁a"},new string[]{"▁ridge"},new string[]{"▁."}},
new int[]{287,128003,458,394,302,128010,1044,6412,294,128010,992,552,294,1495,270,13753,287,128007,1044,6412,128007,992,552,1263,1263,128002,1125,267,11278,277,266,15056,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{17,20}, new int[]{128007,128007}),
new Fixture(
"edge_single_char_label", "The light is red.", "Go or stop?",
new string[]{"g","s"},
new string[]{},
new string[]{},
new string[]{"the","light","is","red","."}, new string[]{"(","[P]","Go or stop?","(","[L]","g","[L]","s",")",")","[SEP_TEXT]","the","light","is","red","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁Go","▁or","▁stop","?"},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁g"},new string[]{"[L]"},new string[]{"▁s"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁the"},new string[]{"▁light"},new string[]{"▁is"},new string[]{"▁red"},new string[]{"▁."}},
new int[]{287,128003,1968,289,1001,302,287,128007,2905,128007,1550,1263,1263,128002,262,731,269,1267,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{7,9}, new int[]{128007,128007}),
new Fixture(
"edge_combining_accent", "Le café est fermé", "Language?",
new string[]{"french","other"},
new string[]{},
new string[]{},
new string[]{"le","cafe","́","est","ferme","́","."}, new string[]{"(","[P]","Language?","(","[L]","french","[L]","other",")",")","[SEP_TEXT]","le","cafe","́","est","ferme","́","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁Language","?"},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁french"},new string[]{"[L]"},new string[]{"▁other"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁le"},new string[]{"▁cafe"},new string[]{"▁́"},new string[]{"▁est"},new string[]{"▁fer","me"},new string[]{"▁́"},new string[]{"▁."}},
new int[]{287,128003,6931,302,287,128007,12771,128007,340,1263,1263,128002,5352,10639,7077,11148,27397,3831,7077,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{5,7}, new int[]{128007,128007}),
new Fixture(
"edge_wordclass_edges", "Signal: Ⅷ and ½ and Ⅰ measured", "Classify signal.",
new string[]{"roman","fraction"},
new string[]{},
new string[]{},
new string[]{"signal",":","ⅷ","and","½","and","ⅰ","measured","."}, new string[]{"(","[P]","Classify signal.","(","[L]","roman","[L]","fraction",")",")","[SEP_TEXT]","signal",":","ⅷ","and","½","and","ⅰ","measured","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁Class","ify","▁signal","."},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁roman"},new string[]{"[L]"},new string[]{"▁fraction"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁signal"},new string[]{"▁:"},new string[]{"▁","ⅷ"},new string[]{"▁and"},new string[]{"▁","½"},new string[]{"▁and"},new string[]{"▁","ⅰ"},new string[]{"▁measured"},new string[]{"▁."}},
new int[]{287,128003,3854,12336,3048,260,287,128007,27757,128007,9004,1263,1263,128002,3048,877,507,3,263,507,3,263,507,3,4836,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{7,9}, new int[]{128007,128007}),
new Fixture(
"edge_url_email", "Contact [email protected] or https://host.tld/x?y=1 now", "Is contact info present?",
new string[]{"yes","no"},
new string[]{},
new string[]{},
new string[]{"contact","[email protected]","or","https://host.tld/x?y=1","now","."}, new string[]{"(","[P]","Is contact info present?","(","[L]","yes","[L]","no",")",")","[SEP_TEXT]","contact","[email protected]","or","https://host.tld/x?y=1","now","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁I","s","▁contact","▁info","▁present","?"},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁yes"},new string[]{"[L]"},new string[]{"▁no"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁contact"},new string[]{"▁admin","@","example",".","com"},new string[]{"▁or"},new string[]{"▁https",":","/","/","host",".","t","ld","/","x","?","y","=","1"},new string[]{"▁now"},new string[]{"▁."}},
new int[]{287,128003,273,268,789,2470,910,302,287,128007,2489,128007,363,1263,1263,128002,789,8113,1683,28748,260,549,289,3597,294,320,320,15333,260,297,17879,320,982,302,608,1510,435,394,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{9,11}, new int[]{128007,128007}),
new Fixture(
"edge_at_mention", "Ping @user_1 about the -refactor_plan now", "Classify message.",
new string[]{"request","chatter"},
new string[]{},
new string[]{},
new string[]{"ping","@user_1","about","the","-","refactor_plan","now","."}, new string[]{"(","[P]","Classify message.","(","[L]","request","[L]","chatter",")",")","[SEP_TEXT]","ping","@user_1","about","the","-","refactor_plan","now","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁Class","ify","▁message","."},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁request"},new string[]{"[L]"},new string[]{"▁chatter"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁ping"},new string[]{"▁@","user","_","1"},new string[]{"▁about"},new string[]{"▁the"},new string[]{"▁-"},new string[]{"▁refactor","_","plan"},new string[]{"▁now"},new string[]{"▁."}},
new int[]{287,128003,3854,12336,1424,260,287,128007,1409,128007,24223,1263,1263,128002,21934,1944,11576,616,435,314,262,341,93660,616,14970,394,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{7,9}, new int[]{128007,128007}),
new Fixture(
"edge_crlf", "alpha\r\nbeta\r\rgammadelta", "Count lines.",
new string[]{"three","four"},
new string[]{},
new string[]{},
new string[]{"alpha","beta","gamma","delta","."}, new string[]{"(","[P]","Count lines.","(","[L]","three","[L]","four",")",")","[SEP_TEXT]","alpha","beta","gamma","delta","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁Count","▁lines","."},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁three"},new string[]{"[L]"},new string[]{"▁four"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁alpha"},new string[]{"▁beta"},new string[]{"▁gamma"},new string[]{"▁delta"},new string[]{"▁."}},
new int[]{287,128003,13812,1728,260,287,128007,475,128007,654,1263,1263,128002,12007,6979,25661,20172,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{6,8}, new int[]{128007,128007}),
new Fixture(
"edge_nbsp", "status ok", "Classify status.",
new string[]{"ok","bad"},
new string[]{},
new string[]{},
new string[]{"status","ok","."}, new string[]{"(","[P]","Classify status.","(","[L]","ok","[L]","bad",")",")","[SEP_TEXT]","status","ok","."}, new string[][]{new string[]{"▁("},new string[]{"[P]"},new string[]{"▁Class","ify","▁status","."},new string[]{"▁("},new string[]{"[L]"},new string[]{"▁ok"},new string[]{"[L]"},new string[]{"▁bad"},new string[]{"▁)"},new string[]{"▁)"},new string[]{"[SEP_TEXT]"},new string[]{"▁status"},new string[]{"▁ok"},new string[]{"▁."}},
new int[]{287,128003,3854,12336,1980,260,287,128007,5522,128007,966,1263,1263,128002,1980,5522,323}, new byte[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, new int[]{7,9}, new int[]{128007,128007}),
};
}