Editor/Reference/ReferenceIndexService.cs
#nullable enable
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Threading;
namespace Editor.Mcp
{
public sealed class ReferenceSearchQuery
{
public string Query { get; init; } = "";
public string? Project { get; init; }
public string? Package { get; init; }
public string? PathPrefix { get; init; }
public string? Extension { get; init; }
public string? Source { get; init; }
public int Limit { get; init; } = 20;
public int Offset { get; init; }
public string Sort { get; init; } = "relevance";
}
public sealed class ReferenceSearchItem
{
public string Id { get; init; } = "";
public string Project { get; init; } = "";
public string? Package { get; init; }
public string Title { get; init; } = "";
public string Type { get; init; } = "";
public string Source { get; init; } = "local";
public string Path { get; init; } = "";
public string[] MatchedFields { get; init; } = Array.Empty<string>();
public string Snippet { get; init; } = "";
public double Score { get; init; }
}
public sealed class ReferenceSearchPage
{
public string Query { get; init; } = "";
public int Total { get; init; }
public int Offset { get; init; }
public int Limit { get; init; }
public bool HasMore { get; init; }
public string CatalogVersion { get; init; } = "";
public IReadOnlyList<ReferenceSearchItem> Items { get; init; } = Array.Empty<ReferenceSearchItem>();
}
public sealed class ReferenceIndexStatus
{
public string Root { get; init; } = "";
public string IndexPath { get; init; } = "";
public string CatalogVersion { get; init; } = "";
public int Documents { get; init; }
public int Terms { get; init; }
public DateTimeOffset BuiltAt { get; init; }
public bool Rebuilt { get; init; }
}
internal sealed class ReferenceIndexDocument
{
public string Id { get; set; } = "";
public string Project { get; set; } = "";
public string? Package { get; set; }
public string Title { get; set; } = "";
public string Type { get; set; } = "";
public string Source { get; set; } = "local";
public string Path { get; set; } = "";
public string DisplayPath { get; set; } = "";
public string Text { get; set; } = "";
}
internal sealed class ReferenceIndexFile
{
public int Version { get; set; } = 1;
public string Root { get; set; } = "";
public string CatalogVersion { get; set; } = "";
public DateTimeOffset BuiltAt { get; set; }
public List<ReferenceIndexDocument> Documents { get; set; } = new();
public Dictionary<string, List<int>> Postings { get; set; } = new(StringComparer.OrdinalIgnoreCase);
}
public static class ReferenceIndexService
{
private static readonly object Sync = new();
private static ReferenceIndexFile? _index;
private static string? _indexPath;
public static string DefaultRoot => Environment.GetEnvironmentVariable("SBOX_REFERENCE_ROOT")
?? @"F:\ySBOX\xSboxGameProjectReference";
public static string GetIndexPath(string projectRoot) => Path.Combine(projectRoot, ".sbox", "reference-index.v1.json");
public static ReferenceIndexStatus Rebuild(bool force = false, string? projectRoot = null, string? referenceRoot = null)
{
projectRoot ??= Sandbox.Project.Current?.GetRootPath() ?? Environment.CurrentDirectory;
referenceRoot ??= DefaultRoot;
var existingPath = GetIndexPath(projectRoot);
if (!force)
{
lock (Sync)
if (_index is not null && string.Equals(_indexPath, existingPath, StringComparison.OrdinalIgnoreCase) && string.Equals(_index.Root, referenceRoot, StringComparison.OrdinalIgnoreCase)) return ToStatus(_index, existingPath, false);
if (File.Exists(existingPath))
{
try
{
var existing = JsonSerializer.Deserialize<ReferenceIndexFile>(File.ReadAllText(existingPath));
if (existing is not null && existing.Documents.Count > 0 && string.Equals(existing.Root, referenceRoot, StringComparison.OrdinalIgnoreCase))
{
lock (Sync) { _index = existing; _indexPath = existingPath; }
return ToStatus(existing, existingPath, false);
}
}
catch { }
}
}
var catalogPath = Path.Combine(referenceRoot, "reference", "catalog.json");
if (!File.Exists(catalogPath)) throw new FileNotFoundException("Reference catalog was not found.", catalogPath);
var documents = new List<ReferenceIndexDocument>();
var catalogVersion = ComputeHash(catalogPath);
using (var stream = File.OpenRead(catalogPath))
using (var catalog = JsonDocument.Parse(stream))
{
if (catalog.RootElement.TryGetProperty("projects", out var projects) && projects.ValueKind == JsonValueKind.Array)
{
foreach (var project in projects.EnumerateArray())
{
var key = GetString(project, "project_key") ?? "unknown";
var metadata = project.TryGetProperty("metadata", out var m) ? m : default;
var title = GetString(metadata, "title") ?? key;
var type = GetString(metadata, "type") ?? "project";
var package = GetString(metadata, "package_id") ?? key;
var symbols = project.TryGetProperty("key_symbols", out var symbolsElement) ? symbolsElement.ToString() : "";
var systems = project.TryGetProperty("systems", out var systemsElement) ? systemsElement.ToString() : "";
documents.Add(new ReferenceIndexDocument
{
Id = "project:" + key,
Project = key,
Package = package,
Title = title,
Type = type,
Source = "reference-catalog",
Path = Path.Combine(referenceRoot, "projects", key),
DisplayPath = $"projects/{key}",
Text = string.Join(" ", key, title, type, package, symbols, systems)
});
}
}
}
AddDocumentationFiles(documents, Path.Combine(referenceRoot, "reference"), "reference-notes");
AddDocumentationFiles(documents, Path.Combine(Path.GetDirectoryName(referenceRoot) ?? referenceRoot, "1AISbox", "Special-Documentation"), "special-documentation");
AddSourceFiles(documents, Path.Combine(Path.GetDirectoryName(referenceRoot) ?? referenceRoot, "1AISbox", "Hearth-MCP"), "project-notes", 200);
var index = new ReferenceIndexFile
{
Root = referenceRoot,
CatalogVersion = catalogVersion,
BuiltAt = DateTimeOffset.UtcNow,
Documents = documents
};
BuildPostings(index);
var path = GetIndexPath(projectRoot);
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
File.WriteAllText(path, JsonSerializer.Serialize(index, new JsonSerializerOptions { WriteIndented = false }));
lock (Sync) { _index = index; _indexPath = path; }
return ToStatus(index, path, true);
}
public static ReferenceIndexStatus Ensure(string? projectRoot = null, string? referenceRoot = null)
{
projectRoot ??= Sandbox.Project.Current?.GetRootPath() ?? Environment.CurrentDirectory;
referenceRoot ??= DefaultRoot;
var path = GetIndexPath(projectRoot);
lock (Sync)
{
if (_index is not null && string.Equals(_indexPath, path, StringComparison.OrdinalIgnoreCase)) return ToStatus(_index, path, false);
if (File.Exists(path))
{
try
{
_index = JsonSerializer.Deserialize<ReferenceIndexFile>(File.ReadAllText(path));
_indexPath = path;
if (_index is not null && _index.Documents.Count > 0) return ToStatus(_index, path, false);
}
catch { _index = null; }
}
}
return Rebuild(true, projectRoot, referenceRoot);
}
public static ReferenceSearchPage Search(ReferenceSearchQuery query, string? projectRoot = null, string? referenceRoot = null)
{
var status = Ensure(projectRoot, referenceRoot);
ReferenceIndexFile index;
lock (Sync) { index = _index!; }
var normalized = (query.Query ?? "").Trim();
var tokens = Tokenize(normalized).ToArray();
IEnumerable<ReferenceIndexDocument> candidates = index.Documents;
if (tokens.Length > 0)
{
var ids = new HashSet<int>();
foreach (var token in tokens)
if (index.Postings.TryGetValue(token, out var posting)) foreach (var id in posting) ids.Add(id);
candidates = ids.Select(id => index.Documents[id]);
}
candidates = candidates.Where(d => MatchFilter(d, query));
var ranked = candidates.Select(d => Score(d, normalized, tokens)).Where(x => tokens.Length == 0 || x.Score > 0);
if (!string.Equals(query.Sort, "path", StringComparison.OrdinalIgnoreCase)) ranked = ranked.OrderByDescending(x => x.Score).ThenBy(x => x.Document.Path, StringComparer.OrdinalIgnoreCase);
else ranked = ranked.OrderBy(x => x.Document.Path, StringComparer.OrdinalIgnoreCase);
var all = ranked.ToList();
var limit = Math.Clamp(query.Limit <= 0 ? 20 : query.Limit, 1, 100);
var offset = Math.Max(0, query.Offset);
var items = all.Skip(offset).Take(limit).Select(x => ToItem(x.Document, normalized, x.Score)).ToArray();
return new ReferenceSearchPage { Query = normalized, Total = all.Count, Offset = offset, Limit = limit, HasMore = offset + items.Length < all.Count, CatalogVersion = status.CatalogVersion, Items = items };
}
public static ReferenceSearchItem? Get(string id, string? projectRoot = null, string? referenceRoot = null)
{
Ensure(projectRoot, referenceRoot);
lock (Sync) return _index?.Documents.FirstOrDefault(d => d.Id.Equals(id, StringComparison.OrdinalIgnoreCase)) is { } d ? ToItem(d, "", 1) : null;
}
private static void AddDocumentationFiles(List<ReferenceIndexDocument> docs, string root, string source)
{
if (!Directory.Exists(root)) return;
foreach (var file in SafeFiles(root, new[] { ".md", ".yml", ".yaml", ".json" }).Take(5000))
{
var text = SafeRead(file, 65536);
if (text is null) continue;
docs.Add(new ReferenceIndexDocument { Id = source + ":" + StableId(file), Project = source, Title = Path.GetFileNameWithoutExtension(file), Type = Path.GetExtension(file), Source = source, Path = file, DisplayPath = DisplayPath(root, file, source), Text = text });
}
}
private static void AddSourceFiles(List<ReferenceIndexDocument> docs, string root, string source, int maxPerFolder)
{
if (!Directory.Exists(root)) return;
foreach (var file in SafeFiles(root, new[] { ".cs", ".razor", ".sbproj", ".md", ".scss", ".shader", ".hlsl", ".vmat", ".scene", ".postprocess", ".prefab" }).Take(maxPerFolder))
{
var text = SafeRead(file, 32768);
if (text is null) continue;
docs.Add(new ReferenceIndexDocument { Id = source + ":" + StableId(file), Project = "Hearth-MCP", Title = Path.GetFileName(file), Type = Path.GetExtension(file), Source = source, Path = file, DisplayPath = DisplayPath(root, file, source), Text = text });
}
}
private static IEnumerable<string> SafeFiles(string root, string[] extensions)
{
try { return Directory.EnumerateFiles(root, "*", SearchOption.AllDirectories).Where(f => extensions.Contains(Path.GetExtension(f), StringComparer.OrdinalIgnoreCase) && !f.Contains("node_modules", StringComparison.OrdinalIgnoreCase) && !f.Contains(".git", StringComparison.OrdinalIgnoreCase) && !IsSensitivePath(f)); }
catch { return Array.Empty<string>(); }
}
private static string? SafeRead(string path, int max)
{
try { using var reader = new StreamReader(path); var buffer = new char[max]; var count = reader.ReadBlock(buffer, 0, buffer.Length); return new string(buffer, 0, count); } catch { return null; }
}
private static void BuildPostings(ReferenceIndexFile index)
{
for (var i = 0; i < index.Documents.Count; i++) foreach (var token in Tokenize(index.Documents[i].Text).Distinct(StringComparer.OrdinalIgnoreCase).Take(500))
if (!index.Postings.TryGetValue(token, out var list)) index.Postings[token] = new List<int> { i }; else list.Add(i);
}
private static bool MatchFilter(ReferenceIndexDocument d, ReferenceSearchQuery q) =>
(string.IsNullOrWhiteSpace(q.Project) || d.Project.Contains(q.Project, StringComparison.OrdinalIgnoreCase)) &&
(string.IsNullOrWhiteSpace(q.Package) || (d.Package?.Contains(q.Package, StringComparison.OrdinalIgnoreCase) ?? false)) &&
(string.IsNullOrWhiteSpace(q.PathPrefix) || d.Path.StartsWith(q.PathPrefix, StringComparison.OrdinalIgnoreCase)) &&
(string.IsNullOrWhiteSpace(q.Extension) || d.Path.EndsWith(q.Extension, StringComparison.OrdinalIgnoreCase)) &&
(string.IsNullOrWhiteSpace(q.Source) || d.Source.Equals(q.Source, StringComparison.OrdinalIgnoreCase));
private static (ReferenceIndexDocument Document, double Score) Score(ReferenceIndexDocument d, string query, string[] tokens)
{
if (tokens.Length == 0) return (d, 0);
var title = d.Title.ToLowerInvariant(); var path = d.Path.ToLowerInvariant(); var text = d.Text.ToLowerInvariant(); var score = 0d; var fields = new List<string>();
foreach (var token in tokens) { if (title.Contains(token)) { score += 8; fields.Add("title"); } if (path.Contains(token)) { score += 5; fields.Add("path"); } if (text.Contains(token)) { score += 1; fields.Add("content"); } }
if (title.Contains(query.ToLowerInvariant())) score += 10;
return (d, score);
}
private static ReferenceSearchItem ToItem(ReferenceIndexDocument d, string query, double score) => new() { Id = d.Id, Project = d.Project, Package = d.Package, Title = d.Title, Type = d.Type, Source = d.Source, Path = string.IsNullOrWhiteSpace(d.DisplayPath) ? $"{d.Source}/{Path.GetFileName(d.Path)}" : d.DisplayPath.Replace('\\', '/'), Score = score, MatchedFields = Tokenize(query).Where(t => d.Text.Contains(t, StringComparison.OrdinalIgnoreCase) || d.Title.Contains(t, StringComparison.OrdinalIgnoreCase)).Distinct().ToArray(), Snippet = ToolGuard.Clip(d.Text.Replace('\n', ' ').Replace('\r', ' '), 320) };
private static ReferenceIndexStatus ToStatus(ReferenceIndexFile i, string path, bool rebuilt) => new() { Root = "[configured-reference-root]", IndexPath = ".sbox/reference-index.v1.json", CatalogVersion = i.CatalogVersion, Documents = i.Documents.Count, Terms = i.Postings.Count, BuiltAt = i.BuiltAt, Rebuilt = rebuilt };
private static string DisplayPath(string root, string file, string source)
{
var relative = Path.GetRelativePath(root, file).Replace('\\', '/');
return source.Equals("reference-notes", StringComparison.OrdinalIgnoreCase) ? $"reference/{relative}" : $"{source}/{relative}";
}
private static bool IsSensitivePath(string path)
{
var fileName = Path.GetFileName(path);
var parts = path.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
var extension = Path.GetExtension(path);
return parts.Any(part => part.Equals(".secrets", StringComparison.OrdinalIgnoreCase) || part.Equals("bin", StringComparison.OrdinalIgnoreCase) || part.Equals("obj", StringComparison.OrdinalIgnoreCase) || part.Equals(".vs", StringComparison.OrdinalIgnoreCase))
|| fileName.Equals(".env", StringComparison.OrdinalIgnoreCase)
|| fileName.StartsWith(".env.", StringComparison.OrdinalIgnoreCase)
|| fileName.Contains("secret", StringComparison.OrdinalIgnoreCase)
|| extension is ".clixml" or ".pem" or ".key" or ".pfx" or ".p12";
}
private static IEnumerable<string> Tokenize(string text) => (text ?? "").Split(new[] { ' ', '\t', '\r', '\n', '/', '\\', '.', ',', ':', ';', '(', ')', '[', ']', '{', '}', '<', '>', '"', '\'', '-', '_' }, StringSplitOptions.RemoveEmptyEntries).Where(x => x.Length >= 2).Select(x => x.ToLowerInvariant());
private static string GetString(JsonElement e, string property) => e.ValueKind == JsonValueKind.Object && e.TryGetProperty(property, out var p) ? p.GetString() ?? "" : "";
private static string StableId(string value) { using var sha = SHA256.Create(); return Convert.ToHexString(sha.ComputeHash(Encoding.UTF8.GetBytes(value))).Substring(0, 16).ToLowerInvariant(); }
private static string ComputeHash(string path) { using var sha = SHA256.Create(); using var stream = File.OpenRead(path); return Convert.ToHexString(sha.ComputeHash(stream)).ToLowerInvariant(); }
}
}