RG-T41 Adding Search

Core/Resgrid.Config/SearchConfig.cs

@@ -1,16 +1,19 @@
 namespace Resgrid.Config
 {
 	/// <summary>
-	/// Shared Lucene.NET search host (Unified Search plan section 9.2, absorbed by RMS-1). Off by default: with the
-	/// host disabled the Records queue keeps running on the indexed projection table and free-text search is
-	/// reported as unavailable rather than silently degrading to LIKE.
+	/// Shared Lucene.NET search host (Unified Search plan section 9.2 / R7). Off by default: with the host disabled the
+	/// Records queue keeps running on the indexed projection table and free-text search is reported as unavailable
+	/// rather than silently degrading to LIKE. Environment keys follow RESGRID__SearchConfig__{Field}.
 	/// </summary>
 	public static class SearchConfig
 	{
 		/// <summary>Master switch for the search host in every process.</summary>
 		public static bool Enabled = false;
 
-		/// <summary>Root directory shared by the API/Web reader and the worker writer (Docker volume).</summary>
+		/// <summary>
+		/// Local root directory for every index this process opens. With the object store enabled this is the per-pod
+		/// cache (an emptyDir in Kubernetes); without it, the single shared directory (Compose bind mount).
+		/// </summary>
 		public static string IndexPath = "/data/search";
 
 		/// <summary>Hard limit on hits a single query may return.</summary>
@@ -24,5 +27,34 @@ public static class SearchConfig
 
 		/// <summary>Maximum departments one maintenance sweep rebuilds before yielding to the next run.</summary>
 		public static int MaxRebuildsPerSweep = 5;
+
+		/// <summary>Closed calls from this many calendar years (including the current one) enter a rebuild; active calls always do.</summary>
+		public static int CallRebuildYears = 3;
+
+		// ---- Object store (RustFS / any S3-compatible endpoint), plan R7. Empty endpoint = disabled. -------------
+
+		/// <summary>S3-compatible endpoint, e.g. https://rustfs.internal:9000. Empty disables publish/pull.</summary>
+		public static string S3Endpoint = "";
+
+		public static string S3AccessKey = "";
+
+		public static string S3SecretKey = "";
+
+		public static string S3Bucket = "";
+
+		public static string S3Region = "us-east-1";
+
+		public static bool S3UseSsl = true;
+
+		public static bool S3ForcePathStyle = true;
+
+		/// <summary>Key prefix inside the bucket; each index lives under {S3Prefix}/{indexName}/.</summary>
+		public static string S3Prefix = "search";
+
+		/// <summary>How often a reader process re-reads the manifest (seconds). The records sweep commits at most once a minute.</summary>
+		public static int ReaderPullSeconds = 30;
+
+		/// <summary>Database publish lease duration (seconds) held by the writer around commit-and-upload.</summary>
+		public static int PublishLeaseSeconds = 120;
 	}
 }

Core/Resgrid.Model/FeatureFlagKeys.cs

@@ -84,5 +84,8 @@ public static class FeatureFlagKeys
 
 		/// <summary>RMS-6 records analytics: response-performance, workload, executive, accreditation and community-risk dashboards over finalized Records. Depends on Records.System. Seeded off by M0187.</summary>
 		public const string RecordsAnalytics = "Records.Analytics";
+
+		/// <summary>Unified Search: cross-entity search over the global Lucene index plus the system-functionality command palette. Requires SearchConfig.Enabled in every process. Seeded off by M0208 (registry §4F).</summary>
+		public const string SearchUnified = "Search.Unified";
 	}
 }

Core/Resgrid.Model/Providers/ISearchIndexStore.cs

@@ -0,0 +1,38 @@
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using Resgrid.Model.Search;
+
+namespace Resgrid.Model.Providers
+{
+	/// <summary>
+	/// The durable and distribution layer behind the per-pod Lucene caches (Unified Search plan R7). The object store
+	/// is never the live Lucene directory: the single writer uploads immutable segment files after each commit and
+	/// then swaps the manifest with a conditional PUT; readers poll the manifest and pull missing files into their
+	/// local directory. An implementation whose <see cref="Enabled"/> is false makes every host behave as a plain
+	/// local-directory host (single-host Compose).
+	/// </summary>
+	public interface ISearchIndexStore
+	{
+		bool Enabled { get; }
+
+		/// <summary>The current manifest, or null when the prefix has never been published.</summary>
+		Task<SearchIndexManifest> GetManifestAsync(string indexName, CancellationToken cancellationToken = default);
+
+		/// <summary>Object names (relative to the index prefix) currently in the store, excluding the manifest.</summary>
+		Task<HashSet<string>> ListFilesAsync(string indexName, CancellationToken cancellationToken = default);
+
+		Task UploadFileAsync(string indexName, string fileName, string localPath, CancellationToken cancellationToken = default);
+
+		Task DownloadFileAsync(string indexName, string fileName, string localPath, CancellationToken cancellationToken = default);
+
+		Task DeleteFilesAsync(string indexName, IEnumerable<string> fileNames, CancellationToken cancellationToken = default);
+
+		/// <summary>
+		/// Writes the manifest conditionally: <paramref name="expectedETag"/> null means "must not exist yet"; otherwise
+		/// the PUT carries If-Match. Throws <see cref="SearchIndexManifestConflictException"/> when the precondition
+		/// fails. Returns the manifest with its new ETag.
+		/// </summary>
+		Task<SearchIndexManifest> PutManifestAsync(string indexName, SearchIndexManifest manifest, string expectedETag, CancellationToken cancellationToken = default);
+	}
+}

Core/Resgrid.Model/Repositories/ISearchRepositories.cs

@@ -0,0 +1,50 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using Resgrid.Model.Search;
+
+namespace Resgrid.Model.Repositories
+{
+	public interface ISearchProjectionsRepository : IRepository<SearchProjection>
+	{
+		Task<SearchProjection> GetAsync(int departmentId, string entityType, string entityId);
+
+		/// <summary>Insert or update by (DepartmentId, EntityType, EntityId); bumps RowVersion and ModifiedOn, clears DeletedOn.</summary>
+		Task<SearchProjection> UpsertAsync(SearchProjection projection, CancellationToken cancellationToken = default);
+
+		/// <summary>Soft-deletes one row; returns false when it did not exist.</summary>
+		Task<bool> SoftDeleteAsync(int departmentId, string entityType, string entityId, CancellationToken cancellationToken = default);
+
+		/// <summary>Soft-deletes live rows of one family that were not touched since <paramref name="notTouchedSince"/> (rebuild reconciliation).</summary>
+		Task<int> SoftDeleteStaleAsync(int departmentId, string entityType, DateTime notTouchedSince, CancellationToken cancellationToken = default);
+
+		/// <summary>Rows (including soft-deleted) after the (ModifiedOn, id) cursor, oldest first — the catch-up feed.</summary>
+		Task<IEnumerable<SearchProjection>> GetModifiedSinceAsync(int departmentId, DateTime? since, int take, string sinceId = null);
+
+		/// <summary>Live rows for a department, paged, for a full index rebuild.</summary>
+		Task<IEnumerable<SearchProjection>> GetLivePageAsync(int departmentId, int skip, int take);
+
+		/// <summary>Live rows by projection id, for post-retrieval loading of hits.</summary>
+		Task<IEnumerable<SearchProjection>> GetByIdsAsync(int departmentId, IEnumerable<string> projectionIds);
+
+		Task<int> HardDeleteDepartmentAsync(int departmentId, CancellationToken cancellationToken = default);
+	}
+
+	public interface ISearchIndexStatesRepository : IRepository<SearchIndexState>
+	{
+		Task<SearchIndexState> GetAsync(string indexName, int departmentId);
+
+		Task<IEnumerable<SearchIndexState>> GetAllForIndexAsync(string indexName);
+	}
+
+	public interface ISearchIndexLeasesRepository : IRepository<SearchIndexLease>
+	{
+		/// <summary>Takes or renews the lease when it is free, expired, or already held by <paramref name="owner"/>.</summary>
+		Task<bool> TryAcquireAsync(string indexName, string owner, TimeSpan duration, DateTime utcNow, CancellationToken cancellationToken = default);
+
+		Task ReleaseAsync(string indexName, string owner, CancellationToken cancellationToken = default);
+
+		Task RecordPublishedAsync(string indexName, string owner, string revision, DateTime utcNow, CancellationToken cancellationToken = default);
+	}
+}

Core/Resgrid.Model/Search/SearchContracts.cs

@@ -0,0 +1,121 @@
+using System;
+using System.Collections.Generic;
+
+namespace Resgrid.Model.Search
+{
+	/// <summary>Index names hosted by the shared Lucene host (Unified Search plan R1/R4).</summary>
+	public static class SearchIndexNames
+	{
+		public const string Records = "records";
+		public const string Global = "global";
+	}
+
+	/// <summary>Entity families the global index and the unified endpoint know about. Stable strings: clients switch on them.</summary>
+	public static class SearchEntityTypes
+	{
+		public const string Call = "Call";
+		public const string Unit = "Unit";
+		public const string Personnel = "Personnel";
+		public const string Contact = "Contact";
+		public const string Message = "Message";
+		public const string Document = "Document";
+		public const string Note = "Note";
+		public const string Record = "Record";
+		public const string Action = "Action";
+
+		public static readonly IReadOnlyList<string> Indexed = new[] { Call, Unit, Personnel, Contact, Message, Document, Note };
+	}
+
+	/// <summary>Index state values stored on SearchIndexState.State (same numbering as the RMS records index).</summary>
+	public enum SearchIndexBuildState
+	{
+		Unknown = 0,
+		Ready = 1,
+		Rebuilding = 2,
+		Failed = 3,
+		RebuildRequested = 4
+	}
+
+	/// <summary>
+	/// The generation key of the global index: (schemaVersion, protectedCatalogVersion, policyEpoch). Any change rebuilds
+	/// the department's documents so an enrollment or a permission-policy change can never serve stale hits (plan R2.3, R7).
+	/// </summary>
+	public static class GlobalSearchGeneration
+	{
+		/// <summary>Bump when GlobalSearchDocumentBuilder or the projection allowlist changes.</summary>
+		public const int SchemaVersion = 1;
+
+		public static string Compute(int protectedCatalogVersion, long policyEpoch)
+		{
+			return $"{SchemaVersion}.{protectedCatalogVersion}.{policyEpoch}";
+		}
+	}
+
+	/// <summary>The manifest a writer publishes to the object store after every commit and readers poll (plan R7).</summary>
+	public class SearchIndexManifest
+	{
+		public string IndexName { get; set; }
+
+		/// <summary>Opaque id of this publish; readers compare it to decide whether to sync.</summary>
+		public string Revision { get; set; }
+
+		/// <summary>Lucene segments_N generation of the published commit.</summary>
+		public long SegmentsGeneration { get; set; }
+
+		public int SchemaVersion { get; set; }
+
+		public List<SearchIndexManifestFile> Files { get; set; } = new List<SearchIndexManifestFile>();
+
+		public DateTime PublishedOnUtc { get; set; }
+
+		public string PublishedBy { get; set; }
+
+		/// <summary>Transport ETag returned by the store; used for the conditional PUT on the next publish. Not serialized.</summary>
+		[Newtonsoft.Json.JsonIgnore]
+		public string ETag { get; set; }
+	}
+
+	public class SearchIndexManifestFile
+	{
+		public string Name { get; set; }
+		public long Length { get; set; }
+	}
+
+	/// <summary>A conditional manifest PUT lost: another writer published since this process last read the manifest.</summary>
+	public class SearchIndexManifestConflictException : Exception
+	{
+		public SearchIndexManifestConflictException(string indexName, string message)
+			: base(message)
+		{
+			IndexName = indexName;
+		}
+
+		public string IndexName { get; }
+	}
+
+	/// <summary>Outcome of one maintenance sweep of the global index (worker 70).</summary>
+	public class SearchIndexSweepResult
+	{
+		public int DepartmentsChecked { get; set; }
+		public int DepartmentsRebuilt { get; set; }
+		public int ProjectionsRebuilt { get; set; }
+		public int DocumentsIndexed { get; set; }
+		public int DocumentsDeleted { get; set; }
+		public int Errors { get; set; }
+		public bool Skipped { get; set; }
+		public string Message { get; set; }
+	}
+
+	public class SearchIndexHealth
+	{
+		public string IndexName { get; set; }
+		public bool Enabled { get; set; }
+		public bool Online { get; set; }
+		public string IndexPath { get; set; }
+		public int DocumentCount { get; set; }
+		public bool StoreEnabled { get; set; }
+		public string LastSyncedRevision { get; set; }
+		public DateTime? LastSyncedOnUtc { get; set; }
+		public string Error { get; set; }
+	}
+}