diff --git a/Core/Resgrid.Config/SearchConfig.cs b/Core/Resgrid.Config/SearchConfig.cs index ad85f1e31..f3804a476 100644 --- a/Core/Resgrid.Config/SearchConfig.cs +++ b/Core/Resgrid.Config/SearchConfig.cs @@ -1,16 +1,19 @@ namespace Resgrid.Config { /// - /// Shared Lucene.NET search host (Unified Search plan section 9.2, absorbed by RMS-1). Off by default: with the - /// host disabled the Records queue keeps running on the indexed projection table and free-text search is - /// reported as unavailable rather than silently degrading to LIKE. + /// Shared Lucene.NET search host (Unified Search plan section 9.2 / R7). Off by default: with the host disabled the + /// Records queue keeps running on the indexed projection table and free-text search is reported as unavailable + /// rather than silently degrading to LIKE. Environment keys follow RESGRID__SearchConfig__{Field}. /// public static class SearchConfig { /// Master switch for the search host in every process. public static bool Enabled = false; - /// Root directory shared by the API/Web reader and the worker writer (Docker volume). + /// + /// Local root directory for every index this process opens. With the object store enabled this is the per-pod + /// cache (an emptyDir in Kubernetes); without it, the single shared directory (Compose bind mount). + /// public static string IndexPath = "/data/search"; /// Hard limit on hits a single query may return. @@ -24,5 +27,34 @@ public static class SearchConfig /// Maximum departments one maintenance sweep rebuilds before yielding to the next run. public static int MaxRebuildsPerSweep = 5; + + /// Closed calls from this many calendar years (including the current one) enter a rebuild; active calls always do. + public static int CallRebuildYears = 3; + + // ---- Object store (RustFS / any S3-compatible endpoint), plan R7. Empty endpoint = disabled. ------------- + + /// S3-compatible endpoint, e.g. https://rustfs.internal:9000. Empty disables publish/pull. + public static string S3Endpoint = ""; + + public static string S3AccessKey = ""; + + public static string S3SecretKey = ""; + + public static string S3Bucket = ""; + + public static string S3Region = "us-east-1"; + + public static bool S3UseSsl = true; + + public static bool S3ForcePathStyle = true; + + /// Key prefix inside the bucket; each index lives under {S3Prefix}/{indexName}/. + public static string S3Prefix = "search"; + + /// How often a reader process re-reads the manifest (seconds). The records sweep commits at most once a minute. + public static int ReaderPullSeconds = 30; + + /// Database publish lease duration (seconds) held by the writer around commit-and-upload. + public static int PublishLeaseSeconds = 120; } } diff --git a/Core/Resgrid.Model/FeatureFlagKeys.cs b/Core/Resgrid.Model/FeatureFlagKeys.cs index 447fb118f..acd2d689a 100644 --- a/Core/Resgrid.Model/FeatureFlagKeys.cs +++ b/Core/Resgrid.Model/FeatureFlagKeys.cs @@ -84,5 +84,8 @@ public static class FeatureFlagKeys /// RMS-6 records analytics: response-performance, workload, executive, accreditation and community-risk dashboards over finalized Records. Depends on Records.System. Seeded off by M0187. public const string RecordsAnalytics = "Records.Analytics"; + + /// Unified Search: cross-entity search over the global Lucene index plus the system-functionality command palette. Requires SearchConfig.Enabled in every process. Seeded off by M0208 (registry §4F). + public const string SearchUnified = "Search.Unified"; } } diff --git a/Core/Resgrid.Model/Providers/ISearchIndexStore.cs b/Core/Resgrid.Model/Providers/ISearchIndexStore.cs new file mode 100644 index 000000000..e69cc8023 --- /dev/null +++ b/Core/Resgrid.Model/Providers/ISearchIndexStore.cs @@ -0,0 +1,38 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Resgrid.Model.Search; + +namespace Resgrid.Model.Providers +{ + /// + /// The durable and distribution layer behind the per-pod Lucene caches (Unified Search plan R7). The object store + /// is never the live Lucene directory: the single writer uploads immutable segment files after each commit and + /// then swaps the manifest with a conditional PUT; readers poll the manifest and pull missing files into their + /// local directory. An implementation whose is false makes every host behave as a plain + /// local-directory host (single-host Compose). + /// + public interface ISearchIndexStore + { + bool Enabled { get; } + + /// The current manifest, or null when the prefix has never been published. + Task GetManifestAsync(string indexName, CancellationToken cancellationToken = default); + + /// Object names (relative to the index prefix) currently in the store, excluding the manifest. + Task> ListFilesAsync(string indexName, CancellationToken cancellationToken = default); + + Task UploadFileAsync(string indexName, string fileName, string localPath, CancellationToken cancellationToken = default); + + Task DownloadFileAsync(string indexName, string fileName, string localPath, CancellationToken cancellationToken = default); + + Task DeleteFilesAsync(string indexName, IEnumerable fileNames, CancellationToken cancellationToken = default); + + /// + /// Writes the manifest conditionally: null means "must not exist yet"; otherwise + /// the PUT carries If-Match. Throws when the precondition + /// fails. Returns the manifest with its new ETag. + /// + Task PutManifestAsync(string indexName, SearchIndexManifest manifest, string expectedETag, CancellationToken cancellationToken = default); + } +} diff --git a/Core/Resgrid.Model/Repositories/ISearchRepositories.cs b/Core/Resgrid.Model/Repositories/ISearchRepositories.cs new file mode 100644 index 000000000..c8c7cb3fd --- /dev/null +++ b/Core/Resgrid.Model/Repositories/ISearchRepositories.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Resgrid.Model.Search; + +namespace Resgrid.Model.Repositories +{ + public interface ISearchProjectionsRepository : IRepository + { + Task GetAsync(int departmentId, string entityType, string entityId); + + /// Insert or update by (DepartmentId, EntityType, EntityId); bumps RowVersion and ModifiedOn, clears DeletedOn. + Task UpsertAsync(SearchProjection projection, CancellationToken cancellationToken = default); + + /// Soft-deletes one row; returns false when it did not exist. + Task SoftDeleteAsync(int departmentId, string entityType, string entityId, CancellationToken cancellationToken = default); + + /// Soft-deletes live rows of one family that were not touched since (rebuild reconciliation). + Task SoftDeleteStaleAsync(int departmentId, string entityType, DateTime notTouchedSince, CancellationToken cancellationToken = default); + + /// Rows (including soft-deleted) after the (ModifiedOn, id) cursor, oldest first — the catch-up feed. + Task> GetModifiedSinceAsync(int departmentId, DateTime? since, int take, string sinceId = null); + + /// Live rows for a department, paged, for a full index rebuild. + Task> GetLivePageAsync(int departmentId, int skip, int take); + + /// Live rows by projection id, for post-retrieval loading of hits. + Task> GetByIdsAsync(int departmentId, IEnumerable projectionIds); + + Task HardDeleteDepartmentAsync(int departmentId, CancellationToken cancellationToken = default); + } + + public interface ISearchIndexStatesRepository : IRepository + { + Task GetAsync(string indexName, int departmentId); + + Task> GetAllForIndexAsync(string indexName); + } + + public interface ISearchIndexLeasesRepository : IRepository + { + /// Takes or renews the lease when it is free, expired, or already held by . + Task TryAcquireAsync(string indexName, string owner, TimeSpan duration, DateTime utcNow, CancellationToken cancellationToken = default); + + Task ReleaseAsync(string indexName, string owner, CancellationToken cancellationToken = default); + + Task RecordPublishedAsync(string indexName, string owner, string revision, DateTime utcNow, CancellationToken cancellationToken = default); + } +} diff --git a/Core/Resgrid.Model/Search/SearchContracts.cs b/Core/Resgrid.Model/Search/SearchContracts.cs new file mode 100644 index 000000000..0a1123693 --- /dev/null +++ b/Core/Resgrid.Model/Search/SearchContracts.cs @@ -0,0 +1,121 @@ +using System; +using System.Collections.Generic; + +namespace Resgrid.Model.Search +{ + /// Index names hosted by the shared Lucene host (Unified Search plan R1/R4). + public static class SearchIndexNames + { + public const string Records = "records"; + public const string Global = "global"; + } + + /// Entity families the global index and the unified endpoint know about. Stable strings: clients switch on them. + public static class SearchEntityTypes + { + public const string Call = "Call"; + public const string Unit = "Unit"; + public const string Personnel = "Personnel"; + public const string Contact = "Contact"; + public const string Message = "Message"; + public const string Document = "Document"; + public const string Note = "Note"; + public const string Record = "Record"; + public const string Action = "Action"; + + public static readonly IReadOnlyList Indexed = new[] { Call, Unit, Personnel, Contact, Message, Document, Note }; + } + + /// Index state values stored on SearchIndexState.State (same numbering as the RMS records index). + public enum SearchIndexBuildState + { + Unknown = 0, + Ready = 1, + Rebuilding = 2, + Failed = 3, + RebuildRequested = 4 + } + + /// + /// The generation key of the global index: (schemaVersion, protectedCatalogVersion, policyEpoch). Any change rebuilds + /// the department's documents so an enrollment or a permission-policy change can never serve stale hits (plan R2.3, R7). + /// + public static class GlobalSearchGeneration + { + /// Bump when GlobalSearchDocumentBuilder or the projection allowlist changes. + public const int SchemaVersion = 1; + + public static string Compute(int protectedCatalogVersion, long policyEpoch) + { + return $"{SchemaVersion}.{protectedCatalogVersion}.{policyEpoch}"; + } + } + + /// The manifest a writer publishes to the object store after every commit and readers poll (plan R7). + public class SearchIndexManifest + { + public string IndexName { get; set; } + + /// Opaque id of this publish; readers compare it to decide whether to sync. + public string Revision { get; set; } + + /// Lucene segments_N generation of the published commit. + public long SegmentsGeneration { get; set; } + + public int SchemaVersion { get; set; } + + public List Files { get; set; } = new List(); + + public DateTime PublishedOnUtc { get; set; } + + public string PublishedBy { get; set; } + + /// Transport ETag returned by the store; used for the conditional PUT on the next publish. Not serialized. + [Newtonsoft.Json.JsonIgnore] + public string ETag { get; set; } + } + + public class SearchIndexManifestFile + { + public string Name { get; set; } + public long Length { get; set; } + } + + /// A conditional manifest PUT lost: another writer published since this process last read the manifest. + public class SearchIndexManifestConflictException : Exception + { + public SearchIndexManifestConflictException(string indexName, string message) + : base(message) + { + IndexName = indexName; + } + + public string IndexName { get; } + } + + /// Outcome of one maintenance sweep of the global index (worker 70). + public class SearchIndexSweepResult + { + public int DepartmentsChecked { get; set; } + public int DepartmentsRebuilt { get; set; } + public int ProjectionsRebuilt { get; set; } + public int DocumentsIndexed { get; set; } + public int DocumentsDeleted { get; set; } + public int Errors { get; set; } + public bool Skipped { get; set; } + public string Message { get; set; } + } + + public class SearchIndexHealth + { + public string IndexName { get; set; } + public bool Enabled { get; set; } + public bool Online { get; set; } + public string IndexPath { get; set; } + public int DocumentCount { get; set; } + public bool StoreEnabled { get; set; } + public string LastSyncedRevision { get; set; } + public DateTime? LastSyncedOnUtc { get; set; } + public string Error { get; set; } + } +} diff --git a/Core/Resgrid.Model/Search/SearchProjection.cs b/Core/Resgrid.Model/Search/SearchProjection.cs new file mode 100644 index 000000000..1715531eb --- /dev/null +++ b/Core/Resgrid.Model/Search/SearchProjection.cs @@ -0,0 +1,197 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations.Schema; +using Newtonsoft.Json; + +namespace Resgrid.Model.Search +{ + /// + /// The safe, rebuildable search row for one entity in the global index (Unified Search plan R2.3, R3). One table + /// with an entity-type discriminator rather than one per family: the sweep, keyset checkpoint, rebuild and erasure + /// logic are then written once. Only allowlisted fields land here; cataloged (protected) columns are included only + /// for a department where protection is not enforced (R2.15), and a value carrying a protected-data envelope or + /// the redaction placeholder is never written. Keyed by (DepartmentId, EntityType, EntityId). + /// + [Table("SearchProjections")] + public class SearchProjection : IEntity + { + public string SearchProjectionId { get; set; } + + public int DepartmentId { get; set; } + + /// . + public string EntityType { get; set; } + + public string EntityId { get; set; } + + public string Title { get; set; } + + public string Summary { get; set; } + + /// Analyzed free text (never stored in the index). + public string SearchText { get; set; } + + /// Identifiers worth exact and prefix matching: call number, incident number, callsign, id number. + public string Keywords { get; set; } + + public string Category { get; set; } + + public string Status { get; set; } + + public int? Priority { get; set; } + + public int? GroupId { get; set; } + + public string OwnerUserId { get; set; } + + /// Comma-separated user ids that may see the row regardless of other rules (message recipients). + public string ParticipantUserIds { get; set; } + + public bool IsAdminOnly { get; set; } + + public bool IsActive { get; set; } + + public DateTime OccurredOn { get; set; } + + /// Relative web path to open the entity. + public string Url { get; set; } + + public string MetadataJson { get; set; } + + public int ProtectedCatalogVersion { get; set; } + + public long PolicyEpoch { get; set; } + + /// True when cataloged text columns were included because protection was not enforced at projection time. + public bool IncludesProtectedText { get; set; } + + public DateTime CreatedOn { get; set; } + + public DateTime ModifiedOn { get; set; } + + public long RowVersion { get; set; } + + public DateTime? DeletedOn { get; set; } + + [NotMapped] + [JsonIgnore] + public object IdValue + { + get { return SearchProjectionId; } + set { SearchProjectionId = value?.ToString(); } + } + + [NotMapped] + public string TableName => "SearchProjections"; + + [NotMapped] + public string IdName => "SearchProjectionId"; + + [NotMapped] + public int IdType => 1; + + [NotMapped] + public IEnumerable IgnoredProperties => new string[] { "IdValue", "IdType", "TableName", "IdName" }; + + public static string BuildKey(int departmentId, string entityType, string entityId) + { + return departmentId + "|" + entityType + "|" + entityId; + } + } + + /// Per-index, per-department generation key and checkpoint for the shared host (plan R4 Phase 1b item 3). + [Table("SearchIndexStates")] + public class SearchIndexState : IEntity + { + public int SearchIndexStateId { get; set; } + + public string IndexName { get; set; } + + public int DepartmentId { get; set; } + + public int SchemaVersion { get; set; } + + public int ProtectedCatalogVersion { get; set; } + + public long PolicyEpoch { get; set; } + + /// {schemaVersion}.{protectedCatalogVersion}.{policyEpoch} + public string Generation { get; set; } + + /// . + public int State { get; set; } + + public int DocumentCount { get; set; } + + public DateTime? LastRebuiltOn { get; set; } + + public DateTime? LastIndexedModifiedOn { get; set; } + + /// Set by the admin rebuild endpoint; the next sweep rebuilds the department and clears it. + public DateTime? RebuildRequestedOn { get; set; } + + public DateTime CreatedOn { get; set; } + + public DateTime ModifiedOn { get; set; } + + [NotMapped] + [JsonIgnore] + public object IdValue + { + get { return SearchIndexStateId; } + set { SearchIndexStateId = (int)value; } + } + + [NotMapped] + public string TableName => "SearchIndexStates"; + + [NotMapped] + public string IdName => "SearchIndexStateId"; + + [NotMapped] + public int IdType => 0; + + [NotMapped] + public IEnumerable IgnoredProperties => new string[] { "IdValue", "IdType", "TableName", "IdName" }; + } + + /// + /// The database publish lease: the second guard (after the Recreate rollout strategy) that keeps two worker pods + /// from publishing the same index prefix at once (plan R7 writer sequence step 2). One row per index name. + /// + [Table("SearchIndexLeases")] + public class SearchIndexLease : IEntity + { + public string IndexName { get; set; } + + public string LeaseOwner { get; set; } + + public DateTime? LeaseExpiresOn { get; set; } + + public string LastPublishedRevision { get; set; } + + public DateTime? LastPublishedOn { get; set; } + + public DateTime ModifiedOn { get; set; } + + [NotMapped] + [JsonIgnore] + public object IdValue + { + get { return IndexName; } + set { IndexName = value?.ToString(); } + } + + [NotMapped] + public string TableName => "SearchIndexLeases"; + + [NotMapped] + public string IdName => "IndexName"; + + [NotMapped] + public int IdType => 1; + + [NotMapped] + public IEnumerable IgnoredProperties => new string[] { "IdValue", "IdType", "TableName", "IdName" }; + } +} diff --git a/Core/Resgrid.Model/Search/UnifiedSearchContracts.cs b/Core/Resgrid.Model/Search/UnifiedSearchContracts.cs new file mode 100644 index 000000000..1de796a28 --- /dev/null +++ b/Core/Resgrid.Model/Search/UnifiedSearchContracts.cs @@ -0,0 +1,159 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace Resgrid.Model.Search +{ + /// Who is asking. Built by the controller from authenticated state; never from request input (plan 2026-08-15 correction). + public class SearchPrincipal + { + public string UserId { get; set; } + + public int DepartmentId { get; set; } + + public bool IsDepartmentAdmin { get; set; } + + public bool IsGroupAdmin { get; set; } + + /// Claim check against the caller's principal: (resource, action) → held. + public Func HasClaim { get; set; } = (r, a) => false; + + /// Department module toggles (Messaging, Mapping, ...) as the web layer sees them; null = all enabled. + public Func IsModuleEnabled { get; set; } + + public bool HasResourceClaim(string resource, string action) + { + try { return HasClaim != null && HasClaim(resource, action); } + catch { return false; } + } + + public bool ModuleEnabled(string module) + { + if (string.IsNullOrWhiteSpace(module) || IsModuleEnabled == null) + return true; + try { return IsModuleEnabled(module); } + catch { return true; } + } + } + + public class UnifiedSearchRequest + { + public string Text { get; set; } + + /// Restrict to these ; null or empty means every indexed family plus Records and Actions. + public List EntityTypes { get; set; } + + public bool IncludeActions { get; set; } = true; + + public bool IncludeRecords { get; set; } = true; + + public int Skip { get; set; } + + public int Take { get; set; } = 20; + + /// Typeahead: prefix-match the title of every family; short result list, no records federation. + public bool Prefix { get; set; } + } + + public class UnifiedSearchHit + { + public string EntityType { get; set; } + public string EntityId { get; set; } + public string Title { get; set; } + public string Summary { get; set; } + public string Url { get; set; } + public float Score { get; set; } + public DateTime? OccurredOn { get; set; } + public string Category { get; set; } + public string Status { get; set; } + public IDictionary Metadata { get; set; } = new Dictionary(); + } + + public class UnifiedSearchResult + { + public List Hits { get; set; } = new List(); + + public List Actions { get; set; } = new List(); + + /// Authorized total, or null when a hit was dropped by per-entity authorization and the total cannot be proven. + public int? Total { get; set; } + + public bool Truncated { get; set; } + + /// False when the department's search flag is off or the caller may not search at all. + public bool Available { get; set; } = true; + + /// True when the index could not serve (host off, index not built yet, error); actions still return. + public bool Degraded { get; set; } + + public string DegradedReason { get; set; } + + public int QueryTimeMs { get; set; } + } + + /// Categories for system functionality entries. + public static class SystemActionCategories + { + public const string Navigate = "Navigate"; + public const string Create = "Create"; + public const string Manage = "Manage"; + public const string Account = "Account"; + } + + /// Module switch names understood by . + public static class SystemActionModules + { + public const string Messaging = "Messaging"; + public const string Mapping = "Mapping"; + public const string Shifts = "Shifts"; + public const string Logs = "Logs"; + public const string Reports = "Reports"; + public const string Documents = "Documents"; + public const string Calendar = "Calendar"; + public const string Notes = "Notes"; + public const string Training = "Training"; + public const string Inventory = "Inventory"; + public const string Maintenance = "Maintenance"; + } + + /// + /// One piece of system functionality a user can jump to: a page or an action. Static catalog, filtered per caller + /// by claim, department-admin status, module switch and feature flag before it is ever scored. + /// + public class SystemActionDefinition + { + public string Key { get; set; } + public string Title { get; set; } + public string Description { get; set; } + public string[] Keywords { get; set; } = Array.Empty(); + public string Category { get; set; } = SystemActionCategories.Navigate; + + /// Relative web path, e.g. /User/Dispatch/Dashboard. "{userId}" is replaced with the caller's id. + public string WebPath { get; set; } + + /// Claim resource + action required, e.g. ("Call", "Create"); null = any member. + public string ClaimResource { get; set; } + public string ClaimAction { get; set; } + + public bool DepartmentAdminOnly { get; set; } + + /// Department module switch that must be on (see ). + public string Module { get; set; } + + /// Feature flag key that must evaluate true for the department (FeatureFlagKeys). + public string FeatureFlag { get; set; } + + /// Hide when the Records module is on: the legacy Logs pages are replaced after cutover. + public bool HiddenWhenRecordsEnabled { get; set; } + } + + public class SystemActionHit + { + public string Key { get; set; } + public string Title { get; set; } + public string Description { get; set; } + public string Category { get; set; } + public string Url { get; set; } + public float Score { get; set; } + } +} diff --git a/Core/Resgrid.Model/Services/ISearchServices.cs b/Core/Resgrid.Model/Services/ISearchServices.cs new file mode 100644 index 000000000..8c16cea7d --- /dev/null +++ b/Core/Resgrid.Model/Services/ISearchServices.cs @@ -0,0 +1,144 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Resgrid.Model.Search; + +namespace Resgrid.Model.Services +{ + /// + /// Writes the safe projection row for an entity (Unified Search plan R2.3). Called from the owning service's save + /// and delete paths; every method swallows and logs its own failures so a projection problem never fails the + /// entity write. Rebuilds live on because they need the entity + /// services, which would otherwise form a constructor cycle with the services that call this one. + /// + public interface ISearchProjectionService + { + Task ProjectCallAsync(Call call, CancellationToken cancellationToken = default); + + Task ProjectUnitAsync(Unit unit, CancellationToken cancellationToken = default); + + /// Null groupId / isActive keep the values already on the projection row (the profile save path knows neither). + Task ProjectPersonnelAsync(int departmentId, UserProfile profile, int? groupId, bool? isActive, CancellationToken cancellationToken = default); + + Task ProjectContactAsync(Contact contact, CancellationToken cancellationToken = default); + + Task ProjectMessageAsync(Message message, CancellationToken cancellationToken = default); + + Task ProjectDocumentAsync(Document document, CancellationToken cancellationToken = default); + + Task ProjectNoteAsync(Note note, CancellationToken cancellationToken = default); + + Task RemoveAsync(int departmentId, string entityType, string entityId, CancellationToken cancellationToken = default); + + /// Builds the projection row without saving it (used by rebuilds and tests). Null when nothing safe can be indexed. + Task BuildCallAsync(Call call); + Task BuildUnitAsync(Unit unit); + Task BuildPersonnelAsync(int departmentId, UserProfile profile, int? groupId, bool isActive); + Task BuildContactAsync(Contact contact); + Task BuildMessageAsync(Message message); + Task BuildDocumentAsync(Document document); + Task BuildNoteAsync(Note note); + + /// Upserts a prebuilt row (rebuild path). + Task UpsertAsync(SearchProjection projection, CancellationToken cancellationToken = default); + } + + /// + /// Worker command 70: keeps the global index in step with SearchProjections for every department that has a state + /// row. Generation change, missing index or an admin request rebuilds the department (projections first, then the + /// index); otherwise rows modified since the last sweep are re-indexed and soft-deleted rows removed. + /// + public interface ISearchIndexMaintenanceService + { + Task SweepAsync(CancellationToken cancellationToken = default); + + /// Forces a full projection + index rebuild for one department regardless of its generation key. + Task RebuildDepartmentAsync(int departmentId, CancellationToken cancellationToken = default); + + /// Creates the department's state row (if missing) and flags it for rebuild; the next sweep picks it up. + Task RequestRebuildAsync(int departmentId, CancellationToken cancellationToken = default); + } + + /// Write side of the global index. Only the worker process holds the IndexWriter; publish happens on commit. + public interface IGlobalSearchIndexer + { + Task IndexAsync(IEnumerable projections, string generation, CancellationToken cancellationToken = default); + + Task DeleteAsync(int departmentId, string entityType, string entityId, CancellationToken cancellationToken = default); + + Task DeleteDepartmentAsync(int departmentId, CancellationToken cancellationToken = default); + + /// Commits and, when the object store is enabled, publishes under the database lease. + Task CommitAsync(CancellationToken cancellationToken = default); + + /// Expunges deleted documents, commits, publishes and prunes superseded objects (erasure proof). + Task ExpungeDeletesAsync(CancellationToken cancellationToken = default); + + Task CountDocumentsAsync(int departmentId); + + /// True when a local index exists for this process (after a pull or a write). + bool IndexExists { get; } + } + + public class GlobalSearchQuery + { + public string Text { get; set; } + public List EntityTypes { get; set; } + public string ViewerUserId { get; set; } + public bool IncludeAdminOnly { get; set; } + public bool Prefix { get; set; } + public int Skip { get; set; } + public int Take { get; set; } = 50; + } + + public class GlobalSearchHit + { + public string ProjectionId { get; set; } + public string EntityType { get; set; } + public string EntityId { get; set; } + public string Title { get; set; } + public string Summary { get; set; } + public string Url { get; set; } + public string Category { get; set; } + public string Status { get; set; } + public long OccurredOnTicks { get; set; } + public string MetadataJson { get; set; } + public float Score { get; set; } + } + + public class GlobalSearchResult + { + public List Hits { get; set; } = new List(); + public int Total { get; set; } + public bool Truncated { get; set; } + public bool Available { get; set; } = true; + } + + /// Read side of the global index. The department clause is injected by the caller from authenticated state. + public interface IGlobalSearchService + { + bool IsAvailable { get; } + + Task SearchAsync(int departmentId, GlobalSearchQuery query, CancellationToken cancellationToken = default); + + Task GetHealthAsync(); + } + + /// + /// The unified endpoint behind the web command palette and the v4 API: global index hits re-checked per entity, + /// Records federated from the RMS index, and system functionality from the action catalog (plan R3, R4 Phase 2). + /// + public interface IUnifiedSearchService + { + Task SearchAsync(UnifiedSearchRequest request, SearchPrincipal principal, CancellationToken cancellationToken = default); + } + + /// Searches the static system-functionality catalog for one caller. + public interface ISystemActionsService + { + Task> SearchAsync(string text, SearchPrincipal principal, int max = 8, CancellationToken cancellationToken = default); + + /// Every entry the caller may use, unscored (the empty-query command palette). + Task> ListAsync(SearchPrincipal principal, CancellationToken cancellationToken = default); + } +} diff --git a/Core/Resgrid.Search/GlobalIndexFields.cs b/Core/Resgrid.Search/GlobalIndexFields.cs new file mode 100644 index 000000000..1e7345c9f --- /dev/null +++ b/Core/Resgrid.Search/GlobalIndexFields.cs @@ -0,0 +1,75 @@ +using System.Collections.Generic; +using System.IO; +using Lucene.Net.Analysis; +using Lucene.Net.Analysis.Core; +using Lucene.Net.Analysis.Miscellaneous; +using Lucene.Net.Analysis.NGram; +using Lucene.Net.Analysis.Standard; +using Lucene.Net.Util; + +namespace Resgrid.Search +{ + /// Field names of the global index (GlobalSearchGeneration.SchemaVersion governs changes here). + public static class GlobalIndexFields + { + public const LuceneVersion Version = LuceneIndexVersion.Version; + + public const string Key = "Key"; + public const string ProjectionId = "ProjectionId"; + public const string DepartmentId = "DepartmentId"; + public const string EntityType = "EntityType"; + public const string EntityId = "EntityId"; + public const string Title = "Title"; + /// Edge n-grams of the title tokens: typeahead. + public const string TitlePrefix = "TitlePrefix"; + public const string Summary = "Summary"; + public const string SearchText = "SearchText"; + public const string Keywords = "Keywords"; + /// Edge n-grams of identifier tokens (call numbers, callsigns). + public const string KeywordsPrefix = "KeywordsPrefix"; + /// Whole keywords lower-cased, one term each: exact identifier match. + public const string KeywordExact = "KeywordExact"; + public const string Category = "Category"; + public const string Status = "Status"; + public const string Priority = "Priority"; + public const string GroupId = "GroupId"; + public const string OwnerUserId = "OwnerUserId"; + public const string ParticipantUserIds = "ParticipantUserIds"; + public const string IsAdminOnly = "IsAdminOnly"; + public const string IsActive = "IsActive"; + public const string OccurredOn = "OccurredOn"; + public const string OccurredOnSort = "OccurredOnSort"; + public const string Url = "Url"; + public const string MetadataJson = "MetadataJson"; + public const string Generation = "Generation"; + + public const int PrefixMinGram = 1; + public const int PrefixMaxGram = 20; + + /// Index-time analyzer: standard for text, edge n-grams for the prefix fields. Query time uses the plain standard analyzer. + public static Analyzer CreateIndexAnalyzer() + { + var standard = new StandardAnalyzer(Version); + var perField = new Dictionary + { + [TitlePrefix] = new EdgeNGramAnalyzer(), + [KeywordsPrefix] = new EdgeNGramAnalyzer() + }; + return new PerFieldAnalyzerWrapper(standard, perField); + } + + public static Analyzer CreateQueryAnalyzer() => new StandardAnalyzer(Version); + } + + /// Standard tokens, lower-cased, expanded to edge n-grams (1..20) so "eng" matches "Engine". + public sealed class EdgeNGramAnalyzer : Analyzer + { + protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) + { + var source = new StandardTokenizer(GlobalIndexFields.Version, reader); + TokenStream stream = new LowerCaseFilter(GlobalIndexFields.Version, source); + stream = new EdgeNGramTokenFilter(GlobalIndexFields.Version, stream, GlobalIndexFields.PrefixMinGram, GlobalIndexFields.PrefixMaxGram); + return new TokenStreamComponents(source, stream); + } + } +} diff --git a/Core/Resgrid.Search/GlobalSearchDocumentBuilder.cs b/Core/Resgrid.Search/GlobalSearchDocumentBuilder.cs new file mode 100644 index 000000000..b497e3ee8 --- /dev/null +++ b/Core/Resgrid.Search/GlobalSearchDocumentBuilder.cs @@ -0,0 +1,97 @@ +using System; +using System.Collections.Generic; +using Lucene.Net.Documents; +using Resgrid.Model.Search; +using Document = Lucene.Net.Documents.Document; + +namespace Resgrid.Search +{ + /// + /// SearchProjection to Lucene Document. The projection is already the safe field set (plan R2.3, R2.15); nothing is + /// read from the source entity here, so the index can never hold more than the projection table does. + /// + public static class GlobalSearchDocumentBuilder + { + public static Document Build(SearchProjection p, string generation) + { + if (p == null) + throw new ArgumentNullException(nameof(p)); + + var doc = new Document + { + new StringField(GlobalIndexFields.Key, SearchProjection.BuildKey(p.DepartmentId, p.EntityType, p.EntityId), Field.Store.YES), + new StringField(GlobalIndexFields.ProjectionId, p.SearchProjectionId ?? string.Empty, Field.Store.YES), + new StringField(GlobalIndexFields.DepartmentId, p.DepartmentId.ToString(), Field.Store.YES), + new StringField(GlobalIndexFields.EntityType, p.EntityType ?? string.Empty, Field.Store.YES), + new StringField(GlobalIndexFields.EntityId, p.EntityId ?? string.Empty, Field.Store.YES), + new StringField(GlobalIndexFields.IsAdminOnly, p.IsAdminOnly ? "1" : "0", Field.Store.NO), + new StringField(GlobalIndexFields.IsActive, p.IsActive ? "1" : "0", Field.Store.NO), + new StringField(GlobalIndexFields.Generation, generation ?? string.Empty, Field.Store.YES), + new Int64Field(GlobalIndexFields.OccurredOn, p.OccurredOn.Ticks, Field.Store.YES), + new NumericDocValuesField(GlobalIndexFields.OccurredOnSort, p.OccurredOn.Ticks) + }; + + if (!string.IsNullOrWhiteSpace(p.Title)) + { + doc.Add(new TextField(GlobalIndexFields.Title, p.Title, Field.Store.YES)); + doc.Add(new TextField(GlobalIndexFields.TitlePrefix, p.Title, Field.Store.NO)); + } + if (!string.IsNullOrWhiteSpace(p.Summary)) + doc.Add(new TextField(GlobalIndexFields.Summary, p.Summary, Field.Store.YES)); + if (!string.IsNullOrWhiteSpace(p.SearchText)) + doc.Add(new TextField(GlobalIndexFields.SearchText, p.SearchText, Field.Store.NO)); + if (!string.IsNullOrWhiteSpace(p.Keywords)) + { + doc.Add(new TextField(GlobalIndexFields.Keywords, p.Keywords, Field.Store.NO)); + doc.Add(new TextField(GlobalIndexFields.KeywordsPrefix, p.Keywords, Field.Store.NO)); + foreach (var keyword in SplitKeywords(p.Keywords)) + doc.Add(new StringField(GlobalIndexFields.KeywordExact, keyword, Field.Store.NO)); + } + if (!string.IsNullOrWhiteSpace(p.Category)) + doc.Add(new StringField(GlobalIndexFields.Category, p.Category, Field.Store.YES)); + if (!string.IsNullOrWhiteSpace(p.Status)) + doc.Add(new StringField(GlobalIndexFields.Status, p.Status, Field.Store.YES)); + if (p.Priority.HasValue) + doc.Add(new StringField(GlobalIndexFields.Priority, p.Priority.Value.ToString(), Field.Store.YES)); + if (p.GroupId.HasValue) + doc.Add(new StringField(GlobalIndexFields.GroupId, p.GroupId.Value.ToString(), Field.Store.YES)); + if (!string.IsNullOrWhiteSpace(p.OwnerUserId)) + doc.Add(new StringField(GlobalIndexFields.OwnerUserId, p.OwnerUserId, Field.Store.NO)); + foreach (var id in SplitCsv(p.ParticipantUserIds)) + doc.Add(new StringField(GlobalIndexFields.ParticipantUserIds, id, Field.Store.NO)); + if (!string.IsNullOrWhiteSpace(p.Url)) + doc.Add(new StoredField(GlobalIndexFields.Url, p.Url)); + if (!string.IsNullOrWhiteSpace(p.MetadataJson)) + doc.Add(new StoredField(GlobalIndexFields.MetadataJson, p.MetadataJson)); + + return doc; + } + + public static IEnumerable SplitCsv(string csv) + { + if (string.IsNullOrWhiteSpace(csv)) + yield break; + + foreach (var part in csv.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)) + { + var trimmed = part.Trim(); + if (trimmed.Length > 0) + yield return trimmed; + } + } + + /// Keywords are separated by whitespace or commas; each becomes one lower-cased exact term. + public static IEnumerable SplitKeywords(string keywords) + { + if (string.IsNullOrWhiteSpace(keywords)) + yield break; + + foreach (var part in keywords.Split(new[] { ' ', ',', ';', '\t', '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries)) + { + var trimmed = part.Trim().ToLowerInvariant(); + if (trimmed.Length > 0) + yield return trimmed; + } + } + } +} diff --git a/Core/Resgrid.Search/LuceneGlobalSearchIndexer.cs b/Core/Resgrid.Search/LuceneGlobalSearchIndexer.cs new file mode 100644 index 000000000..29a45aa71 --- /dev/null +++ b/Core/Resgrid.Search/LuceneGlobalSearchIndexer.cs @@ -0,0 +1,105 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Lucene.Net.Index; +using Lucene.Net.Search; +using Resgrid.Model.Repositories; +using Resgrid.Model.Search; +using Resgrid.Model.Services; + +namespace Resgrid.Search +{ + /// + /// Write side of the global index: upsert by document key, delete by key or by department, commit-and-publish. Holds + /// no state of its own; the host owns the single writer. Only the worker process resolves this in anger. + /// + public class LuceneGlobalSearchIndexer : IGlobalSearchIndexer + { + private readonly LuceneGlobalIndexHost _host; + private readonly ISearchIndexLeasesRepository _leases; + + public LuceneGlobalSearchIndexer(LuceneGlobalIndexHost host) + : this(host, null) + { + } + + public LuceneGlobalSearchIndexer(LuceneGlobalIndexHost host, ISearchIndexLeasesRepository leases) + { + _host = host ?? throw new ArgumentNullException(nameof(host)); + _leases = leases; + } + + public bool IndexExists => _host.IndexExists; + + public Task IndexAsync(IEnumerable projections, string generation, CancellationToken cancellationToken = default) + { + var count = 0; + foreach (var projection in projections ?? Array.Empty()) + { + cancellationToken.ThrowIfCancellationRequested(); + if (projection == null || string.IsNullOrWhiteSpace(projection.EntityType) || string.IsNullOrWhiteSpace(projection.EntityId)) + continue; + + var key = SearchProjection.BuildKey(projection.DepartmentId, projection.EntityType, projection.EntityId); + count += _host.Write(writer => + { + if (projection.DeletedOn.HasValue) + { + writer.DeleteDocuments(new Term(GlobalIndexFields.Key, key)); + return 0; + } + writer.UpdateDocument(new Term(GlobalIndexFields.Key, key), GlobalSearchDocumentBuilder.Build(projection, generation)); + return 1; + }); + } + + return Task.FromResult(count); + } + + public Task DeleteAsync(int departmentId, string entityType, string entityId, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + _host.Write(writer => { writer.DeleteDocuments(new Term(GlobalIndexFields.Key, SearchProjection.BuildKey(departmentId, entityType, entityId))); return 0; }); + return Task.CompletedTask; + } + + public Task DeleteDepartmentAsync(int departmentId, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + _host.Write(writer => { writer.DeleteDocuments(new Term(GlobalIndexFields.DepartmentId, departmentId.ToString())); return 0; }); + return Task.CompletedTask; + } + + public Task CommitAsync(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + return SearchIndexPublishCoordinator.CommitAndPublishAsync(_host, _leases, cancellationToken); + } + + public Task ExpungeDeletesAsync(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + return SearchIndexPublishCoordinator.ExpungeAndPublishAsync(_host, _leases, cancellationToken); + } + + public Task CountDocumentsAsync(int departmentId) + { + var manager = _host.GetSearcherManager(); + if (manager == null) + return Task.FromResult(0); + + _host.MaybeRefresh(); + var searcher = manager.Acquire(); + try + { + var hits = searcher.Search(new TermQuery(new Term(GlobalIndexFields.DepartmentId, departmentId.ToString())), 1); + return Task.FromResult(hits.TotalHits); + } + finally + { + manager.Release(searcher); + } + } + } +} diff --git a/Core/Resgrid.Search/LuceneGlobalSearchService.cs b/Core/Resgrid.Search/LuceneGlobalSearchService.cs new file mode 100644 index 000000000..5c1d1142e --- /dev/null +++ b/Core/Resgrid.Search/LuceneGlobalSearchService.cs @@ -0,0 +1,305 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Lucene.Net.Analysis.TokenAttributes; +using Lucene.Net.Index; +using Lucene.Net.QueryParsers.Classic; +using Lucene.Net.Search; +using Resgrid.Config; +using Resgrid.Framework; +using Resgrid.Model.Search; +using Resgrid.Model.Services; + +namespace Resgrid.Search +{ + /// + /// Query side of the global index. The department clause is always injected; message documents are visible only + /// to their sender or a recipient, and admin-only documents/notes only to department admins — both resolve inside + /// the query so counts cannot disclose rows the viewer cannot open. User text is escaped, so wildcards, ranges and + /// field selectors from request input never reach the parser (plan R2.4). Every hit is still re-checked by the + /// caller (UnifiedSearchService) before it is shown. + /// + public class LuceneGlobalSearchService : IGlobalSearchService + { + private static readonly string[] TextFields = + { + GlobalIndexFields.Title, GlobalIndexFields.Keywords, GlobalIndexFields.Summary, GlobalIndexFields.SearchText + }; + + private static readonly IDictionary TextBoosts = new Dictionary + { + [GlobalIndexFields.Title] = 5f, + [GlobalIndexFields.Keywords] = 4f, + [GlobalIndexFields.Summary] = 2f, + [GlobalIndexFields.SearchText] = 1f + }; + + private readonly LuceneGlobalIndexHost _host; + + public LuceneGlobalSearchService(LuceneGlobalIndexHost host) + { + _host = host ?? throw new ArgumentNullException(nameof(host)); + } + + public bool IsAvailable => _host.Enabled && _host.IndexExists; + + public Task SearchAsync(int departmentId, GlobalSearchQuery query, CancellationToken cancellationToken = default) + { + query = query ?? new GlobalSearchQuery(); + var result = new GlobalSearchResult(); + + if (!_host.Enabled || departmentId <= 0) + { + result.Available = false; + return Task.FromResult(result); + } + + SearcherManager manager; + try + { + manager = _host.GetSearcherManager(); + } + catch (Exception ex) + { + Logging.LogException(ex, "Global search reader could not be opened."); + result.Available = false; + return Task.FromResult(result); + } + + if (manager == null) + { + result.Available = false; + return Task.FromResult(result); + } + + cancellationToken.ThrowIfCancellationRequested(); + _host.MaybeRefresh(); + + var lucene = BuildQuery(departmentId, query); + var take = query.Take <= 0 ? 50 : Math.Min(query.Take, Math.Max(1, SearchConfig.MaxResults)); + var skip = Math.Max(0, query.Skip); + var window = Math.Min(skip + take, Math.Max(1, SearchConfig.MaxResults)); + + var searcher = manager.Acquire(); + try + { + var hasText = !string.IsNullOrWhiteSpace(query.Text); + var topDocs = hasText + ? searcher.Search(lucene, window) + : searcher.Search(lucene, window, new Sort(new SortField(GlobalIndexFields.OccurredOnSort, SortFieldType.INT64, true))); + + result.Total = topDocs.TotalHits; + result.Truncated = topDocs.TotalHits > SearchConfig.MaxResults; + + foreach (var scoreDoc in topDocs.ScoreDocs.Skip(skip).Take(take)) + { + var doc = searcher.Doc(scoreDoc.Doc); + result.Hits.Add(new GlobalSearchHit + { + ProjectionId = doc.Get(GlobalIndexFields.ProjectionId), + EntityType = doc.Get(GlobalIndexFields.EntityType), + EntityId = doc.Get(GlobalIndexFields.EntityId), + Title = doc.Get(GlobalIndexFields.Title), + Summary = doc.Get(GlobalIndexFields.Summary), + Url = doc.Get(GlobalIndexFields.Url), + Category = doc.Get(GlobalIndexFields.Category), + Status = doc.Get(GlobalIndexFields.Status), + MetadataJson = doc.Get(GlobalIndexFields.MetadataJson), + OccurredOnTicks = long.TryParse(doc.Get(GlobalIndexFields.OccurredOn), out var ticks) ? ticks : 0, + Score = float.IsNaN(scoreDoc.Score) ? 0f : scoreDoc.Score + }); + } + } + finally + { + manager.Release(searcher); + } + + return Task.FromResult(result); + } + + public Task GetHealthAsync() + { + var health = new SearchIndexHealth + { + IndexName = _host.IndexName, + Enabled = _host.Enabled, + IndexPath = _host.IndexPath, + StoreEnabled = _host.StoreEnabled, + LastSyncedRevision = _host.LastSyncedRevision, + LastSyncedOnUtc = _host.LastSyncedOnUtc + }; + if (!_host.Enabled) + return Task.FromResult(health); + + try + { + var manager = _host.GetSearcherManager(); + if (manager == null) + { + health.Online = false; + health.Error = "Index not created yet."; + return Task.FromResult(health); + } + + _host.MaybeRefresh(); + var searcher = manager.Acquire(); + try + { + health.Online = true; + health.DocumentCount = searcher.IndexReader.NumDocs; + } + finally + { + manager.Release(searcher); + } + } + catch (Exception ex) + { + Logging.LogException(ex, "Global search health check failed."); + health.Online = false; + health.Error = ex.Message; + } + + return Task.FromResult(health); + } + + /// Visible for tests: the exact query the service runs. + public static Query BuildQuery(int departmentId, GlobalSearchQuery request) + { + var query = new BooleanQuery + { + { new TermQuery(new Term(GlobalIndexFields.DepartmentId, departmentId.ToString())), Occur.MUST } + }; + + if (request.EntityTypes != null && request.EntityTypes.Count > 0) + { + var types = new BooleanQuery(); + foreach (var type in request.EntityTypes.Where(t => !string.IsNullOrWhiteSpace(t)).Distinct(StringComparer.OrdinalIgnoreCase)) + types.Add(new TermQuery(new Term(GlobalIndexFields.EntityType, type.Trim())), Occur.SHOULD); + if (types.Clauses.Count > 0) + query.Add(types, Occur.MUST); + } + + // Messages: only the sender or a recipient may see them. (+Message +(owner OR participant)) OR (NOT Message). + var viewer = request.ViewerUserId ?? string.Empty; + var messageScope = new BooleanQuery(); + var messageForViewer = new BooleanQuery { { new TermQuery(new Term(GlobalIndexFields.EntityType, SearchEntityTypes.Message)), Occur.MUST } }; + var viewerMatch = new BooleanQuery(); + if (viewer.Length > 0) + { + viewerMatch.Add(new TermQuery(new Term(GlobalIndexFields.OwnerUserId, viewer)), Occur.SHOULD); + viewerMatch.Add(new TermQuery(new Term(GlobalIndexFields.ParticipantUserIds, viewer)), Occur.SHOULD); + } + else + { + viewerMatch.Add(new TermQuery(new Term(GlobalIndexFields.Key, " none")), Occur.SHOULD); + } + messageForViewer.Add(viewerMatch, Occur.MUST); + messageScope.Add(messageForViewer, Occur.SHOULD); + messageScope.Add(new BooleanQuery + { + { new MatchAllDocsQuery(), Occur.MUST }, + { new TermQuery(new Term(GlobalIndexFields.EntityType, SearchEntityTypes.Message)), Occur.MUST_NOT } + }, Occur.SHOULD); + query.Add(messageScope, Occur.MUST); + + if (!request.IncludeAdminOnly) + query.Add(new TermQuery(new Term(GlobalIndexFields.IsAdminOnly, "1")), Occur.MUST_NOT); + + if (!string.IsNullOrWhiteSpace(request.Text)) + { + var text = request.Text.Trim(); + var tokens = Tokenize(text); + var textQuery = new BooleanQuery(); + + if (request.Prefix) + { + // Typeahead: every token must prefix-match a title or keyword token. + var prefix = new BooleanQuery(); + foreach (var token in tokens) + { + var either = new BooleanQuery + { + { new TermQuery(new Term(GlobalIndexFields.TitlePrefix, token)) { Boost = 3f }, Occur.SHOULD }, + { new TermQuery(new Term(GlobalIndexFields.KeywordsPrefix, token)) { Boost = 4f }, Occur.SHOULD } + }; + prefix.Add(either, Occur.MUST); + } + if (prefix.Clauses.Count > 0) + textQuery.Add(prefix, Occur.SHOULD); + } + else + { + using var analyzer = GlobalIndexFields.CreateQueryAnalyzer(); + var parser = new MultiFieldQueryParser(GlobalIndexFields.Version, TextFields, analyzer, TextBoosts) + { + DefaultOperator = Operator.AND, + AllowLeadingWildcard = false + }; + Query parsed; + try + { + parsed = parser.Parse(QueryParserBase.Escape(text)); + } + catch (ParseException) + { + parsed = null; + } + if (parsed != null) + textQuery.Add(parsed, Occur.SHOULD); + + // A final partial token still prefix-matches (the user is mid-word). + if (tokens.Count > 0) + { + var last = tokens[tokens.Count - 1]; + var lastPrefix = new BooleanQuery(); + foreach (var token in tokens.Take(tokens.Count - 1)) + { + lastPrefix.Add(new BooleanQuery + { + { new TermQuery(new Term(GlobalIndexFields.TitlePrefix, token)), Occur.SHOULD }, + { new TermQuery(new Term(GlobalIndexFields.KeywordsPrefix, token)), Occur.SHOULD } + }, Occur.MUST); + } + lastPrefix.Add(new BooleanQuery + { + { new TermQuery(new Term(GlobalIndexFields.TitlePrefix, last)) { Boost = 2f }, Occur.SHOULD }, + { new TermQuery(new Term(GlobalIndexFields.KeywordsPrefix, last)) { Boost = 3f }, Occur.SHOULD } + }, Occur.MUST); + textQuery.Add(lastPrefix, Occur.SHOULD); + } + } + + textQuery.Add(new TermQuery(new Term(GlobalIndexFields.KeywordExact, text.ToLowerInvariant())) { Boost = 6f }, Occur.SHOULD); + query.Add(textQuery, Occur.MUST); + } + + return query; + } + + /// Standard-analyzer tokens of the user text (lower-cased, stop words removed), never more than 12. + public static List Tokenize(string text) + { + var tokens = new List(); + if (string.IsNullOrWhiteSpace(text)) + return tokens; + + using var analyzer = GlobalIndexFields.CreateQueryAnalyzer(); + using var stream = analyzer.GetTokenStream(GlobalIndexFields.Title, new StringReader(text)); + var term = stream.AddAttribute(); + stream.Reset(); + while (stream.IncrementToken() && tokens.Count < 12) + { + var value = term.ToString(); + if (value.Length > 0) + tokens.Add(value); + } + stream.End(); + return tokens; + } + } +} diff --git a/Core/Resgrid.Search/LuceneIndexHost.cs b/Core/Resgrid.Search/LuceneIndexHost.cs new file mode 100644 index 000000000..37488596c --- /dev/null +++ b/Core/Resgrid.Search/LuceneIndexHost.cs @@ -0,0 +1,531 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Lucene.Net.Analysis; +using Lucene.Net.Index; +using Lucene.Net.Search; +using Lucene.Net.Store; +using Resgrid.Config; +using Resgrid.Framework; +using Resgrid.Model.Providers; +using Resgrid.Model.Search; +using Directory = Lucene.Net.Store.Directory; + +namespace Resgrid.Search +{ + /// + /// The shared Lucene host, one process-wide instance per index (Unified Search plan section 3.4, R7). The worker + /// process opens the single IndexWriter and reads near-real-time through it; every other process opens a + /// read-only SearcherManager over the committed segments. The live directory is always local. When the object + /// store is enabled the writer publishes every commit (immutable segment files first, then segments_N, then the + /// manifest with a conditional PUT) and readers pull the manifest into their local cache; without a store the + /// single shared directory and Lucene's own file locking coordinate the processes as before. + /// + public class LuceneIndexHost : IDisposable + { + private const string ManifestFileName = "manifest.json"; + private const string LockFileName = "write.lock"; + + private readonly object _sync = new object(); + private readonly object _writerSyncGate = new object(); + private readonly bool _ownsDirectory; + private readonly ISearchIndexStore _store; + private readonly string _localPathOverride; + private Directory _directory; + private IndexWriter _writer; + private SnapshotDeletionPolicy _snapshots; + private SearcherManager _searcherManager; + private bool _searcherIsNrt; + private bool _writerSyncedFromStore; + private bool _disposed; + + private string _appliedRevision; + private long _appliedGeneration = -1; + private string _manifestETag; + private DateTime _lastPullAttemptUtc = DateTime.MinValue; + private Task _pullTask; + + /// Production constructor: the configured local path under SearchConfig.IndexPath. + public LuceneIndexHost(string indexName, Analyzer analyzer, ISearchIndexStore store) + { + // Deliberately no I/O here. This type is a container singleton, so opening the configured path from the + // constructor makes every process that merely composes its container depend on the local volume being + // present and writable — a process with search disabled must still start. The directory opens on use. + IndexName = indexName ?? throw new ArgumentNullException(nameof(indexName)); + Analyzer = analyzer ?? throw new ArgumentNullException(nameof(analyzer)); + _store = store ?? NullSearchIndexStore.Instance; + _ownsDirectory = true; + } + + /// Test seam: host any directory (e.g. RAMDirectory or a temp FSDirectory) without touching the configured path. + public LuceneIndexHost(string indexName, Analyzer analyzer, Directory directory, bool ownsDirectory = false, ISearchIndexStore store = null) + { + IndexName = indexName ?? throw new ArgumentNullException(nameof(indexName)); + Analyzer = analyzer ?? throw new ArgumentNullException(nameof(analyzer)); + _directory = directory ?? throw new ArgumentNullException(nameof(directory)); + _ownsDirectory = ownsDirectory; + _store = store ?? NullSearchIndexStore.Instance; + if (directory is FSDirectory fs) + _localPathOverride = fs.Directory.FullName; + } + + public string IndexName { get; } + + public Analyzer Analyzer { get; } + + public string IndexPath => _localPathOverride ?? Path.Combine(SearchConfig.IndexPath ?? string.Empty, IndexName); + + public bool Enabled => SearchConfig.Enabled; + + /// True when an object store is configured and this host has a real local path to sync into. + public bool StoreEnabled => _store.Enabled && (_localPathOverride != null || _directory == null || _directory is FSDirectory); + + public string LastSyncedRevision => _appliedRevision; + + public DateTime? LastSyncedOnUtc { get; private set; } + + /// True while this process holds the writer (worker); false in reader processes. + public bool IsWriter { get { lock (_sync) { return _writer != null; } } } + + /// The backing store, opening the configured path on first use. + private Directory Store + { + get + { + if (_directory != null) + return _directory; + + lock (_sync) + return _directory ??= OpenConfiguredDirectory(); + } + } + + public bool IndexExists + { + get + { + try { return DirectoryReader.IndexExists(Store); } + catch (Exception ex) + { + Logging.LogException(ex, $"Search index '{IndexName}' existence check failed."); + return false; + } + } + } + + /// + /// Opens (once) the single IndexWriter. Only the worker process should ever call this. With the object store + /// enabled the latest published revision is pulled first, so a fresh pod never starts from an empty directory + /// and diverges from what readers already hold (plan R7 writer sequence step 1). + /// + public IndexWriter GetWriter() + { + EnsureWriterSyncedFromStore(); + lock (_sync) + { + ThrowIfDisposed(); + if (_writer != null) + return _writer; + + _snapshots = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); + var config = new IndexWriterConfig(LuceneIndexVersion.Version, Analyzer) + { + OpenMode = OpenMode.CREATE_OR_APPEND, + MergePolicy = new TieredMergePolicy { ForceMergeDeletesPctAllowed = 0 }, + RAMBufferSizeMB = Math.Max(1, SearchConfig.RamBufferSizeMb), + IndexDeletionPolicy = _snapshots + }; + _writer = new IndexWriter(Store, config); + + // A reader opened before the writer existed keeps working; from here on prefer the NRT view. + if (_searcherManager != null && !_searcherIsNrt) + { + _searcherManager.Dispose(); + _searcherManager = null; + } + + return _writer; + } + } + + /// The reader for this process, or null when no index exists locally yet (reader processes: until the first pull completes). + public SearcherManager GetSearcherManager() + { + lock (_sync) + { + ThrowIfDisposed(); + if (_searcherManager != null) + return _searcherManager; + + if (_writer != null) + { + _searcherManager = new SearcherManager(_writer, true, null); + _searcherIsNrt = true; + return _searcherManager; + } + + if (StoreEnabled) + StartBackgroundPullIfDue(force: _appliedRevision == null); + + if (!DirectoryReader.IndexExists(Store)) + return null; + + _searcherManager = new SearcherManager(Store, null); + _searcherIsNrt = false; + return _searcherManager; + } + } + + /// Refreshes the reader; in a reader process with the store enabled also schedules a manifest poll when one is due. + public void MaybeRefresh() + { + SearcherManager manager; + lock (_sync) + { + manager = _searcherManager; + if (_writer == null && StoreEnabled) + StartBackgroundPullIfDue(force: false); + } + + try { manager?.MaybeRefresh(); } + catch (Exception ex) { Logging.LogException(ex, $"Search index '{IndexName}' reader refresh failed."); } + } + + /// + /// Writer startup pull (plan R7 writer sequence step 1), taken on a separate gate so the pull's own refresh of the + /// reader can take _sync: holding _sync while waiting on the pull thread would deadlock. + /// + private void EnsureWriterSyncedFromStore() + { + if (!StoreEnabled || _writerSyncedFromStore) + return; + + lock (_writerSyncGate) + { + if (_writerSyncedFromStore) + return; + try + { + // No SynchronizationContext in the worker; the blocking wait cannot deadlock now that _sync is free. + Task.Run(() => PullCoreAsync(CancellationToken.None)).GetAwaiter().GetResult(); + } + catch (Exception ex) + { + // A store outage must not stop the writer from working locally; publish will fail loudly later. + Logging.LogException(ex, $"Search index '{IndexName}' could not be pulled from the object store before opening the writer; continuing with the local directory."); + } + _writerSyncedFromStore = true; + } + } + + /// Serializes mutations with the committed-segment erasure pass. + public int Write(Func mutation) + { + EnsureWriterSyncedFromStore(); + lock (_sync) { ThrowIfDisposed(); return mutation(GetWriter()); } + } + + /// Commits the writer. Publishing is a separate step so the caller can hold the database lease around it. + public void Commit() + { + EnsureWriterSyncedFromStore(); + lock (_sync) + { + ThrowIfDisposed(); + GetWriter().Commit(); + _searcherManager?.MaybeRefresh(); + } + } + + /// Force-merges deletes and commits; throws if deleted documents remain in the committed index (erasure proof). + public void ExpungeDeletes() + { + EnsureWriterSyncedFromStore(); + lock (_sync) + { + ThrowIfDisposed(); + var writer = GetWriter(); + writer.ForceMergeDeletes(true); + writer.Commit(); + _searcherManager?.MaybeRefreshBlocking(); + writer.DeleteUnusedFiles(); + using var committed = DirectoryReader.Open(Store); + if (committed.HasDeletions) throw new InvalidOperationException("Deleted documents remain in the committed index; erasure cannot be acknowledged."); + } + } + + /// + /// Publishes the latest commit to the object store (plan R7 writer sequence steps 4–6): immutable files that the + /// bucket lacks, then segments_N, then the manifest under a conditional PUT, then prune. No-op without a store. + /// Throws when another writer has published meanwhile. + /// + public async Task PublishAsync(string publishedBy, CancellationToken cancellationToken = default) + { + if (!StoreEnabled) + return null; + + EnsureWriterSyncedFromStore(); + IndexCommit commit; + IndexWriter writer; + lock (_sync) + { + ThrowIfDisposed(); + writer = GetWriter(); + if (!DirectoryReader.IndexExists(Store)) + return null; + commit = _snapshots.Snapshot(); + } + + try + { + var localFiles = commit.FileNames.Where(f => !string.Equals(f, LockFileName, StringComparison.OrdinalIgnoreCase)).Distinct().ToList(); + var segmentsFile = commit.SegmentsFileName; + var remote = await _store.ListFilesAsync(IndexName, cancellationToken) ?? new HashSet(StringComparer.Ordinal); + + foreach (var file in localFiles.Where(f => !string.Equals(f, segmentsFile, StringComparison.Ordinal))) + { + cancellationToken.ThrowIfCancellationRequested(); + if (remote.Contains(file)) + continue; + await _store.UploadFileAsync(IndexName, file, Path.Combine(IndexPath, file), cancellationToken); + } + + await _store.UploadFileAsync(IndexName, segmentsFile, Path.Combine(IndexPath, segmentsFile), cancellationToken); + + var manifest = new SearchIndexManifest + { + IndexName = IndexName, + Revision = Guid.NewGuid().ToString("N"), + SegmentsGeneration = commit.Generation, + SchemaVersion = 1, + PublishedOnUtc = DateTime.UtcNow, + PublishedBy = publishedBy, + Files = localFiles.Select(f => new SearchIndexManifestFile { Name = f, Length = SafeLength(Path.Combine(IndexPath, f)) }).ToList() + }; + + string expected = _manifestETag; + if (expected == null) + { + // Fresh process that never read the manifest: adopt the remote only when we synced from it. + var current = await _store.GetManifestAsync(IndexName, cancellationToken); + if (current != null) + { + if (!_writerSyncedFromStore || !string.Equals(current.Revision, _appliedRevision, StringComparison.Ordinal)) + throw new SearchIndexManifestConflictException(IndexName, $"Search index '{IndexName}' has a published manifest (revision {current.Revision}) this writer did not start from; refusing to overwrite it."); + expected = current.ETag; + } + } + + var stored = await _store.PutManifestAsync(IndexName, manifest, expected, cancellationToken); + _manifestETag = stored?.ETag; + _appliedRevision = manifest.Revision; + _appliedGeneration = manifest.SegmentsGeneration; + LastSyncedOnUtc = DateTime.UtcNow; + + var keep = new HashSet(localFiles, StringComparer.Ordinal); + var stale = remote.Where(r => !keep.Contains(r)).ToList(); + if (stale.Count > 0) + await _store.DeleteFilesAsync(IndexName, stale, cancellationToken); + + return manifest; + } + finally + { + lock (_sync) + { + try { _snapshots.Release(commit); } catch (Exception ex) { Logging.LogException(ex); } + try { writer.DeleteUnusedFiles(); } catch (Exception ex) { Logging.LogException(ex); } + } + } + } + + /// Pulls the latest published revision into the local directory (reader processes; writer at startup). Returns true when files changed. + public Task PullAsync(CancellationToken cancellationToken = default) + { + if (!StoreEnabled) + return Task.FromResult(false); + return PullCoreAsync(cancellationToken); + } + + /// + /// Writer recovery after a manifest conflict: drop the local writer and directory, pull the published revision and + /// let the maintenance sweep re-index from the projection tables (its checkpoint only advances after a publish). + /// + public async Task ResetFromStoreAsync(CancellationToken cancellationToken = default) + { + lock (_sync) + { + ThrowIfDisposed(); + try { _searcherManager?.Dispose(); } catch (Exception ex) { Logging.LogException(ex); } + try { _writer?.Dispose(); } catch (Exception ex) { Logging.LogException(ex); } + _searcherManager = null; + _writer = null; + _snapshots = null; + WipeLocalFiles(); + _appliedRevision = null; + _appliedGeneration = -1; + _manifestETag = null; + _writerSyncedFromStore = false; + } + + if (StoreEnabled) + await PullCoreAsync(cancellationToken); + } + + private void StartBackgroundPullIfDue(bool force) + { + // Called under _sync. + if (_pullTask != null && !_pullTask.IsCompleted) + return; + var due = force || (DateTime.UtcNow - _lastPullAttemptUtc).TotalSeconds >= Math.Max(5, SearchConfig.ReaderPullSeconds); + if (!due) + return; + _lastPullAttemptUtc = DateTime.UtcNow; + _pullTask = Task.Run(async () => + { + try { await PullCoreAsync(CancellationToken.None); } + catch (Exception ex) { Logging.LogException(ex, $"Search index '{IndexName}' pull from the object store failed."); } + }); + } + + private async Task PullCoreAsync(CancellationToken cancellationToken) + { + _lastPullAttemptUtc = DateTime.UtcNow; + var manifest = await _store.GetManifestAsync(IndexName, cancellationToken); + if (manifest == null) + return false; + if (string.Equals(manifest.Revision, _appliedRevision, StringComparison.Ordinal)) + { + _manifestETag = manifest.ETag ?? _manifestETag; + return false; + } + + var localPath = IndexPath; + System.IO.Directory.CreateDirectory(localPath); + + // A generation that went backwards means the writer was rebuilt from scratch: file names will be reused + // with different content, so start from an empty directory rather than trusting name+length matches. + if (_appliedGeneration >= 0 && manifest.SegmentsGeneration < _appliedGeneration) + { + lock (_sync) { WipeLocalFiles(keepOpenHandles: true); } + } + + var wanted = new Dictionary(StringComparer.Ordinal); + foreach (var f in manifest.Files ?? new List()) + wanted[f.Name] = f.Length; + + string segmentsFile = wanted.Keys.FirstOrDefault(n => n.StartsWith("segments_", StringComparison.Ordinal)); + foreach (var pair in wanted.Where(p => !string.Equals(p.Key, segmentsFile, StringComparison.Ordinal))) + await DownloadIfNeededAsync(localPath, pair.Key, pair.Value, cancellationToken); + if (segmentsFile != null) + await DownloadIfNeededAsync(localPath, segmentsFile, wanted[segmentsFile], cancellationToken); + + foreach (var existing in System.IO.Directory.EnumerateFiles(localPath)) + { + var name = Path.GetFileName(existing); + if (wanted.ContainsKey(name) || string.Equals(name, LockFileName, StringComparison.OrdinalIgnoreCase) || name.EndsWith(".tmp", StringComparison.OrdinalIgnoreCase)) + continue; + try { File.Delete(existing); } + catch (Exception ex) { Logging.LogException(ex, $"Search index '{IndexName}': stale local file {name} could not be removed yet."); } + } + + _appliedRevision = manifest.Revision; + _appliedGeneration = manifest.SegmentsGeneration; + _manifestETag = manifest.ETag; + LastSyncedOnUtc = DateTime.UtcNow; + + lock (_sync) + { + if (_disposed) + return true; + if (_searcherManager != null && !_searcherIsNrt) + { + try { _searcherManager.MaybeRefreshBlocking(); } + catch (Exception ex) { Logging.LogException(ex, $"Search index '{IndexName}' reader refresh after pull failed."); } + } + } + + return true; + } + + private async Task DownloadIfNeededAsync(string localPath, string name, long length, CancellationToken cancellationToken) + { + var target = Path.Combine(localPath, name); + if (File.Exists(target) && new FileInfo(target).Length == length && !name.StartsWith("segments_", StringComparison.Ordinal)) + return; + + var tmp = target + ".tmp"; + try { if (File.Exists(tmp)) File.Delete(tmp); } catch { } + await _store.DownloadFileAsync(IndexName, name, tmp, cancellationToken); + if (File.Exists(target)) + File.Delete(target); + File.Move(tmp, target); + } + + private void WipeLocalFiles(bool keepOpenHandles = false) + { + // Called under _sync. + try + { + if (!System.IO.Directory.Exists(IndexPath)) + return; + foreach (var file in System.IO.Directory.EnumerateFiles(IndexPath)) + { + var name = Path.GetFileName(file); + if (string.Equals(name, LockFileName, StringComparison.OrdinalIgnoreCase)) + continue; + try { File.Delete(file); } + catch (Exception ex) { if (!keepOpenHandles) Logging.LogException(ex); } + } + } + catch (Exception ex) { Logging.LogException(ex); } + } + + private static long SafeLength(string path) + { + try { return new FileInfo(path).Length; } catch { return 0; } + } + + private Directory OpenConfiguredDirectory() + { + var path = IndexPath; + System.IO.Directory.CreateDirectory(path); + return FSDirectory.Open(path); + } + + private void ThrowIfDisposed() + { + if (_disposed) + throw new ObjectDisposedException(GetType().Name); + } + + public void Dispose() + { + lock (_sync) + { + if (_disposed) + return; + _disposed = true; + + try { _searcherManager?.Dispose(); } catch (Exception ex) { Logging.LogException(ex); } + try { _writer?.Dispose(); } catch (Exception ex) { Logging.LogException(ex); } + // _directory, never Store: a host that was disposed without ever indexing must not open the + // configured path on its way out. + if (_ownsDirectory && _directory != null) + { + try { _directory.Dispose(); } catch (Exception ex) { Logging.LogException(ex); } + } + try { Analyzer.Dispose(); } catch (Exception ex) { Logging.LogException(ex); } + } + } + } + + /// Every host and every process must agree on the Lucene version. + public static class LuceneIndexVersion + { + public const Lucene.Net.Util.LuceneVersion Version = Lucene.Net.Util.LuceneVersion.LUCENE_48; + } +} diff --git a/Core/Resgrid.Search/LuceneRecordsIndexHost.cs b/Core/Resgrid.Search/LuceneRecordsIndexHost.cs index bfe8093d5..66c74a2c9 100644 --- a/Core/Resgrid.Search/LuceneRecordsIndexHost.cs +++ b/Core/Resgrid.Search/LuceneRecordsIndexHost.cs @@ -1,198 +1,50 @@ -using System; -using System.IO; -using Lucene.Net.Analysis; using Lucene.Net.Analysis.Standard; -using Lucene.Net.Index; -using Lucene.Net.Search; -using Lucene.Net.Store; -using Resgrid.Config; -using Resgrid.Framework; +using Resgrid.Model.Providers; +using Resgrid.Model.Search; using Directory = Lucene.Net.Store.Directory; namespace Resgrid.Search { /// - /// The shared Lucene host for the records index (Unified Search plan section 3.4): one process-wide - /// instance per index. The worker process opens the single IndexWriter and reads near-real-time through it; - /// every other process opens a read-only SearcherManager over the committed segments and refreshes on demand. - /// Cross-process coordination is Lucene's own directory locking on the shared volume. + /// The RMS-owned records index on the shared host (RMS plan section 5.10). Kept as its own type so the RMS + /// indexer, search service, maintenance service and tests are untouched by the host generalisation (plan R4 Phase 1b). /// - public sealed class LuceneRecordsIndexHost : IDisposable + public sealed class LuceneRecordsIndexHost : LuceneIndexHost { - private readonly object _sync = new object(); - private readonly bool _ownsDirectory; - private Directory _directory; - private IndexWriter _writer; - private SearcherManager _searcherManager; - private bool _searcherIsNrt; - private bool _disposed; - + /// Local-directory host without an object store (tests, single-host Compose). Autofac prefers the store constructor. public LuceneRecordsIndexHost() + : this(NullSearchIndexStore.Instance) { - // Deliberately no I/O here. This type is a container singleton, so opening the configured path from the - // constructor makes every process that merely composes its container depend on the shared volume being - // present and writable — a process with search disabled must still start. The directory opens on use. - _ownsDirectory = true; - Analyzer = new StandardAnalyzer(RecordsIndexFields.Version); - } - - /// Test seam: host any directory (e.g. RAMDirectory) without touching the configured path. - public LuceneRecordsIndexHost(Directory directory, bool ownsDirectory = false) - { - _directory = directory ?? throw new ArgumentNullException(nameof(directory)); - _ownsDirectory = ownsDirectory; - Analyzer = new StandardAnalyzer(RecordsIndexFields.Version); - } - - public Analyzer Analyzer { get; } - - public string IndexPath => Path.Combine(SearchConfig.IndexPath ?? string.Empty, RecordsIndexFields.IndexName); - - public bool Enabled => SearchConfig.Enabled; - - /// The backing store, opening the configured path on first use. - private Directory Store - { - get - { - if (_directory != null) - return _directory; - - lock (_sync) - return _directory ??= OpenConfiguredDirectory(); - } } - public bool IndexExists + public LuceneRecordsIndexHost(ISearchIndexStore store) + : base(SearchIndexNames.Records, new StandardAnalyzer(RecordsIndexFields.Version), store) { - get - { - try { return DirectoryReader.IndexExists(Store); } - catch (Exception ex) - { - Logging.LogException(ex, "Records search index existence check failed."); - return false; - } - } - } - - /// Opens (once) the single IndexWriter. Only the worker process should ever call this. - public IndexWriter GetWriter() - { - lock (_sync) - { - ThrowIfDisposed(); - if (_writer != null) - return _writer; - - var config = new IndexWriterConfig(RecordsIndexFields.Version, Analyzer) - { - OpenMode = OpenMode.CREATE_OR_APPEND, - MergePolicy = new TieredMergePolicy { ForceMergeDeletesPctAllowed = 0 }, - RAMBufferSizeMB = Math.Max(1, SearchConfig.RamBufferSizeMb) - }; - _writer = new IndexWriter(Store, config); - - // A reader opened before the writer existed keeps working; from here on prefer the NRT view. - if (_searcherManager != null && !_searcherIsNrt) - { - _searcherManager.Dispose(); - _searcherManager = null; - } - - return _writer; - } } - /// The reader for this process, or null when no index has been created yet. - public SearcherManager GetSearcherManager() - { - lock (_sync) - { - ThrowIfDisposed(); - if (_searcherManager != null) - return _searcherManager; - - if (_writer != null) - { - _searcherManager = new SearcherManager(_writer, true, null); - _searcherIsNrt = true; - return _searcherManager; - } - - if (!DirectoryReader.IndexExists(Store)) - return null; - - _searcherManager = new SearcherManager(Store, null); - _searcherIsNrt = false; - return _searcherManager; - } - } - - public void MaybeRefresh() - { - SearcherManager manager; - lock (_sync) - { - manager = _searcherManager; - } - - try { manager?.MaybeRefresh(); } - catch (Exception ex) { Logging.LogException(ex, "Records search reader refresh failed."); } - } - - /// Serializes mutations with the committed-segment erasure pass. - public int Write(Func mutation) - { - lock (_sync) { ThrowIfDisposed(); return mutation(GetWriter()); } - } - - public void ExpungeDeletes() + /// Test seam: host any directory (e.g. RAMDirectory) without touching the configured path. + public LuceneRecordsIndexHost(Directory directory, bool ownsDirectory = false, ISearchIndexStore store = null) + : base(SearchIndexNames.Records, new StandardAnalyzer(RecordsIndexFields.Version), directory, ownsDirectory, store) { - lock (_sync) - { - ThrowIfDisposed(); - var writer = GetWriter(); - writer.ForceMergeDeletes(true); - writer.Commit(); - _searcherManager?.MaybeRefreshBlocking(); - writer.DeleteUnusedFiles(); - using var committed = DirectoryReader.Open(Store); - if (committed.HasDeletions) throw new InvalidOperationException("Deleted records remain in the committed index; erasure cannot be acknowledged."); - } } + } - private static Directory OpenConfiguredDirectory() + /// The Unified Search global index: every Tier 1 entity family through SearchProjections (plan R3, R4 Phase 2). + public sealed class LuceneGlobalIndexHost : LuceneIndexHost + { + public LuceneGlobalIndexHost() + : this(NullSearchIndexStore.Instance) { - var path = Path.Combine(SearchConfig.IndexPath ?? string.Empty, RecordsIndexFields.IndexName); - System.IO.Directory.CreateDirectory(path); - return FSDirectory.Open(path); } - private void ThrowIfDisposed() + public LuceneGlobalIndexHost(ISearchIndexStore store) + : base(SearchIndexNames.Global, GlobalIndexFields.CreateIndexAnalyzer(), store) { - if (_disposed) - throw new ObjectDisposedException(nameof(LuceneRecordsIndexHost)); } - public void Dispose() + public LuceneGlobalIndexHost(Directory directory, bool ownsDirectory = false, ISearchIndexStore store = null) + : base(SearchIndexNames.Global, GlobalIndexFields.CreateIndexAnalyzer(), directory, ownsDirectory, store) { - lock (_sync) - { - if (_disposed) - return; - _disposed = true; - - try { _searcherManager?.Dispose(); } catch (Exception ex) { Logging.LogException(ex); } - try { _writer?.Dispose(); } catch (Exception ex) { Logging.LogException(ex); } - // _directory, never Store: a host that was disposed without ever indexing must not open the - // configured path on its way out. - if (_ownsDirectory && _directory != null) - { - try { _directory.Dispose(); } catch (Exception ex) { Logging.LogException(ex); } - } - try { Analyzer.Dispose(); } catch (Exception ex) { Logging.LogException(ex); } - } } } } diff --git a/Core/Resgrid.Search/LuceneRecordsIndexer.cs b/Core/Resgrid.Search/LuceneRecordsIndexer.cs index 11c8ff370..f15802bcc 100644 --- a/Core/Resgrid.Search/LuceneRecordsIndexer.cs +++ b/Core/Resgrid.Search/LuceneRecordsIndexer.cs @@ -12,17 +12,25 @@ namespace Resgrid.Search { /// /// Write side of the records index: upsert by document key, delete by key or by department, explicit commit. - /// Holds no state of its own; the host owns the single writer. + /// Holds no state of its own; the host owns the single writer. Commit and erasure publish to the object store + /// under the database lease when one is configured (plan R7); tests construct it without a lease repository. /// public class LuceneRecordsIndexer : IRecordsSearchIndexer { private readonly LuceneRecordsIndexHost _host; private readonly IRmsSearchWriteFence _fence; + private readonly ISearchIndexLeasesRepository _leases; public LuceneRecordsIndexer(LuceneRecordsIndexHost host, IRmsSearchWriteFence fence) + : this(host, fence, null) + { + } + + public LuceneRecordsIndexer(LuceneRecordsIndexHost host, IRmsSearchWriteFence fence, ISearchIndexLeasesRepository leases) { _host = host ?? throw new ArgumentNullException(nameof(host)); _fence = fence ?? throw new ArgumentNullException(nameof(fence)); + _leases = leases; } public async Task IndexAsync(IEnumerable documents, CancellationToken cancellationToken = default) @@ -73,16 +81,13 @@ public Task DeleteDepartmentAsync(int departmentId, CancellationToken cancellati public Task CommitAsync(CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); - _host.Write(writer => { writer.Commit(); return 0; }); - _host.MaybeRefresh(); - return Task.CompletedTask; + return SearchIndexPublishCoordinator.CommitAndPublishAsync(_host, _leases, cancellationToken); } public Task ExpungeDeletesAsync(CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); - _host.ExpungeDeletes(); - return Task.CompletedTask; + return SearchIndexPublishCoordinator.ExpungeAndPublishAsync(_host, _leases, cancellationToken); } public Task CountDocumentsAsync(int departmentId) diff --git a/Core/Resgrid.Search/Resgrid.Search.csproj b/Core/Resgrid.Search/Resgrid.Search.csproj index 210bd21a0..c37dd1bee 100644 --- a/Core/Resgrid.Search/Resgrid.Search.csproj +++ b/Core/Resgrid.Search/Resgrid.Search.csproj @@ -12,6 +12,8 @@ + + diff --git a/Core/Resgrid.Search/SearchIndexPublishCoordinator.cs b/Core/Resgrid.Search/SearchIndexPublishCoordinator.cs new file mode 100644 index 000000000..b918f1c28 --- /dev/null +++ b/Core/Resgrid.Search/SearchIndexPublishCoordinator.cs @@ -0,0 +1,81 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Resgrid.Config; +using Resgrid.Framework; +using Resgrid.Model.Repositories; +using Resgrid.Model.Search; + +namespace Resgrid.Search +{ + /// + /// Commit-and-publish under the database publish lease (plan R7 writer sequence steps 2–6). The lease is the second + /// guard after the Recreate rollout strategy; the manifest's conditional PUT is the third. Without an object store + /// this is just a commit. A manifest conflict resets the local writer from the store and rethrows so the caller's + /// checkpoint is not advanced; the next sweep re-indexes from the projection tables. + /// + public static class SearchIndexPublishCoordinator + { + private static readonly string Owner = $"{Environment.MachineName}:{Environment.ProcessId}"; + + public static async Task CommitAndPublishAsync(LuceneIndexHost host, ISearchIndexLeasesRepository leases, CancellationToken cancellationToken) + { + host.Commit(); + await PublishAsync(host, leases, cancellationToken); + } + + public static async Task ExpungeAndPublishAsync(LuceneIndexHost host, ISearchIndexLeasesRepository leases, CancellationToken cancellationToken) + { + host.ExpungeDeletes(); + // Erasure is acknowledged by the caller only after this returns: the superseded segment objects must be + // gone from the bucket as well (plan R7 consequences), so a publish failure here must propagate. + await PublishAsync(host, leases, cancellationToken); + } + + public static async Task PublishAsync(LuceneIndexHost host, ISearchIndexLeasesRepository leases, CancellationToken cancellationToken) + { + if (host == null || !host.StoreEnabled) + { + host?.MaybeRefresh(); + return; + } + + var leaseHeld = false; + var duration = TimeSpan.FromSeconds(Math.Max(30, SearchConfig.PublishLeaseSeconds)); + if (leases != null) + { + leaseHeld = await leases.TryAcquireAsync(host.IndexName, Owner, duration, DateTime.UtcNow, cancellationToken); + if (!leaseHeld) + throw new InvalidOperationException($"Search index '{host.IndexName}' publish lease is held by another writer; not publishing."); + } + + try + { + SearchIndexManifest manifest; + try + { + manifest = await host.PublishAsync(Owner, cancellationToken); + } + catch (SearchIndexManifestConflictException ex) + { + Logging.LogException(ex, $"Search index '{host.IndexName}' manifest conflict; resetting the local writer from the object store."); + try { await host.ResetFromStoreAsync(cancellationToken); } + catch (Exception reset) { Logging.LogException(reset, $"Search index '{host.IndexName}' reset after conflict failed."); } + throw; + } + + if (manifest != null && leases != null && leaseHeld) + await leases.RecordPublishedAsync(host.IndexName, Owner, manifest.Revision, DateTime.UtcNow, cancellationToken); + } + finally + { + if (leases != null && leaseHeld) + { + try { await leases.ReleaseAsync(host.IndexName, Owner, CancellationToken.None); } + catch (Exception ex) { Logging.LogException(ex, $"Search index '{host.IndexName}' publish lease release failed; it expires on its own."); } + } + host.MaybeRefresh(); + } + } + } +} diff --git a/Core/Resgrid.Search/SearchModule.cs b/Core/Resgrid.Search/SearchModule.cs index dd1f41285..f8d4aba53 100644 --- a/Core/Resgrid.Search/SearchModule.cs +++ b/Core/Resgrid.Search/SearchModule.cs @@ -1,20 +1,32 @@ using Autofac; +using Resgrid.Model.Providers; using Resgrid.Model.Services; namespace Resgrid.Search { /// - /// Registers the shared Lucene host (one per process) and the records index services. Reader and writer - /// share the host; which side a process uses is decided by who calls it, so the same module serves Web, - /// API and Worker. + /// Registers the object store (one per process), one shared Lucene host per index (records, global) and the read / + /// write services over them. Reader and writer share the host; which side a process uses is decided by who calls + /// it, so the same module serves Web, API and Worker. /// public class SearchModule : Module { protected override void Load(ContainerBuilder builder) { + builder.Register(c => + { + var s3 = new S3SearchIndexStore(); + return s3.Enabled ? (ISearchIndexStore)s3 : NullSearchIndexStore.Instance; + }).As().SingleInstance(); + builder.RegisterType().AsSelf().SingleInstance(); + builder.RegisterType().AsSelf().SingleInstance(); + builder.RegisterType().As().InstancePerLifetimeScope(); builder.RegisterType().As().InstancePerLifetimeScope(); + + builder.RegisterType().As().InstancePerLifetimeScope(); + builder.RegisterType().As().InstancePerLifetimeScope(); } } } diff --git a/Core/Resgrid.Search/Store/NullSearchIndexStore.cs b/Core/Resgrid.Search/Store/NullSearchIndexStore.cs new file mode 100644 index 000000000..3b4ef9df6 --- /dev/null +++ b/Core/Resgrid.Search/Store/NullSearchIndexStore.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Resgrid.Model.Providers; +using Resgrid.Model.Search; + +namespace Resgrid.Search +{ + /// No object store configured: every host is a plain local-directory host (single-host Compose, tests). + public sealed class NullSearchIndexStore : ISearchIndexStore + { + public static readonly NullSearchIndexStore Instance = new NullSearchIndexStore(); + + public bool Enabled => false; + + public Task GetManifestAsync(string indexName, CancellationToken cancellationToken = default) => Task.FromResult(null); + + public Task> ListFilesAsync(string indexName, CancellationToken cancellationToken = default) => Task.FromResult(new HashSet()); + + public Task UploadFileAsync(string indexName, string fileName, string localPath, CancellationToken cancellationToken = default) => throw Disabled(); + + public Task DownloadFileAsync(string indexName, string fileName, string localPath, CancellationToken cancellationToken = default) => throw Disabled(); + + public Task DeleteFilesAsync(string indexName, IEnumerable fileNames, CancellationToken cancellationToken = default) => throw Disabled(); + + public Task PutManifestAsync(string indexName, SearchIndexManifest manifest, string expectedETag, CancellationToken cancellationToken = default) => throw Disabled(); + + private static InvalidOperationException Disabled() => new InvalidOperationException("No search index object store is configured (SearchConfig.S3Endpoint is empty)."); + } +} diff --git a/Core/Resgrid.Search/Store/S3SearchIndexStore.cs b/Core/Resgrid.Search/Store/S3SearchIndexStore.cs new file mode 100644 index 000000000..acfdfb540 --- /dev/null +++ b/Core/Resgrid.Search/Store/S3SearchIndexStore.cs @@ -0,0 +1,168 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using Amazon; +using Amazon.Runtime; +using Amazon.S3; +using Amazon.S3.Model; +using Newtonsoft.Json; +using Resgrid.Config; +using Resgrid.Model.Providers; +using Resgrid.Model.Search; + +namespace Resgrid.Search +{ + /// + /// RustFS / S3-compatible implementation of the search index object store (Unified Search plan R7, verified against + /// RustFS 1.0.0 on 2026-09-17: conditional PUT and bucket default encryption are CI-gated there). Objects live under + /// {S3Prefix}/{indexName}/{fileName}; the manifest is {S3Prefix}/{indexName}/manifest.json. The manifest PUT is + /// conditional: If-None-Match: * on first publish, If-Match: {etag} afterwards, so a stale writer can never win. + /// ETags are passed through exactly as the store returned them (quoted), which is what the store expects. + /// Encryption at rest is the bucket's default encryption; no SSE headers are sent per object. + /// + public sealed class S3SearchIndexStore : ISearchIndexStore, IDisposable + { + private const string ManifestName = "manifest.json"; + private readonly Lazy _client; + + public S3SearchIndexStore() + { + _client = new Lazy(CreateClient, LazyThreadSafetyMode.ExecutionAndPublication); + } + + /// Test seam. + public S3SearchIndexStore(IAmazonS3 client) + { + _client = new Lazy(() => client); + } + + public bool Enabled => !string.IsNullOrWhiteSpace(SearchConfig.S3Endpoint) && !string.IsNullOrWhiteSpace(SearchConfig.S3Bucket); + + private static IAmazonS3 CreateClient() + { + var config = new AmazonS3Config + { + ServiceURL = SearchConfig.S3Endpoint, + ForcePathStyle = SearchConfig.S3ForcePathStyle, + UseHttp = !SearchConfig.S3UseSsl, + AuthenticationRegion = string.IsNullOrWhiteSpace(SearchConfig.S3Region) ? "us-east-1" : SearchConfig.S3Region + }; + var credentials = new BasicAWSCredentials(SearchConfig.S3AccessKey ?? string.Empty, SearchConfig.S3SecretKey ?? string.Empty); + return new AmazonS3Client(credentials, config); + } + + private static string Prefix(string indexName) + { + var prefix = (SearchConfig.S3Prefix ?? string.Empty).Trim('/'); + return string.IsNullOrEmpty(prefix) ? indexName + "/" : prefix + "/" + indexName + "/"; + } + + private static string Key(string indexName, string fileName) => Prefix(indexName) + fileName; + + public async Task GetManifestAsync(string indexName, CancellationToken cancellationToken = default) + { + try + { + using var response = await _client.Value.GetObjectAsync(new GetObjectRequest { BucketName = SearchConfig.S3Bucket, Key = Key(indexName, ManifestName) }, cancellationToken); + using var reader = new StreamReader(response.ResponseStream); + var json = await reader.ReadToEndAsync(); + var manifest = JsonConvert.DeserializeObject(json) ?? new SearchIndexManifest { IndexName = indexName }; + manifest.ETag = response.ETag; + return manifest; + } + catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound || string.Equals(ex.ErrorCode, "NoSuchKey", StringComparison.OrdinalIgnoreCase)) + { + return null; + } + } + + public async Task> ListFilesAsync(string indexName, CancellationToken cancellationToken = default) + { + var prefix = Prefix(indexName); + var names = new HashSet(StringComparer.Ordinal); + string token = null; + do + { + var response = await _client.Value.ListObjectsV2Async(new ListObjectsV2Request { BucketName = SearchConfig.S3Bucket, Prefix = prefix, ContinuationToken = token }, cancellationToken); + foreach (var obj in response.S3Objects ?? new List()) + { + var name = obj.Key.Substring(prefix.Length); + if (name.Length == 0 || name.Contains('/') || string.Equals(name, ManifestName, StringComparison.Ordinal)) + continue; + names.Add(name); + } + token = response.IsTruncated == true ? response.NextContinuationToken : null; + } while (token != null); + + return names; + } + + public Task UploadFileAsync(string indexName, string fileName, string localPath, CancellationToken cancellationToken = default) + { + return _client.Value.PutObjectAsync(new PutObjectRequest + { + BucketName = SearchConfig.S3Bucket, + Key = Key(indexName, fileName), + FilePath = localPath, + ContentType = "application/octet-stream", + DisablePayloadSigning = false + }, cancellationToken); + } + + public async Task DownloadFileAsync(string indexName, string fileName, string localPath, CancellationToken cancellationToken = default) + { + using var response = await _client.Value.GetObjectAsync(new GetObjectRequest { BucketName = SearchConfig.S3Bucket, Key = Key(indexName, fileName) }, cancellationToken); + await response.WriteResponseStreamToFileAsync(localPath, false, cancellationToken); + } + + public async Task DeleteFilesAsync(string indexName, IEnumerable fileNames, CancellationToken cancellationToken = default) + { + foreach (var batch in (fileNames ?? Enumerable.Empty()).Distinct().Chunk(1000)) + { + await _client.Value.DeleteObjectsAsync(new DeleteObjectsRequest + { + BucketName = SearchConfig.S3Bucket, + Objects = batch.Select(n => new KeyVersion { Key = Key(indexName, n) }).ToList(), + Quiet = true + }, cancellationToken); + } + } + + public async Task PutManifestAsync(string indexName, SearchIndexManifest manifest, string expectedETag, CancellationToken cancellationToken = default) + { + var request = new PutObjectRequest + { + BucketName = SearchConfig.S3Bucket, + Key = Key(indexName, ManifestName), + ContentBody = JsonConvert.SerializeObject(manifest), + ContentType = "application/json" + }; + if (expectedETag == null) + request.IfNoneMatch = "*"; + else + request.IfMatch = expectedETag; + + try + { + var response = await _client.Value.PutObjectAsync(request, cancellationToken); + manifest.ETag = response.ETag; + return manifest; + } + catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.PreconditionFailed || ex.StatusCode == HttpStatusCode.Conflict + || string.Equals(ex.ErrorCode, "PreconditionFailed", StringComparison.OrdinalIgnoreCase) || string.Equals(ex.ErrorCode, "ConditionalRequestConflict", StringComparison.OrdinalIgnoreCase)) + { + throw new SearchIndexManifestConflictException(indexName, $"Search index '{indexName}' manifest was published by another writer (store returned {(int)ex.StatusCode} {ex.ErrorCode})."); + } + } + + public void Dispose() + { + if (_client.IsValueCreated) + _client.Value.Dispose(); + } + } +} diff --git a/Core/Resgrid.Services/CallsService.cs b/Core/Resgrid.Services/CallsService.cs index b86d3254c..130ec21d3 100644 --- a/Core/Resgrid.Services/CallsService.cs +++ b/Core/Resgrid.Services/CallsService.cs @@ -5,6 +5,7 @@ using Resgrid.Model; using Resgrid.Model.Providers; using Resgrid.Model.Repositories; +using Resgrid.Model.Search; using Resgrid.Model.Services; using Resgrid.Model.Identity; using System.Text; @@ -48,6 +49,8 @@ public class CallsService : ICallsService // (broker client) until a save actually needs it. private readonly Lazy _protectedWriteService; + private readonly Lazy _searchProjections; + public CallsService(ICallsRepository callsRepository, ICommunicationService communicationService, ICallDispatchesRepository callDispatchesRepository, ICallTypesRepository callTypesRepository, ICallEmailFactory callEmailFactory, ICacheProvider cacheProvider, ICallNotesRepository callNotesRepository, @@ -57,7 +60,7 @@ public CallsService(ICallsRepository callsRepository, ICommunicationService comm ICallProtocolsRepository callProtocolsRepository, IGeoLocationProvider geoLocationProvider, IDepartmentsService departmentsService, ICallReferencesRepository callReferencesRepository, ICallContactsRepository callContactsRepository, IIndoorMapService indoorMapService, ICallVideoFeedRepository callVideoFeedRepository, - Lazy protectedWriteService) + Lazy protectedWriteService, Lazy searchProjections = null) { _protectedWriteService = protectedWriteService; _callsRepository = callsRepository; @@ -80,6 +83,7 @@ public CallsService(ICallsRepository callsRepository, ICommunicationService comm _callContactsRepository = callContactsRepository; _indoorMapService = indoorMapService; _callVideoFeedRepository = callVideoFeedRepository; + _searchProjections = searchProjections; } public async Task SaveCallAsync(Call call, CancellationToken cancellationToken = default(CancellationToken)) @@ -237,6 +241,7 @@ public CallsService(ICallsRepository callsRepository, ICommunicationService comm } } + if (_searchProjections != null) await _searchProjections.Value.ProjectCallAsync(savedCall, cancellationToken); return savedCall; } @@ -342,7 +347,9 @@ public async Task> GetClosedCallsByDepartmentYearAsync(int department public async Task DeleteCallByIdAsync(int callId, CancellationToken cancellationToken = default(CancellationToken)) { var call = await GetCallByIdAsync(callId); - return await _callsRepository.DeleteAsync(call, cancellationToken); + var deleted = await _callsRepository.DeleteAsync(call, cancellationToken); + if (deleted && call != null && _searchProjections != null) await _searchProjections.Value.RemoveAsync(call.DepartmentId, SearchEntityTypes.Call, call.CallId.ToString(), cancellationToken); + return deleted; } public async Task ReOpenCallByIdAsync(int callId, CancellationToken cancellationToken = default(CancellationToken)) @@ -354,7 +361,9 @@ public async Task> GetClosedCallsByDepartmentYearAsync(int department call.ClosedOn = null; call.CompletedNotes = null; - return await _callsRepository.SaveOrUpdateAsync(call, cancellationToken); + var softDeleted = await _callsRepository.SaveOrUpdateAsync(call, cancellationToken); + if (_searchProjections != null) await _searchProjections.Value.ProjectCallAsync(softDeleted, cancellationToken); + return softDeleted; } public async Task GetCallByIdAsync(int callId, bool bypassCache = true) diff --git a/Core/Resgrid.Services/ContactsService.cs b/Core/Resgrid.Services/ContactsService.cs index ff3375919..bd824ce66 100644 --- a/Core/Resgrid.Services/ContactsService.cs +++ b/Core/Resgrid.Services/ContactsService.cs @@ -32,12 +32,14 @@ public class ContactsService : IContactsService /// RMS-5: which system owns structure writes; lazy because Records depends on Contacts (RMS plan section 4.3). private readonly Lazy _ownershipGate; + private readonly Lazy _searchProjections; + public ContactsService(IContactsRepository contactsRepository, IContactNotesRepository contactNotesRepository, IContactCategoryRepository contactCategoryRepository, IContactNoteTypesRepository contactNoteTypesRepository, IContactAssociationsRepository contactAssociationsRepository, IContactPreplanRepository contactPreplanRepository, IContactPreplanHazardRepository contactPreplanHazardRepository, IContactAttachmentRepository contactAttachmentRepository, ICallsRepository callsRepository, ICallContactsRepository callContactsRepository, - IEventAggregator eventAggregator, Lazy protectedWriteService, Lazy ownershipGate) + IEventAggregator eventAggregator, Lazy protectedWriteService, Lazy ownershipGate, Lazy searchProjections = null) { _ownershipGate = ownershipGate; _contactsRepository = contactsRepository; @@ -52,6 +54,7 @@ public ContactsService(IContactsRepository contactsRepository, IContactNotesRepo _callContactsRepository = callContactsRepository; _eventAggregator = eventAggregator; _protectedWriteService = protectedWriteService; + _searchProjections = searchProjections; } public async Task> GetAllContactsForDepartmentAsync(int departmentId) @@ -114,6 +117,7 @@ public async Task> GetContactCategoriesForDepartmentAsync( if (protectedWrite.Changed || (existingContactForRestore != null && protectedWrite.Success)) savedContact = await _contactsRepository.SaveOrUpdateAsync(savedContact, cancellationToken); + if (_searchProjections != null) await _searchProjections.Value.ProjectContactAsync(savedContact, cancellationToken); return savedContact; } diff --git a/Core/Resgrid.Services/DepartmentsService.cs b/Core/Resgrid.Services/DepartmentsService.cs index 354b1f08e..5c811158a 100644 --- a/Core/Resgrid.Services/DepartmentsService.cs +++ b/Core/Resgrid.Services/DepartmentsService.cs @@ -9,6 +9,7 @@ using Resgrid.Model.Events; using Resgrid.Model.Providers; using Resgrid.Model.Repositories; +using Resgrid.Model.Search; using Resgrid.Model.Services; using Resgrid.Providers.Bus; using Resgrid.Model.Identity; @@ -44,11 +45,13 @@ public class DepartmentsService : IDepartmentsService private readonly ILimitsService _limitsService; + private readonly Lazy _searchProjections; + public DepartmentsService(IDepartmentsRepository departmentRepository, IDepartmentMembersRepository departmentMembersRepository, ISubscriptionsService subscriptionsService, IDepartmentCallEmailsRepository departmentCallEmailsRepository, IDepartmentCallPruningRepository departmentCallPruningRepository, ICacheProvider cacheProvider, IUsersService usersService, IDepartmentSettingsService departmentSettingsService, IUserProfileService userProfileRepository, ILimitsService limitsService, - IEventAggregator eventAggregator, IIdentityRepository identityRepository, IDepartmentCallPruningRepository departmentCallPruningDapperRepository) + IEventAggregator eventAggregator, IIdentityRepository identityRepository, IDepartmentCallPruningRepository departmentCallPruningDapperRepository, Lazy searchProjections = null) { _departmentRepository = departmentRepository; _departmentMembersRepository = departmentMembersRepository; @@ -63,6 +66,7 @@ public DepartmentsService(IDepartmentsRepository departmentRepository, IDepartme _identityRepository = identityRepository; _departmentCallPruningDapperRepository = departmentCallPruningDapperRepository; _limitsService = limitsService; + _searchProjections = searchProjections; } #endregion Private Members and Constructors @@ -341,6 +345,7 @@ private void SendMembershipVisibilityRefresh(int departmentId) { member.IsDeleted = true; await _departmentMembersRepository.SaveOrUpdateAsync(member, cancellationToken); + if (_searchProjections != null) await _searchProjections.Value.RemoveAsync(departmentId, SearchEntityTypes.Personnel, userIdToDelete, cancellationToken); var member2 = await _departmentMembersRepository.GetDepartmentMemberByDepartmentIdAndUserIdAsync(departmentId, userIdToDelete); @@ -719,6 +724,7 @@ async Task getDepartmentMember() public async Task SaveDepartmentMemberAsync(DepartmentMember departmentMember, CancellationToken cancellationToken = default(CancellationToken)) { var saved = await _departmentMembersRepository.SaveOrUpdateAsync(departmentMember, cancellationToken); + if (_searchProjections != null && saved != null && (saved.IsDeleted || !saved.IsActive)) await _searchProjections.Value.RemoveAsync(saved.DepartmentId, SearchEntityTypes.Personnel, saved.UserId, cancellationToken); InvalidateDepartmentMemberInCache(departmentMember.UserId, departmentMember.DepartmentId); InvalidateDepartmentUserInCache(departmentMember.UserId, departmentMember.User); diff --git a/Core/Resgrid.Services/DocumentsService.cs b/Core/Resgrid.Services/DocumentsService.cs index 94d7b6d6a..2f9dce7ef 100644 --- a/Core/Resgrid.Services/DocumentsService.cs +++ b/Core/Resgrid.Services/DocumentsService.cs @@ -8,6 +8,7 @@ using Resgrid.Model.Providers; using Resgrid.Model.Repositories; using Resgrid.Model.Repositories.Queries; +using Resgrid.Model.Search; using Resgrid.Model.Services; using Resgrid.Providers.Bus; using Resgrid.Repositories.DataRepository; @@ -22,15 +23,18 @@ public class DocumentsService : IDocumentsService private readonly IEventAggregator _eventAggregator; private readonly IUnitOfWork _unitOfWork; + private readonly Lazy _searchProjections; + public DocumentsService(IDocumentRepository documentRepository, IDocumentCategoriesRepository documentCategoriesRepository, IEventAggregator eventAggregator, Lazy protectedWriteService, - IUnitOfWork unitOfWork) + IUnitOfWork unitOfWork, Lazy searchProjections = null) { _protectedWriteService = protectedWriteService; _documentRepository = documentRepository; _documentCategoriesRepository = documentCategoriesRepository; _eventAggregator = eventAggregator; _unitOfWork = unitOfWork; + _searchProjections = searchProjections; } public async Task> GetAllDocumentsByDepartmentIdAsync(int departmentId) @@ -91,7 +95,9 @@ public async Task> GetFilteredDocumentsByDepartmentIdAsync(int de if (!preSaveWrite.Success) throw new InvalidOperationException($"Protected write blocked ({preSaveWrite.Reason}); document {document.DocumentId} was NOT saved."); - return await _documentRepository.SaveOrUpdateAsync(document, cancellationToken); + var updated = await _documentRepository.SaveOrUpdateAsync(document, cancellationToken); + if (_searchProjections != null) await _searchProjections.Value.ProjectDocumentAsync(updated, cancellationToken); + return updated; } // An INSERT cannot be enveloped first: the AAD row key IS the identity pk, and only the @@ -132,6 +138,7 @@ public async Task> GetFilteredDocumentsByDepartmentIdAsync(int de _unitOfWork.CommitChanges(); + if (_searchProjections != null) await _searchProjections.Value.ProjectDocumentAsync(saved, cancellationToken); return saved; } catch (Exception ex) @@ -161,7 +168,9 @@ public async Task GetDocumentByIdAsync(int documentId) public async Task DeleteDocumentAsync(Document document, CancellationToken cancellationToken = default(CancellationToken)) { - return await _documentRepository.DeleteAsync(document, cancellationToken); + var deleted = await _documentRepository.DeleteAsync(document, cancellationToken); + if (deleted && document != null && _searchProjections != null) await _searchProjections.Value.RemoveAsync(document.DepartmentId, SearchEntityTypes.Document, document.DocumentId.ToString(), cancellationToken); + return deleted; } public async Task SaveDocumentCategoryAsync(DocumentCategory category, CancellationToken cancellationToken = default(CancellationToken)) diff --git a/Core/Resgrid.Services/MessageService.cs b/Core/Resgrid.Services/MessageService.cs index f6bfba83d..e7fed08e4 100644 --- a/Core/Resgrid.Services/MessageService.cs +++ b/Core/Resgrid.Services/MessageService.cs @@ -21,11 +21,13 @@ public class MessageService : IMessageService private readonly IMessageRecipientRepository _messageRecipientRepository; private readonly Lazy _protectedWriteService; + private readonly Lazy _searchProjections; + public MessageService(IMessageRepository messageRepository, IPushService pushService, ICommunicationService communicationService, IQueueService queueService, IUserProfileService userProfileService, IMessageRecipientRepository messageRecipientRepository, - Lazy protectedWriteService) + Lazy protectedWriteService, Lazy searchProjections = null) { _messageRepository = messageRepository; _pushService = pushService; @@ -34,6 +36,7 @@ public MessageService(IMessageRepository messageRepository, IPushService pushSer _userProfileService = userProfileService; _messageRecipientRepository = messageRecipientRepository; _protectedWriteService = protectedWriteService; + _searchProjections = searchProjections; } public async Task GetMessageByIdAsync(int messageId) @@ -90,6 +93,7 @@ public async Task GetMessageByIdAsync(int messageId) if (protectedWrite.Changed || recipientChanged) saved = await _messageRepository.SaveOrUpdateAsync(saved, cancellationToken); + if (_searchProjections != null) await _searchProjections.Value.ProjectMessageAsync(saved, cancellationToken); return saved; } diff --git a/Core/Resgrid.Services/NotesService.cs b/Core/Resgrid.Services/NotesService.cs index 9783f384b..e3ab1ea41 100644 --- a/Core/Resgrid.Services/NotesService.cs +++ b/Core/Resgrid.Services/NotesService.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -7,6 +8,7 @@ using Resgrid.Model.Events; using Resgrid.Model.Providers; using Resgrid.Model.Repositories; +using Resgrid.Model.Search; using Resgrid.Model.Services; using Resgrid.Providers.Bus; using Resgrid.Repositories.DataRepository; @@ -19,11 +21,14 @@ public class NotesService : INotesService private readonly IEventAggregator _eventAggregator; private readonly INoteCategoriesRepository _noteCategoriesRepository; - public NotesService(INotesRepository notesRepository, IEventAggregator eventAggregator, INoteCategoriesRepository noteCategoriesRepository) + private readonly Lazy _searchProjections; + + public NotesService(INotesRepository notesRepository, IEventAggregator eventAggregator, INoteCategoriesRepository noteCategoriesRepository, Lazy searchProjections = null) { _notesRepository = notesRepository; _eventAggregator = eventAggregator; _noteCategoriesRepository = noteCategoriesRepository; + _searchProjections = searchProjections; } public async Task> GetAllNotesForDepartmentAsync(int departmentId) @@ -45,6 +50,7 @@ public async Task> GetAllNotesForDepartmentAsync(int departmentId) var saved = await _notesRepository.SaveOrUpdateAsync(note, cancellationToken); _eventAggregator.SendMessage(new NoteAddedEvent() { DepartmentId = note.DepartmentId, Note = note }); + if (_searchProjections != null) await _searchProjections.Value.ProjectNoteAsync(saved, cancellationToken); return saved; } @@ -65,7 +71,9 @@ public async Task GetNoteByIdAsync(int noteId) public async Task DeleteAsync(Note note, CancellationToken cancellationToken = default(CancellationToken)) { - return await _notesRepository.DeleteAsync(note, cancellationToken); + var deleted = await _notesRepository.DeleteAsync(note, cancellationToken); + if (deleted && note != null && _searchProjections != null) await _searchProjections.Value.RemoveAsync(note.DepartmentId, SearchEntityTypes.Note, note.NoteId.ToString(), cancellationToken); + return deleted; } public async Task> GetNotesForDepartmentFilteredAsync(int departmentId, bool isAdmin) diff --git a/Core/Resgrid.Services/Records/RecordsSearchIndexMaintenanceService.cs b/Core/Resgrid.Services/Records/RecordsSearchIndexMaintenanceService.cs index d62b31e50..86ab8d3a1 100644 --- a/Core/Resgrid.Services/Records/RecordsSearchIndexMaintenanceService.cs +++ b/Core/Resgrid.Services/Records/RecordsSearchIndexMaintenanceService.cs @@ -67,6 +67,10 @@ public async Task SweepAsync(CancellationToken ca var generation = await ComputeGenerationAsync(cutover.DepartmentId); var state = await _states.GetAsync(cutover.DepartmentId, RmsSearchIndexState.RecordsIndexName); var needsRebuild = state == null || state.State != (int)RmsSearchIndexBuildState.Ready || !string.Equals(state.Generation, generation, StringComparison.Ordinal); + // Missing-index rule (Unified Search plan R7): a fresh pod, an empty bucket after first deploy, or a wiped + // cache leaves the state row Ready with N documents while the local index holds none for the department. + if (!needsRebuild && state.DocumentCount > 0 && await _indexer.CountDocumentsAsync(cutover.DepartmentId) == 0) + needsRebuild = true; if (needsRebuild) { diff --git a/Core/Resgrid.Services/Search/SearchIndexMaintenanceService.cs b/Core/Resgrid.Services/Search/SearchIndexMaintenanceService.cs new file mode 100644 index 000000000..bdeacd01b --- /dev/null +++ b/Core/Resgrid.Services/Search/SearchIndexMaintenanceService.cs @@ -0,0 +1,423 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Resgrid.Config; +using Resgrid.Framework; +using Resgrid.Model; +using Resgrid.Model.Repositories; +using Resgrid.Model.Search; +using Resgrid.Model.Services; + +namespace Resgrid.Services.Search +{ + /// + /// Worker command 70 (plan R4 Phase 2, registry §4F): per department with a state row for the global index, compare + /// the stored generation key with (schemaVersion, protectedCatalogVersion, policyEpoch); rebuild on mismatch, on + /// an admin request, or when the local index is empty while the state says otherwise (missing-index rule, R7); + /// otherwise catch up rows modified since the last sweep. A rebuild regenerates the projection rows from the + /// entity services first, then re-indexes them, so the projection table stays the source of truth and the index is + /// always rebuildable from it. Departments enter the sweep lazily: the first unified search or an admin request + /// creates their state row. + /// + public class SearchIndexMaintenanceService : ISearchIndexMaintenanceService + { + private readonly ISearchIndexStatesRepository _states; + private readonly ISearchProjectionsRepository _projections; + private readonly IGlobalSearchIndexer _indexer; + private readonly IDepartmentDataProtectionService _dataProtection; + private readonly IFeatureToggleService _featureToggles; + private readonly ISearchProjectionService _projectionService; + private readonly ICallsService _calls; + private readonly IUnitsService _units; + private readonly IUserProfileService _profiles; + private readonly IDepartmentsService _departments; + private readonly IDepartmentGroupsService _groups; + private readonly IContactsService _contacts; + private readonly IMessageService _messages; + private readonly IDocumentsService _documents; + private readonly INotesService _notes; + + public SearchIndexMaintenanceService(ISearchIndexStatesRepository states, ISearchProjectionsRepository projections, IGlobalSearchIndexer indexer, + IDepartmentDataProtectionService dataProtection, IFeatureToggleService featureToggles, ISearchProjectionService projectionService, + ICallsService calls, IUnitsService units, IUserProfileService profiles, IDepartmentsService departments, IDepartmentGroupsService groups, + IContactsService contacts, IMessageService messages, IDocumentsService documents, INotesService notes) + { + _states = states; + _projections = projections; + _indexer = indexer; + _dataProtection = dataProtection; + _featureToggles = featureToggles; + _projectionService = projectionService; + _calls = calls; + _units = units; + _profiles = profiles; + _departments = departments; + _groups = groups; + _contacts = contacts; + _messages = messages; + _documents = documents; + _notes = notes; + } + + public async Task SweepAsync(CancellationToken cancellationToken = default) + { + var result = new SearchIndexSweepResult(); + if (!SearchConfig.Enabled) + { + result.Skipped = true; + result.Message = "Search host disabled; global index sweep skipped."; + return result; + } + + var states = (await _states.GetAllForIndexAsync(SearchIndexNames.Global))?.ToList() ?? new List(); + var rebuilds = 0; + + foreach (var state in states) + { + cancellationToken.ThrowIfCancellationRequested(); + result.DepartmentsChecked++; + + try + { + if (!await FlagOnAsync(state.DepartmentId)) + continue; + + var generation = await ComputeGenerationAsync(state.DepartmentId); + var needsRebuild = state.State != (int)SearchIndexBuildState.Ready + || state.RebuildRequestedOn.HasValue + || !string.Equals(state.Generation, generation, StringComparison.Ordinal); + + // Missing-index rule (plan R7): fresh pod, empty bucket or wiped cache. + if (!needsRebuild && state.DocumentCount > 0 && await _indexer.CountDocumentsAsync(state.DepartmentId) == 0) + needsRebuild = true; + + if (needsRebuild) + { + if (rebuilds >= Math.Max(1, SearchConfig.MaxRebuildsPerSweep)) + continue; + rebuilds++; + await RebuildAsync(state.DepartmentId, generation, state, result, cancellationToken); + } + else + { + await CatchUpAsync(state.DepartmentId, generation, state, result, cancellationToken); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + result.Errors++; + Logging.LogException(ex, $"Global search index maintenance failed for department {state.DepartmentId}."); + } + } + + result.Message = $"Checked {result.DepartmentsChecked} department(s); rebuilt {result.DepartmentsRebuilt} ({result.ProjectionsRebuilt} projections); indexed {result.DocumentsIndexed}; deleted {result.DocumentsDeleted}; errors {result.Errors}."; + return result; + } + + public async Task RebuildDepartmentAsync(int departmentId, CancellationToken cancellationToken = default) + { + var result = new SearchIndexSweepResult { DepartmentsChecked = 1 }; + if (!SearchConfig.Enabled) + { + result.Skipped = true; + result.Message = "Search host disabled."; + return result; + } + + var generation = await ComputeGenerationAsync(departmentId); + var state = await _states.GetAsync(SearchIndexNames.Global, departmentId); + await RebuildAsync(departmentId, generation, state, result, cancellationToken); + result.Message = $"Rebuilt department {departmentId}: {result.ProjectionsRebuilt} projection(s), {result.DocumentsIndexed} document(s)."; + return result; + } + + public async Task RequestRebuildAsync(int departmentId, CancellationToken cancellationToken = default) + { + var now = DateTime.UtcNow; + var state = await _states.GetAsync(SearchIndexNames.Global, departmentId) + ?? new SearchIndexState { IndexName = SearchIndexNames.Global, DepartmentId = departmentId, CreatedOn = now, Generation = GlobalSearchGeneration.Compute(0, 0), SchemaVersion = GlobalSearchGeneration.SchemaVersion }; + state.State = (int)SearchIndexBuildState.RebuildRequested; + state.RebuildRequestedOn = now; + state.ModifiedOn = now; + return await _states.SaveOrUpdateAsync(state, cancellationToken, true); + } + + private async Task RebuildAsync(int departmentId, string generation, SearchIndexState state, SearchIndexSweepResult result, CancellationToken cancellationToken) + { + var now = DateTime.UtcNow; + state = state ?? new SearchIndexState { IndexName = SearchIndexNames.Global, DepartmentId = departmentId, CreatedOn = now }; + state.State = (int)SearchIndexBuildState.Rebuilding; + state.Generation = generation; + ApplyGeneration(state, generation); + state.ModifiedOn = now; + state = await _states.SaveOrUpdateAsync(state, cancellationToken, true); + + try + { + result.ProjectionsRebuilt += await RebuildProjectionsAsync(departmentId, cancellationToken); + + await _indexer.DeleteDepartmentAsync(departmentId, cancellationToken); + + DateTime? lastModified = null; + var indexed = 0; + var skip = 0; + var batch = Math.Max(50, SearchConfig.IndexBatchSize); + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + var page = (await _projections.GetLivePageAsync(departmentId, skip, batch))?.ToList() ?? new List(); + if (page.Count == 0) + break; + indexed += await _indexer.IndexAsync(page, generation, cancellationToken); + lastModified = Max(lastModified, page.Max(p => p.ModifiedOn)); + skip += page.Count; + if (page.Count < batch) + break; + } + + // The durable checkpoint must never lead the committed (and published) segments. + await _indexer.CommitAsync(cancellationToken); + state.State = (int)SearchIndexBuildState.Ready; + state.DocumentCount = indexed; + state.LastRebuiltOn = DateTime.UtcNow; + state.LastIndexedModifiedOn = lastModified.HasValue && lastModified.Value > now ? now : lastModified; + state.RebuildRequestedOn = null; + state.ModifiedOn = DateTime.UtcNow; + await _states.SaveOrUpdateAsync(state, cancellationToken, true); + + result.DepartmentsRebuilt++; + result.DocumentsIndexed += indexed; + } + catch + { + state.State = (int)SearchIndexBuildState.Failed; + state.ModifiedOn = DateTime.UtcNow; + await _states.SaveOrUpdateAsync(state, CancellationToken.None, true); + throw; + } + } + + private async Task CatchUpAsync(int departmentId, string generation, SearchIndexState state, SearchIndexSweepResult result, CancellationToken cancellationToken) + { + var checkpoint = state.LastIndexedModifiedOn; + var since = checkpoint.HasValue && checkpoint.Value > DateTime.MinValue.AddSeconds(1) ? checkpoint.Value.AddSeconds(-1) : checkpoint; + string sinceId = null; + var batch = Math.Max(50, Math.Min(5000, SearchConfig.IndexBatchSize)); + var touched = false; + + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + var page = (await _projections.GetModifiedSinceAsync(departmentId, since, batch, sinceId))?.ToList() ?? new List(); + if (page.Count == 0) + break; + + var deleted = page.Where(p => p.DeletedOn.HasValue).ToList(); + var live = page.Where(p => !p.DeletedOn.HasValue).ToList(); + + foreach (var gone in deleted) + await _indexer.DeleteAsync(departmentId, gone.EntityType, gone.EntityId, cancellationToken); + + result.DocumentsIndexed += await _indexer.IndexAsync(live, generation, cancellationToken); + result.DocumentsDeleted += deleted.Count; + touched = true; + + var last = page[page.Count - 1]; + if (since.HasValue && (last.ModifiedOn < since.Value || last.ModifiedOn == since.Value && string.Equals(last.SearchProjectionId, sinceId, StringComparison.Ordinal))) + throw new InvalidOperationException("The search change cursor did not advance; its checkpoint was not saved."); + since = last.ModifiedOn; + sinceId = last.SearchProjectionId; + checkpoint = Max(checkpoint, last.ModifiedOn); + + if (page.Count < batch) + break; + } + + if (touched) + { + await _indexer.CommitAsync(cancellationToken); + state.LastIndexedModifiedOn = checkpoint; + state.DocumentCount = await _indexer.CountDocumentsAsync(departmentId); + state.ModifiedOn = DateTime.UtcNow; + await _states.SaveOrUpdateAsync(state, cancellationToken, true); + } + } + + /// Regenerates every projection row of the department from the entity services, then soft-deletes rows no longer present. + private async Task RebuildProjectionsAsync(int departmentId, CancellationToken cancellationToken) + { + var started = DateTime.UtcNow; + var count = 0; + + count += await Family(departmentId, SearchEntityTypes.Call, async () => + { + var calls = new Dictionary(); + foreach (var c in await _calls.GetActiveCallsByDepartmentAsync(departmentId) ?? new List()) + calls[c.CallId] = c; + var year = DateTime.UtcNow.Year; + for (var i = 0; i < Math.Max(1, SearchConfig.CallRebuildYears); i++) + { + foreach (var c in await _calls.GetClosedCallsByDepartmentYearAsync(departmentId, (year - i).ToString()) ?? new List()) + calls[c.CallId] = c; + } + var n = 0; + foreach (var call in calls.Values) + { + cancellationToken.ThrowIfCancellationRequested(); + if (call.IsDeleted) continue; + var p = await _projectionService.BuildCallAsync(call); + if (p != null) { await _projectionService.UpsertAsync(p, cancellationToken); n++; } + } + return n; + }, started, cancellationToken); + + count += await Family(departmentId, SearchEntityTypes.Unit, async () => + { + var n = 0; + foreach (var unit in await _units.GetUnitsForDepartmentUnlimitedAsync(departmentId) ?? new List()) + { + cancellationToken.ThrowIfCancellationRequested(); + var p = await _projectionService.BuildUnitAsync(unit); + if (p != null) { await _projectionService.UpsertAsync(p, cancellationToken); n++; } + } + return n; + }, started, cancellationToken); + + count += await Family(departmentId, SearchEntityTypes.Personnel, async () => + { + var members = await _departments.GetAllMembersForDepartmentAsync(departmentId) ?? new List(); + var profiles = await _profiles.GetAllProfilesForDepartmentIncDisabledDeletedAsync(departmentId) ?? new Dictionary(); + Dictionary groups; + try { groups = await _groups.GetAllDepartmentGroupsForDepartmentAsync(departmentId) ?? new Dictionary(); } + catch (Exception ex) { Logging.LogException(ex); groups = new Dictionary(); } + var n = 0; + foreach (var member in members) + { + cancellationToken.ThrowIfCancellationRequested(); + if (member.IsDeleted || string.IsNullOrWhiteSpace(member.UserId)) continue; + profiles.TryGetValue(member.UserId, out var profile); + if (profile == null) continue; + groups.TryGetValue(member.UserId, out var group); + var p = await _projectionService.BuildPersonnelAsync(departmentId, profile, group?.DepartmentGroupId, member.IsActive && !member.IsDeleted); + if (p != null) { await _projectionService.UpsertAsync(p, cancellationToken); n++; } + } + return n; + }, started, cancellationToken); + + count += await Family(departmentId, SearchEntityTypes.Contact, async () => + { + var n = 0; + foreach (var contact in await _contacts.GetAllContactsForDepartmentAsync(departmentId) ?? new List()) + { + cancellationToken.ThrowIfCancellationRequested(); + if (contact.IsDeleted) continue; + var p = await _projectionService.BuildContactAsync(contact); + if (p != null) { await _projectionService.UpsertAsync(p, cancellationToken); n++; } + } + return n; + }, started, cancellationToken); + + count += await Family(departmentId, SearchEntityTypes.Document, async () => + { + var n = 0; + foreach (var document in await _documents.GetAllDocumentsByDepartmentIdAsync(departmentId) ?? new List()) + { + cancellationToken.ThrowIfCancellationRequested(); + var p = await _projectionService.BuildDocumentAsync(document); + if (p != null) { await _projectionService.UpsertAsync(p, cancellationToken); n++; } + } + return n; + }, started, cancellationToken); + + count += await Family(departmentId, SearchEntityTypes.Note, async () => + { + var n = 0; + foreach (var note in await _notes.GetAllNotesForDepartmentAsync(departmentId) ?? new List()) + { + cancellationToken.ThrowIfCancellationRequested(); + var p = await _projectionService.BuildNoteAsync(note); + if (p != null) { await _projectionService.UpsertAsync(p, cancellationToken); n++; } + } + return n; + }, started, cancellationToken); + + count += await Family(departmentId, SearchEntityTypes.Message, async () => + { + // Messages have no department-wide list; walk the members' sent and inbox folders once, de-duplicated. + var members = await _departments.GetAllMembersForDepartmentAsync(departmentId) ?? new List(); + var seen = new HashSet(); + var n = 0; + foreach (var member in members.Where(m => !m.IsDeleted && !string.IsNullOrWhiteSpace(m.UserId))) + { + cancellationToken.ThrowIfCancellationRequested(); + var folders = new List(); + try { folders.AddRange(await _messages.GetSentMessagesByUserIdAsync(member.UserId) ?? new List()); } catch (Exception ex) { Logging.LogException(ex); } + try { folders.AddRange(await _messages.GetInboxMessagesByUserIdAsync(member.UserId) ?? new List()); } catch (Exception ex) { Logging.LogException(ex); } + foreach (var message in folders) + { + if (message == null || !message.DepartmentId.HasValue || message.DepartmentId.Value != departmentId || message.IsDeleted || !seen.Add(message.MessageId)) + continue; + var p = await _projectionService.BuildMessageAsync(message); + if (p != null) { await _projectionService.UpsertAsync(p, cancellationToken); n++; } + } + } + return n; + }, started, cancellationToken); + + return count; + } + + private async Task Family(int departmentId, string entityType, Func> rebuild, DateTime started, CancellationToken cancellationToken) + { + try + { + var n = await rebuild(); + await _projections.SoftDeleteStaleAsync(departmentId, entityType, started, cancellationToken); + return n; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + Logging.LogException(ex, $"Search projection rebuild failed for family {entityType}; existing rows are kept."); + return 0; + } + } + + private async Task FlagOnAsync(int departmentId) + { + try { return await _featureToggles.IsEnabledAsync(FeatureFlagKeys.SearchUnified, departmentId); } + catch (Exception ex) { Logging.LogException(ex); return false; } + } + + private async Task ComputeGenerationAsync(int departmentId) + { + var catalogVersion = 0; + long policyEpoch = 0; + try { catalogVersion = await _dataProtection.GetPinnedCatalogVersionAsync(departmentId); } catch (Exception ex) { Logging.LogException(ex); } + try { policyEpoch = (await _dataProtection.GetPolicyByDepartmentIdAsync(departmentId))?.PolicyEpoch ?? 0; } catch (Exception ex) { Logging.LogException(ex); } + return GlobalSearchGeneration.Compute(catalogVersion, policyEpoch); + } + + private static void ApplyGeneration(SearchIndexState state, string generation) + { + var parts = (generation ?? string.Empty).Split('.'); + state.SchemaVersion = parts.Length > 0 && int.TryParse(parts[0], out var schema) ? schema : GlobalSearchGeneration.SchemaVersion; + state.ProtectedCatalogVersion = parts.Length > 1 && int.TryParse(parts[1], out var catalog) ? catalog : 0; + state.PolicyEpoch = parts.Length > 2 && long.TryParse(parts[2], out var epoch) ? epoch : 0; + } + + private static DateTime? Max(DateTime? a, DateTime b) + { + return !a.HasValue || b > a.Value ? b : a; + } + } +} diff --git a/Core/Resgrid.Services/Search/SearchProjectionService.cs b/Core/Resgrid.Services/Search/SearchProjectionService.cs new file mode 100644 index 000000000..e1deccb8f --- /dev/null +++ b/Core/Resgrid.Services/Search/SearchProjectionService.cs @@ -0,0 +1,411 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using Newtonsoft.Json; +using Resgrid.Framework; +using Resgrid.Model; +using Resgrid.Model.Repositories; +using Resgrid.Model.Search; +using Resgrid.Model.Services; + +namespace Resgrid.Services.Search +{ + /// + /// Builds and stores the safe search projection for each Tier 1 entity (plan R3). The allowlist is decided here, per + /// family, against the protected-field catalog: a column the catalog protects (call name/nature/address/incident + /// number, every contact field, document name/description/filename, message subject/body, member identification + /// number) is projected only when Advanced Data Protection is not enforced for the department (R2.15, the RMS + /// narrative precedent). A value carrying an envelope prefix or the redaction placeholder is dropped regardless. + /// Enrollment bumps the generation key, and the rebuild that follows re-projects without those columns. + /// + public class SearchProjectionService : ISearchProjectionService + { + private const int TitleMax = 400; + private const int SummaryMax = 1000; + private const int KeywordsMax = 400; + private const int SearchTextMax = 8000; + + private static readonly Regex HtmlTags = new Regex("<[^>]+>", RegexOptions.Compiled); + private static readonly Regex Whitespace = new Regex("\\s+", RegexOptions.Compiled); + + private readonly ISearchProjectionsRepository _projections; + private readonly IDepartmentDataProtectionService _dataProtection; + + public SearchProjectionService(ISearchProjectionsRepository projections, IDepartmentDataProtectionService dataProtection) + { + _projections = projections ?? throw new ArgumentNullException(nameof(projections)); + _dataProtection = dataProtection ?? throw new ArgumentNullException(nameof(dataProtection)); + } + + // ---- hooks ----------------------------------------------------------------------------------------------- + + public Task ProjectCallAsync(Call call, CancellationToken cancellationToken = default) + => Guarded(SearchEntityTypes.Call, call?.DepartmentId ?? 0, call?.CallId.ToString(), call != null && call.IsDeleted, () => BuildCallAsync(call), cancellationToken); + + public Task ProjectUnitAsync(Unit unit, CancellationToken cancellationToken = default) + => Guarded(SearchEntityTypes.Unit, unit?.DepartmentId ?? 0, unit?.UnitId.ToString(), false, () => BuildUnitAsync(unit), cancellationToken); + + public Task ProjectPersonnelAsync(int departmentId, UserProfile profile, int? groupId, bool? isActive, CancellationToken cancellationToken = default) + => Guarded(SearchEntityTypes.Personnel, departmentId, profile?.UserId, false, async () => + { + if (!groupId.HasValue || !isActive.HasValue) + { + var existing = await _projections.GetAsync(departmentId, SearchEntityTypes.Personnel, profile.UserId); + groupId = groupId ?? existing?.GroupId; + isActive = isActive ?? existing?.IsActive ?? true; + } + return await BuildPersonnelAsync(departmentId, profile, groupId, isActive.Value); + }, cancellationToken); + + public Task ProjectContactAsync(Contact contact, CancellationToken cancellationToken = default) + => Guarded(SearchEntityTypes.Contact, contact?.DepartmentId ?? 0, contact?.ContactId, contact != null && contact.IsDeleted, () => BuildContactAsync(contact), cancellationToken); + + public Task ProjectMessageAsync(Message message, CancellationToken cancellationToken = default) + => Guarded(SearchEntityTypes.Message, message?.DepartmentId ?? 0, message?.MessageId.ToString(), message != null && message.IsDeleted, () => BuildMessageAsync(message), cancellationToken); + + public Task ProjectDocumentAsync(Document document, CancellationToken cancellationToken = default) + => Guarded(SearchEntityTypes.Document, document?.DepartmentId ?? 0, document?.DocumentId.ToString(), false, () => BuildDocumentAsync(document), cancellationToken); + + public Task ProjectNoteAsync(Note note, CancellationToken cancellationToken = default) + => Guarded(SearchEntityTypes.Note, note?.DepartmentId ?? 0, note?.NoteId.ToString(), false, () => BuildNoteAsync(note), cancellationToken); + + public async Task RemoveAsync(int departmentId, string entityType, string entityId, CancellationToken cancellationToken = default) + { + if (departmentId <= 0 || string.IsNullOrWhiteSpace(entityType) || string.IsNullOrWhiteSpace(entityId)) + return; + try { await _projections.SoftDeleteAsync(departmentId, entityType, entityId, cancellationToken); } + catch (Exception ex) { Logging.LogException(ex, $"Search projection removal failed for {entityType} {entityId} in department {departmentId}."); } + } + + public async Task UpsertAsync(SearchProjection projection, CancellationToken cancellationToken = default) + { + if (projection == null) + return null; + return await _projections.UpsertAsync(projection, cancellationToken); + } + + private async Task Guarded(string entityType, int departmentId, string entityId, bool deleted, Func> build, CancellationToken cancellationToken) + { + try + { + if (departmentId <= 0 || string.IsNullOrWhiteSpace(entityId) || entityId == "0") + return; + if (deleted) + { + await _projections.SoftDeleteAsync(departmentId, entityType, entityId, cancellationToken); + return; + } + var projection = await build(); + if (projection == null) + await _projections.SoftDeleteAsync(departmentId, entityType, entityId, cancellationToken); + else + await _projections.UpsertAsync(projection, cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + // Never fail the entity write over the search projection; the next rebuild reconciles. + Logging.LogException(ex, $"Search projection failed for {entityType} {entityId} in department {departmentId}."); + } + } + + // ---- builders -------------------------------------------------------------------------------------------- + + public async Task BuildCallAsync(Call call) + { + if (call == null || call.DepartmentId <= 0 || call.CallId <= 0 || call.IsDeleted) + return null; + + var ctx = await ContextAsync(call.DepartmentId); + var number = Safe(call.Number); + var name = ctx.ProtectedTextAllowed ? Safe(call.Name) : null; + var nature = ctx.ProtectedTextAllowed ? Safe(call.NatureOfCall) : null; + var type = ctx.ProtectedTextAllowed ? Safe(call.Type) : null; + var address = ctx.ProtectedTextAllowed ? Safe(call.Address) : null; + var incident = ctx.ProtectedTextAllowed ? Safe(call.IncidentNumber) : null; + var reference = ctx.ProtectedTextAllowed ? Safe(call.ReferenceNumber) : null; + var external = ctx.ProtectedTextAllowed ? Safe(call.ExternalIdentifier) : null; + var state = Enum.IsDefined(typeof(CallStates), call.State) ? ((CallStates)call.State).ToString() : call.State.ToString(); + + var p = New(call.DepartmentId, SearchEntityTypes.Call, call.CallId.ToString(), ctx); + p.Title = Cap(name ?? (number != null ? "Call " + number : "Call " + call.CallId), TitleMax); + p.Summary = Cap(Join(" · ", nature, type), SummaryMax); + p.Keywords = Cap(Join(" ", number, incident, reference, external), KeywordsMax); + p.SearchText = Cap(Join(" ", address, nature, type), SearchTextMax); + p.Category = type; + p.Status = state; + p.Priority = call.Priority; + p.IsActive = call.State == (int)CallStates.Active; + p.OccurredOn = call.LoggedOn == default ? DateTime.UtcNow : call.LoggedOn; + p.OwnerUserId = Safe(call.ReportingUserId); + p.Url = $"/User/Dispatch/ViewCall?callId={call.CallId}"; + p.MetadataJson = Json(new Dictionary + { + ["Number"] = number, + ["Priority"] = call.Priority.ToString(), + ["State"] = state, + ["LoggedOn"] = p.OccurredOn.ToString("o"), + ["IncidentNumber"] = incident + }); + return p; + } + + public async Task BuildUnitAsync(Unit unit) + { + if (unit == null || unit.DepartmentId <= 0 || unit.UnitId <= 0) + return null; + + var ctx = await ContextAsync(unit.DepartmentId); + var name = Safe(unit.Name); + if (name == null) + return null; + + var p = New(unit.DepartmentId, SearchEntityTypes.Unit, unit.UnitId.ToString(), ctx); + p.Title = Cap(name, TitleMax); + p.Summary = Cap(Safe(unit.Type), SummaryMax); + p.Keywords = Cap(Join(" ", name, Safe(unit.VIN), Safe(unit.PlateNumber)), KeywordsMax); + p.Category = Safe(unit.Type); + p.GroupId = unit.StationGroupId; + p.IsActive = true; + p.OccurredOn = DateTime.UtcNow; + p.Url = "/User/Units"; + p.MetadataJson = Json(new Dictionary { ["Type"] = Safe(unit.Type), ["StationGroupId"] = unit.StationGroupId?.ToString() }); + return p; + } + + public async Task BuildPersonnelAsync(int departmentId, UserProfile profile, int? groupId, bool isActive) + { + if (profile == null || departmentId <= 0 || string.IsNullOrWhiteSpace(profile.UserId)) + return null; + + var ctx = await ContextAsync(departmentId); + var first = Safe(profile.FirstName); + var last = Safe(profile.LastName); + var name = Join(" ", first, last); + if (name == null) + return null; + + // Member identification numbers are cataloged (DepartmentMemberSensitiveData); the legacy profile column + // is treated the same way. Phones, e-mail and addresses are never projected (plan R3). + var idNumber = ctx.ProtectedTextAllowed ? Safe(profile.IdentificationNumber) : null; + + var p = New(departmentId, SearchEntityTypes.Personnel, profile.UserId, ctx); + p.Title = Cap(name, TitleMax); + p.Keywords = Cap(Join(" ", idNumber, first, last), KeywordsMax); + p.GroupId = groupId; + p.OwnerUserId = profile.UserId; + p.IsActive = isActive; + p.OccurredOn = profile.LastUpdated ?? DateTime.UtcNow; + p.Url = $"/User/Personnel/ViewPerson?userId={Uri.EscapeDataString(profile.UserId)}"; + p.MetadataJson = Json(new Dictionary { ["IdentificationNumber"] = idNumber, ["GroupId"] = groupId?.ToString(), ["IsActive"] = isActive ? "true" : "false" }); + return p; + } + + public async Task BuildContactAsync(Contact contact) + { + if (contact == null || contact.DepartmentId <= 0 || string.IsNullOrWhiteSpace(contact.ContactId) || contact.IsDeleted) + return null; + + var ctx = await ContextAsync(contact.DepartmentId); + // Every Contact column is cataloged: in an enforced department nothing textual can be indexed. + if (!ctx.ProtectedTextAllowed) + return null; + + var first = Safe(contact.FirstName); + var last = Safe(contact.LastName); + var company = Safe(contact.CompanyName); + var other = Safe(contact.OtherName); + var title = Join(" ", first, last) ?? company ?? other; + if (title == null) + return null; + + var p = New(contact.DepartmentId, SearchEntityTypes.Contact, contact.ContactId, ctx); + p.Title = Cap(title, TitleMax); + p.Summary = Cap(Join(" · ", company != null && title != company ? company : null, Safe(contact.Description)), SummaryMax); + p.Keywords = Cap(Join(" ", Safe(contact.Email), Digits(contact.CellPhoneNumber), Digits(contact.HomePhoneNumber), Digits(contact.OfficePhoneNumber)), KeywordsMax); + p.SearchText = Cap(Join(" ", other, company, Safe(contact.Email), Safe(contact.OtherInfo), Safe(contact.Website)), SearchTextMax); + p.Category = contact.ContactType == 1 ? "Company" : "Person"; + p.IsActive = true; + p.OccurredOn = DateTime.UtcNow; + p.Url = $"/User/Contacts/View?contactId={Uri.EscapeDataString(contact.ContactId)}"; + p.MetadataJson = Json(new Dictionary { ["ContactType"] = contact.ContactType.ToString(), ["CategoryId"] = Safe(contact.ContactCategoryId) }); + return p; + } + + public async Task BuildMessageAsync(Message message) + { + if (message == null || !message.DepartmentId.HasValue || message.DepartmentId.Value <= 0 || message.MessageId <= 0 || message.IsDeleted) + return null; + + var ctx = await ContextAsync(message.DepartmentId.Value); + var subject = ctx.ProtectedTextAllowed ? Safe(message.Subject) : null; + var body = ctx.ProtectedTextAllowed ? Safe(Strip(message.Body)) : null; + + var recipients = new List(); + if (!string.IsNullOrWhiteSpace(message.ReceivingUserId)) + recipients.Add(message.ReceivingUserId); + if (message.MessageRecipients != null) + recipients.AddRange(message.MessageRecipients.Where(r => r != null && !r.IsDeleted && !string.IsNullOrWhiteSpace(r.UserId)).Select(r => r.UserId)); + + var p = New(message.DepartmentId.Value, SearchEntityTypes.Message, message.MessageId.ToString(), ctx); + p.Title = Cap(subject ?? "Message", TitleMax); + p.Summary = Cap(body == null ? null : body.Substring(0, Math.Min(body.Length, 200)), SummaryMax); + p.SearchText = Cap(body, SearchTextMax); + p.Category = message.Type.ToString(); + p.OwnerUserId = Safe(message.SendingUserId); + p.ParticipantUserIds = recipients.Count == 0 ? null : string.Join(",", recipients.Distinct()); + p.IsActive = !message.ExpireOn.HasValue || message.ExpireOn.Value > DateTime.UtcNow; + p.OccurredOn = message.SentOn == default ? DateTime.UtcNow : message.SentOn; + p.Url = $"/User/Messages/ViewMessage?messageId={message.MessageId}"; + p.MetadataJson = Json(new Dictionary { ["Type"] = message.Type.ToString(), ["SentOn"] = p.OccurredOn.ToString("o"), ["IsBroadcast"] = message.IsBroadcast ? "true" : "false" }); + return p; + } + + public async Task BuildDocumentAsync(Document document) + { + if (document == null || document.DepartmentId <= 0 || document.DocumentId <= 0) + return null; + + var ctx = await ContextAsync(document.DepartmentId); + var name = ctx.ProtectedTextAllowed ? Safe(document.Name) : null; + var description = ctx.ProtectedTextAllowed ? Safe(document.Description) : null; + var filename = ctx.ProtectedTextAllowed ? Safe(document.Filename) : null; + + var p = New(document.DepartmentId, SearchEntityTypes.Document, document.DocumentId.ToString(), ctx); + p.Title = Cap(name ?? "Document " + document.DocumentId, TitleMax); + p.Summary = Cap(description, SummaryMax); + p.Keywords = Cap(filename, KeywordsMax); + p.SearchText = Cap(Join(" ", filename, Safe(document.Category), Safe(document.Type)), SearchTextMax); + p.Category = Safe(document.Category); + p.IsAdminOnly = document.AdminsOnly; + p.OwnerUserId = Safe(document.UserId); + p.IsActive = !document.RemoveOn.HasValue || document.RemoveOn.Value > DateTime.UtcNow; + p.OccurredOn = document.AddedOn == default ? DateTime.UtcNow : document.AddedOn; + p.Url = $"/User/Documents/ViewDocument?documentId={document.DocumentId}"; + p.MetadataJson = Json(new Dictionary { ["Category"] = Safe(document.Category), ["Type"] = Safe(document.Type), ["AddedOn"] = p.OccurredOn.ToString("o") }); + return p; + } + + public async Task BuildNoteAsync(Note note) + { + if (note == null || note.DepartmentId <= 0 || note.NoteId <= 0) + return null; + + // Notes are not in the protected catalog; title and body are department-authored plain text. + var ctx = await ContextAsync(note.DepartmentId); + var title = Safe(note.Title); + var body = Safe(Strip(note.Body)); + if (title == null && body == null) + return null; + + var p = New(note.DepartmentId, SearchEntityTypes.Note, note.NoteId.ToString(), ctx); + p.Title = Cap(title ?? "Note " + note.NoteId, TitleMax); + p.Summary = Cap(body == null ? null : body.Substring(0, Math.Min(body.Length, 200)), SummaryMax); + p.SearchText = Cap(body, SearchTextMax); + p.Category = Safe(note.Category); + p.IsAdminOnly = note.IsAdminOnly; + p.OwnerUserId = Safe(note.UserId); + p.IsActive = !note.ExpiresOn.HasValue || note.ExpiresOn.Value > DateTime.UtcNow; + p.OccurredOn = note.AddedOn == default ? DateTime.UtcNow : note.AddedOn; + p.Url = $"/User/Notes/View?noteId={note.NoteId}"; + p.MetadataJson = Json(new Dictionary { ["Category"] = Safe(note.Category), ["Color"] = Safe(note.Color), ["AddedOn"] = p.OccurredOn.ToString("o") }); + return p; + } + + // ---- helpers --------------------------------------------------------------------------------------------- + + private sealed class ProjectionContext + { + public bool ProtectedTextAllowed; + public int CatalogVersion; + public long PolicyEpoch; + } + + private async Task ContextAsync(int departmentId) + { + var ctx = new ProjectionContext(); + try + { + // Unknown protection state never widens exposure: index metadata only. + ctx.ProtectedTextAllowed = !await _dataProtection.IsProtectionEnforcedAsync(departmentId); + } + catch (Exception ex) + { + Logging.LogException(ex, $"Protection state for department {departmentId} could not be determined; projecting metadata only."); + ctx.ProtectedTextAllowed = false; + } + try { ctx.CatalogVersion = await _dataProtection.GetPinnedCatalogVersionAsync(departmentId); } catch (Exception ex) { Logging.LogException(ex); } + try { ctx.PolicyEpoch = (await _dataProtection.GetPolicyByDepartmentIdAsync(departmentId))?.PolicyEpoch ?? 0; } catch (Exception ex) { Logging.LogException(ex); } + return ctx; + } + + private static SearchProjection New(int departmentId, string entityType, string entityId, ProjectionContext ctx) + { + return new SearchProjection + { + DepartmentId = departmentId, + EntityType = entityType, + EntityId = entityId, + ProtectedCatalogVersion = ctx.CatalogVersion, + PolicyEpoch = ctx.PolicyEpoch, + IncludesProtectedText = ctx.ProtectedTextAllowed, + IsActive = true, + OccurredOn = DateTime.UtcNow + }; + } + + /// Trimmed value, or null when empty, enveloped or redacted. + public static string Safe(string value) + { + if (string.IsNullOrWhiteSpace(value)) + return null; + var trimmed = value.Trim(); + if (ProtectedDataEnvelope.HasEnvelopePrefix(trimmed) || trimmed == ProtectedDataEnvelope.RedactionValue) + return null; + return trimmed; + } + + private static string Digits(string value) + { + var safe = Safe(value); + if (safe == null) + return null; + var digits = new string(safe.Where(char.IsDigit).ToArray()); + return digits.Length >= 4 ? digits : null; + } + + private static string Strip(string html) + { + if (string.IsNullOrWhiteSpace(html)) + return null; + var text = HtmlTags.Replace(html, " "); + text = System.Net.WebUtility.HtmlDecode(text); + return Whitespace.Replace(text, " ").Trim(); + } + + private static string Join(string separator, params string[] parts) + { + var kept = parts.Where(p => !string.IsNullOrWhiteSpace(p)).Select(p => p.Trim()).Distinct(StringComparer.OrdinalIgnoreCase).ToList(); + return kept.Count == 0 ? null : string.Join(separator, kept); + } + + private static string Cap(string value, int max) + { + if (string.IsNullOrEmpty(value)) + return null; + return value.Length <= max ? value : value.Substring(0, max); + } + + private static string Json(Dictionary values) + { + var kept = values.Where(kv => !string.IsNullOrWhiteSpace(kv.Value)).ToDictionary(kv => kv.Key, kv => kv.Value); + return kept.Count == 0 ? null : JsonConvert.SerializeObject(kept); + } + } +} diff --git a/Core/Resgrid.Services/Search/SystemActionCatalog.cs b/Core/Resgrid.Services/Search/SystemActionCatalog.cs new file mode 100644 index 000000000..9d507ccad --- /dev/null +++ b/Core/Resgrid.Services/Search/SystemActionCatalog.cs @@ -0,0 +1,162 @@ +using System.Collections.Generic; +using Resgrid.Model; +using Resgrid.Model.Search; + +namespace Resgrid.Services.Search +{ + /// + /// The system-functionality catalog behind "search also finds features": every page and action the web app's + /// navigation and command palette expose, with the claim / module / feature-flag gate the page itself applies. + /// Paths are the MVC routes (Areas/User). Claim resource names mirror ResgridClaimTypes.Resources; they are string + /// literals here because the services layer does not reference the claims provider. + /// + public static class SystemActionCatalog + { + public const string View = "View"; + public const string Create = "Create"; + public const string Update = "Update"; + + // ResgridClaimTypes.Resources + private const string Call = "Call"; + private const string Personnel = "Personnel"; + private const string Unit = "Unit"; + private const string Contacts = "Contacts"; + private const string Documents = "Documents"; + private const string Notes = "Notes"; + private const string Training = "Training"; + private const string Messages = "Messages"; + private const string Shift = "Shift"; + private const string Schedule = "Schedule"; + private const string Inventory = "Inventory"; + private const string Reports = "Reports"; + private const string Record = "Record"; + private const string Checklist = "Checklist"; + private const string WorkOrder = "WorkOrder"; + private const string Group = "Group"; + private const string Protocols = "Protocols"; + private const string Forms = "Forms"; + private const string Workflow = "Workflow"; + private const string Voice = "Voice"; + private const string Log = "Log"; + + private static SystemActionDefinition Nav(string key, string title, string description, string path, string[] keywords = null, + string claimResource = null, string claimAction = null, string module = null, string flag = null, bool adminOnly = false, bool hiddenWhenRecords = false) + { + return new SystemActionDefinition + { + Key = key, + Title = title, + Description = description, + WebPath = path, + Keywords = keywords ?? new string[0], + Category = SystemActionCategories.Navigate, + ClaimResource = claimResource, + ClaimAction = claimAction, + Module = module, + FeatureFlag = flag, + DepartmentAdminOnly = adminOnly, + HiddenWhenRecordsEnabled = hiddenWhenRecords + }; + } + + private static SystemActionDefinition Act(string key, string title, string description, string path, string category, string[] keywords = null, + string claimResource = null, string claimAction = null, string module = null, string flag = null, bool adminOnly = false, bool hiddenWhenRecords = false) + { + var d = Nav(key, title, description, path, keywords, claimResource, claimAction, module, flag, adminOnly, hiddenWhenRecords); + d.Category = category; + return d; + } + + public static readonly IReadOnlyList All = new List + { + // ---- Home / account + Nav("dashboard", "Dashboard", "Department home: status, staffing and activity", "/User/Home/Dashboard", new[] { "home", "overview", "start" }), + Act("profile", "My Profile", "View and edit your own profile, contact methods and notifications", "/User/Home/EditUserProfile?UserId={userId}", SystemActionCategories.Account, new[] { "account", "settings", "phone", "email", "password", "notifications" }), + Act("departments", "Your Departments", "Switch between the departments you belong to", "/User/Profile/YourDepartments", SystemActionCategories.Account, new[] { "switch", "membership", "organizations" }), + Act("two-factor", "Two-Factor Authentication", "Set up or manage two-factor sign-in", "/User/TwoFactor", SystemActionCategories.Account, new[] { "2fa", "mfa", "authenticator", "security", "passkey" }), + + // ---- Calls / dispatch + Nav("calls", "Calls", "View calls and dispatches", "/User/Dispatch/Dashboard", new[] { "dispatch", "incidents", "active calls", "cad" }, Call, View), + Act("new-call", "New Call", "Create and dispatch a new call", "/User/Dispatch/NewCall", SystemActionCategories.Create, new[] { "dispatch", "incident", "page", "alert", "create call" }, Call, Create), + Nav("archived-calls", "Archived Calls", "Closed and historical calls", "/User/Dispatch/ArchivedCalls", new[] { "closed", "history", "old calls" }, Call, View), + + // ---- Personnel + Nav("personnel", "Personnel", "People in the department, status and staffing", "/User/Personnel", new[] { "people", "members", "users", "staff", "responders", "roster" }, Personnel, View), + Act("add-person", "Add Person", "Manually create a user account in the department", "/User/Personnel/AddPerson", SystemActionCategories.Create, new[] { "new user", "create user", "add member", "invite" }, Personnel, Create), + Act("invites", "Manage Invites", "Send email invites so people create their own accounts", "/User/Department/Invites", SystemActionCategories.Manage, new[] { "invite", "email invite", "onboard" }, Personnel, Create), + Nav("groups", "Groups & Stations", "Department groups and stations", "/User/Groups", new[] { "stations", "battalions", "teams", "groups" }, Group, View), + Act("new-group", "New Group", "Create a group or station", "/User/Groups/NewGroup", SystemActionCategories.Create, new[] { "station", "add group" }, Group, Create), + + // ---- Units + Nav("units", "Units", "Apparatus, vehicles and teams", "/User/Units", new[] { "apparatus", "vehicles", "trucks", "engines", "teams", "rigs" }, Unit, View), + Act("new-unit", "New Unit", "Add an apparatus, vehicle or team", "/User/Units/NewUnit", SystemActionCategories.Create, new[] { "add unit", "apparatus", "vehicle" }, Unit, Create), + Act("unit-staffing", "Unit Staffing", "Assign personnel to units", "/User/Units/UnitStaffing", SystemActionCategories.Manage, new[] { "crew", "assign", "staffing" }, Unit, Update), + + // ---- Contacts + Nav("contacts", "Contacts", "People and organizations outside the department", "/User/Contacts", new[] { "businesses", "vendors", "customers", "address book", "pre-plans" }, Contacts, View), + Act("new-contact", "New Contact", "Add a person or organization contact", "/User/Contacts/Add", SystemActionCategories.Create, new[] { "add contact", "company", "person" }, Contacts, Create), + Nav("contact-categories", "Contact Categories", "Manage contact categories", "/User/Contacts/Categories", new[] { "categories" }, Contacts, Update), + + // ---- Mapping + Nav("mapping", "Mapping", "Large map with layers, personnel and units", "/User/Mapping", new[] { "map", "gps", "avl", "location", "layers" }, module: SystemActionModules.Mapping), + Nav("pois", "Points of Interest", "Manage map points of interest", "/User/Mapping/POIs", new[] { "poi", "hydrants", "landmarks", "map markers" }, module: SystemActionModules.Mapping), + Nav("map-layers", "Map Layers", "Manage map layers", "/User/Mapping/Layers", new[] { "layers", "kml", "geojson" }, module: SystemActionModules.Mapping), + Nav("live-routing", "Live Routing", "Routing and directions for active resources", "/User/Mapping/LiveRouting", new[] { "routes", "directions", "navigation" }, module: SystemActionModules.Mapping), + + // ---- Shifts / calendar + Nav("shifts", "Shifts", "Shift signups, recurring shifts and trades", "/User/Shifts", new[] { "schedule", "signup", "trades", "workshift", "roster" }, Shift, View, SystemActionModules.Shifts), + Nav("calendar", "Calendar", "Events, meetings and trainings you can sign up for", "/User/Calendar", new[] { "events", "meetings", "schedule", "rsvp" }, Schedule, View, SystemActionModules.Calendar), + Act("new-calendar-item", "New Calendar Event", "Create a calendar event", "/User/Calendar/New", SystemActionCategories.Create, new[] { "event", "meeting", "schedule" }, Schedule, Create, SystemActionModules.Calendar), + + // ---- Logs (legacy) / Records + Nav("logs", "Logs", "Run, training, work and meeting logs", "/User/Logs", new[] { "run log", "activity", "reports", "training log", "work log" }, Log, View, SystemActionModules.Logs, hiddenWhenRecords: true), + Act("new-log", "New Log", "Create a run report, training log or work log", "/User/Logs/NewLog", SystemActionCategories.Create, new[] { "run report", "training log", "work log" }, Log, Create, SystemActionModules.Logs, hiddenWhenRecords: true), + Nav("records", "Records", "Records queue: run reports, training and operational records", "/User/Records", new[] { "rms", "run reports", "incident reports", "neris", "logs" }, Record, View, SystemActionModules.Logs, FeatureFlagKeys.RecordsSystem), + Nav("records-dashboard", "Records Dashboard", "Records due, submissions and quality at a glance", "/User/Records/Dashboard", new[] { "rms", "overview", "due" }, Record, View, SystemActionModules.Logs, FeatureFlagKeys.RecordsSystem), + Act("records-settings", "Records Settings", "Lifecycle, numbering, search, retention and visibility settings for Records", "/User/Records/Settings", SystemActionCategories.Manage, new[] { "rms settings", "retention", "numbering" }, Record, View, SystemActionModules.Logs, FeatureFlagKeys.RecordsSystem, adminOnly: true), + + // ---- Reports + Nav("reports", "Reports", "Generate reports from department data", "/User/Reports", new[] { "reporting", "export", "statistics", "analytics" }, Reports, View, SystemActionModules.Reports), + + // ---- Documents / notes / training + Nav("documents", "Documents", "Upload and share documents", "/User/Documents", new[] { "files", "pdf", "word", "excel", "attachments", "sops" }, Documents, View, SystemActionModules.Documents), + Act("new-document", "Upload Document", "Upload a new document", "/User/Documents/NewDocument", SystemActionCategories.Create, new[] { "upload", "file", "attach" }, Documents, Create, SystemActionModules.Documents), + Nav("notes", "Notes", "Department notes: small bits of shared information", "/User/Notes", new[] { "memo", "bulletin", "announcements" }, Notes, View, SystemActionModules.Notes), + Act("new-note", "New Note", "Post a department note", "/User/Notes/NewNote", SystemActionCategories.Create, new[] { "memo", "bulletin", "announce" }, Notes, Create, SystemActionModules.Notes), + Nav("trainings", "Trainings", "Trainings, study guides and procedures", "/User/Trainings", new[] { "study guide", "quiz", "procedures", "education", "certification" }, Training, View, SystemActionModules.Training), + Act("new-training", "New Training", "Create a training with optional quiz", "/User/Trainings/New", SystemActionCategories.Create, new[] { "quiz", "study", "course" }, Training, Create, SystemActionModules.Training), + + // ---- Inventory / readiness + Nav("inventory", "Inventory", "Inventory for stations and units", "/User/Inventory", new[] { "stock", "supplies", "equipment", "assets", "consumables" }, Inventory, View, SystemActionModules.Inventory), + Nav("inventory-status", "Inventory Status", "On-hand, low-stock and expiring inventory", "/User/Inventory/Status", new[] { "on hand", "low stock", "expiring", "counts" }, Inventory, View, SystemActionModules.Inventory), + Act("inventory-transfer", "Transfer Inventory", "Move inventory between locations", "/User/Inventory/Transfer", SystemActionCategories.Manage, new[] { "move stock", "transfer" }, Inventory, Update, SystemActionModules.Inventory), + Act("inventory-issue", "Issue Equipment", "Issue or return equipment to personnel", "/User/Inventory/Issue", SystemActionCategories.Manage, new[] { "issue", "return", "check out", "equipment" }, Inventory, Update, SystemActionModules.Inventory), + Nav("checklists", "Checklists", "Apparatus, station and readiness checklists", "/User/Checklists", new[] { "truck check", "daily check", "readiness", "inspection" }, Checklist, View, null, FeatureFlagKeys.ChecklistsSystem), + Act("new-checklist", "New Checklist", "Author a checklist definition", "/User/Checklists/New", SystemActionCategories.Create, new[] { "checklist definition", "template" }, Checklist, Update, null, FeatureFlagKeys.ChecklistsSystem), + Nav("checklist-templates", "Checklist Templates", "Start from a checklist template", "/User/Checklists/Templates", new[] { "templates", "library" }, Checklist, View, null, FeatureFlagKeys.ChecklistsSystem), + Nav("work-orders", "Work Orders", "Maintenance and repair work orders", "/User/WorkOrders", new[] { "maintenance", "repair", "service", "fleet", "defect" }, WorkOrder, View, SystemActionModules.Maintenance, FeatureFlagKeys.MaintenanceWorkOrders), + Act("new-work-order", "New Work Order", "Open a maintenance work order", "/User/WorkOrders/New", SystemActionCategories.Create, new[] { "repair", "defect", "maintenance request" }, WorkOrder, Update, SystemActionModules.Maintenance, FeatureFlagKeys.MaintenanceWorkOrders), + + // ---- Messaging / chat + Nav("inbox", "Inbox", "Your messages inbox", "/User/Messages/Inbox", new[] { "messages", "mail", "read" }, Messages, View, SystemActionModules.Messaging), + Nav("outbox", "Sent Messages", "Messages you sent", "/User/Messages/Outbox", new[] { "sent", "outbox" }, Messages, View, SystemActionModules.Messaging), + Act("compose-message", "New Message", "Send a message, poll or callback request", "/User/Messages/Compose", SystemActionCategories.Create, new[] { "send", "compose", "email", "poll", "callback", "broadcast" }, Messages, Create, SystemActionModules.Messaging), + Nav("chat", "Chat", "Real-time department chat", "/User/Chat", new[] { "channels", "direct message", "dm", "team chat" }, flag: FeatureFlagKeys.ChatSystem), + + // ---- Automation / configuration + Nav("workflows", "Workflows", "Automations triggered by department events", "/User/Workflows", new[] { "automation", "triggers", "webhooks", "integrations" }, Workflow, View), + Act("new-workflow", "New Workflow", "Create an automation workflow", "/User/Workflows/New", SystemActionCategories.Create, new[] { "automation", "trigger" }, Workflow, Create), + Nav("workflow-runs", "Workflow Runs", "Workflow execution history", "/User/Workflows/Runs", new[] { "runs", "history", "automation log" }, Workflow, View), + Nav("protocols", "Protocols", "Dispatch protocols and procedures", "/User/Protocols", new[] { "dispatch protocols", "sop", "procedures" }, Protocols, View), + Act("new-protocol", "New Protocol", "Create a dispatch protocol", "/User/Protocols/New", SystemActionCategories.Create, new[] { "protocol", "procedure" }, Protocols, Create), + Nav("forms", "Forms", "Custom call and dispatch forms", "/User/Forms", new[] { "custom forms", "fields", "templates" }, Forms, View), + Nav("voice", "Voice", "Voice channels and push-to-talk", "/User/Voice", new[] { "ptt", "push to talk", "radio", "audio" }, Voice, View), + + // ---- Department administration + Act("department-settings", "Department Settings", "Department profile, address, API keys and module settings", "/User/Department", SystemActionCategories.Manage, new[] { "settings", "admin", "configuration", "modules", "api key" }, adminOnly: true), + Act("call-settings", "Call Settings", "Call types, priorities, email import and dispatch settings", "/User/Department/CallSettings", SystemActionCategories.Manage, new[] { "call types", "priorities", "email import", "dispatch settings" }, adminOnly: true), + Act("dispatch-settings", "Dispatch Settings", "Dispatch behaviour and notification settings", "/User/Department/DispatchSettings", SystemActionCategories.Manage, new[] { "dispatch", "notifications", "paging" }, adminOnly: true), + Act("data-protection", "Data Protection", "Advanced Data Protection enrollment and policies", "/User/DataProtection", SystemActionCategories.Manage, new[] { "adp", "encryption", "privacy", "protected data", "kms" }, adminOnly: true) + }; + } +} diff --git a/Core/Resgrid.Services/Search/SystemActionsService.cs b/Core/Resgrid.Services/Search/SystemActionsService.cs new file mode 100644 index 000000000..c11da0ebc --- /dev/null +++ b/Core/Resgrid.Services/Search/SystemActionsService.cs @@ -0,0 +1,212 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Resgrid.Config; +using Resgrid.Framework; +using Resgrid.Model; +using Resgrid.Model.Search; +using Resgrid.Model.Services; + +namespace Resgrid.Services.Search +{ + /// + /// Searches the system-functionality catalog for one caller (plan R3 "Action" family). The catalog is static and + /// small; entries the caller may not use are removed before scoring, so the palette never advertises a page the + /// caller cannot open. Scoring is deterministic: exact title, title prefix, per-token prefix on title / keywords / + /// description, and a one-edit fuzzy match for tokens of four characters or more. + /// + public class SystemActionsService : ISystemActionsService + { + private readonly IFeatureToggleService _featureToggles; + + public SystemActionsService(IFeatureToggleService featureToggles) + { + _featureToggles = featureToggles ?? throw new ArgumentNullException(nameof(featureToggles)); + } + + public async Task> SearchAsync(string text, SearchPrincipal principal, int max = 8, CancellationToken cancellationToken = default) + { + if (principal == null) + return new List(); + + var allowed = await AllowedAsync(principal, cancellationToken); + var tokens = Tokenize(text); + if (tokens.Count == 0) + return allowed.Select(a => ToHit(a, principal, 0f)).Take(Math.Max(1, max)).ToList(); + + var scored = new List<(SystemActionDefinition def, float score)>(); + foreach (var def in allowed) + { + var score = Score(def, tokens, text); + if (score > 0f) + scored.Add((def, score)); + } + + return scored + .OrderByDescending(s => s.score) + .ThenBy(s => s.def.Title, StringComparer.OrdinalIgnoreCase) + .Take(Math.Max(1, max)) + .Select(s => ToHit(s.def, principal, s.score)) + .ToList(); + } + + public async Task> ListAsync(SearchPrincipal principal, CancellationToken cancellationToken = default) + { + if (principal == null) + return new List(); + var allowed = await AllowedAsync(principal, cancellationToken); + return allowed.Select(a => ToHit(a, principal, 0f)).ToList(); + } + + private async Task> AllowedAsync(SearchPrincipal principal, CancellationToken cancellationToken) + { + var flags = new Dictionary(StringComparer.OrdinalIgnoreCase); + async Task FlagAsync(string key) + { + if (string.IsNullOrWhiteSpace(key)) + return true; + if (flags.TryGetValue(key, out var known)) + return known; + bool value; + try { value = await _featureToggles.IsEnabledAsync(key, principal.DepartmentId); } + catch (Exception ex) { Logging.LogException(ex, $"Feature flag {key} could not be evaluated for the command palette; treating as off."); value = false; } + flags[key] = value; + return value; + } + + var recordsOn = await FlagAsync(FeatureFlagKeys.RecordsSystem); + var allowed = new List(); + foreach (var def in SystemActionCatalog.All) + { + cancellationToken.ThrowIfCancellationRequested(); + if (def.DepartmentAdminOnly && !principal.IsDepartmentAdmin) + continue; + if (!string.IsNullOrWhiteSpace(def.ClaimResource) && !principal.IsDepartmentAdmin && !principal.HasResourceClaim(def.ClaimResource, def.ClaimAction ?? SystemActionCatalog.View)) + continue; + if (!principal.ModuleEnabled(def.Module)) + continue; + if (def.HiddenWhenRecordsEnabled && recordsOn) + continue; + if (!await FlagAsync(def.FeatureFlag)) + continue; + allowed.Add(def); + } + return allowed; + } + + private static SystemActionHit ToHit(SystemActionDefinition def, SearchPrincipal principal, float score) + { + var path = (def.WebPath ?? string.Empty).Replace("{userId}", Uri.EscapeDataString(principal.UserId ?? string.Empty)); + return new SystemActionHit + { + Key = def.Key, + Title = def.Title, + Description = def.Description, + Category = def.Category, + Url = (SystemBehaviorConfig.ResgridBaseUrl ?? string.Empty).TrimEnd('/') + path, + Score = score + }; + } + + /// Visible for tests. + public static float Score(SystemActionDefinition def, List tokens, string rawText) + { + var title = (def.Title ?? string.Empty).ToLowerInvariant(); + var titleWords = Words(title); + var keywordWords = (def.Keywords ?? Array.Empty()).SelectMany(k => Words(k.ToLowerInvariant())).ToList(); + var descriptionWords = Words((def.Description ?? string.Empty).ToLowerInvariant()); + var key = (def.Key ?? string.Empty).ToLowerInvariant(); + var normalized = string.Join(" ", tokens); + + var score = 0f; + if (title == normalized || key == normalized) + score += 10f; + else if (title.StartsWith(normalized, StringComparison.Ordinal)) + score += 6f; + + foreach (var token in tokens) + { + var best = 0f; + if (titleWords.Any(w => w.StartsWith(token, StringComparison.Ordinal))) + best = Math.Max(best, 3f); + if (key == token || key.StartsWith(token, StringComparison.Ordinal)) + best = Math.Max(best, 3f); + if (keywordWords.Any(w => w.StartsWith(token, StringComparison.Ordinal))) + best = Math.Max(best, 2.5f); + if (descriptionWords.Any(w => w.StartsWith(token, StringComparison.Ordinal))) + best = Math.Max(best, 1f); + if (best == 0f && token.Length >= 4) + { + if (titleWords.Concat(keywordWords).Any(w => w.Length >= 4 && WithinOneEdit(token, w.Length > token.Length + 1 ? w.Substring(0, Math.Min(w.Length, token.Length + 1)) : w))) + best = 1.5f; + } + if (best == 0f) + { + // Intent words ("new call", "open inbox", "my profile") narrow when they match and are ignored when + // they do not; every other token must match something. + if (IntentWords.Contains(token)) + continue; + return 0f; + } + score += best; + } + + return score; + } + + private static readonly HashSet IntentWords = new HashSet(StringComparer.Ordinal) + { + "new", "create", "add", "open", "view", "show", "go", "to", "the", "my", "manage", "edit", "list", "see", "find" + }; + + public static List Tokenize(string text) + { + if (string.IsNullOrWhiteSpace(text)) + return new List(); + return Words(text.Trim().TrimStart('/').ToLowerInvariant()).Take(8).ToList(); + } + + private static List Words(string text) + { + var words = new List(); + if (string.IsNullOrEmpty(text)) + return words; + var current = new System.Text.StringBuilder(); + foreach (var ch in text) + { + if (char.IsLetterOrDigit(ch)) + { + current.Append(ch); + } + else if (current.Length > 0) + { + words.Add(current.ToString()); + current.Clear(); + } + } + if (current.Length > 0) + words.Add(current.ToString()); + return words; + } + + /// Damerau-free Levenshtein bound of one, early exit. + public static bool WithinOneEdit(string a, string b) + { + if (a == b) return true; + if (Math.Abs(a.Length - b.Length) > 1) return false; + int i = 0, j = 0, edits = 0; + while (i < a.Length && j < b.Length) + { + if (a[i] == b[j]) { i++; j++; continue; } + if (++edits > 1) return false; + if (a.Length > b.Length) i++; + else if (a.Length < b.Length) j++; + else { i++; j++; } + } + edits += (a.Length - i) + (b.Length - j); + return edits <= 1; + } + } +} diff --git a/Core/Resgrid.Services/Search/UnifiedSearchService.cs b/Core/Resgrid.Services/Search/UnifiedSearchService.cs new file mode 100644 index 000000000..a3497083a --- /dev/null +++ b/Core/Resgrid.Services/Search/UnifiedSearchService.cs @@ -0,0 +1,380 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Newtonsoft.Json; +using Resgrid.Framework; +using Resgrid.Model; +using Resgrid.Model.Repositories; +using Resgrid.Model.Search; +using Resgrid.Model.Services; + +namespace Resgrid.Services.Search +{ + /// + /// The unified endpoint (plan R4 Phase 2). Order of operations: department flag → system actions (always, cheap) → + /// global index query with the department clause injected → per-hit authorization using the entity's own rule + /// (call view, unit/person matrix, message sender/recipient, claim for contacts/documents/notes) → Records + /// federated from the RMS index with its own re-check → totals suppressed whenever a hit was dropped. A department + /// that has never searched gets a state row on its first query so worker 70 starts indexing it (lazy activation). + /// + public class UnifiedSearchService : IUnifiedSearchService + { + private const int CandidateWindow = 200; + + private readonly IGlobalSearchService _global; + private readonly ISystemActionsService _actions; + private readonly IFeatureToggleService _featureToggles; + private readonly IAuthorizationService _authorization; + private readonly ISearchIndexStatesRepository _states; + private readonly IRecordsSearchService _recordsSearch; + private readonly IRecordsAuthorizationService _recordsAuthorization; + private readonly IRecordsService _records; + private readonly IRecordsCutoverService _recordsCutover; + + public UnifiedSearchService(IGlobalSearchService global, ISystemActionsService actions, IFeatureToggleService featureToggles, + IAuthorizationService authorization, ISearchIndexStatesRepository states, IRecordsSearchService recordsSearch, + IRecordsAuthorizationService recordsAuthorization, IRecordsService records, IRecordsCutoverService recordsCutover) + { + _global = global; + _actions = actions; + _featureToggles = featureToggles; + _authorization = authorization; + _states = states; + _recordsSearch = recordsSearch; + _recordsAuthorization = recordsAuthorization; + _records = records; + _recordsCutover = recordsCutover; + } + + public async Task SearchAsync(UnifiedSearchRequest request, SearchPrincipal principal, CancellationToken cancellationToken = default) + { + var watch = Stopwatch.StartNew(); + request = request ?? new UnifiedSearchRequest(); + var result = new UnifiedSearchResult(); + + if (principal == null || principal.DepartmentId <= 0 || string.IsNullOrWhiteSpace(principal.UserId)) + { + result.Available = false; + return Finish(result, watch); + } + + if (!await FlagOnAsync(principal.DepartmentId)) + { + result.Available = false; + result.DegradedReason = "Search.Unified is off for this department."; + return Finish(result, watch); + } + + var text = (request.Text ?? string.Empty).Trim(); + if (text.Length > 500) + text = text.Substring(0, 500); + + if (request.IncludeActions) + { + try + { + result.Actions = text.Length == 0 + ? await _actions.ListAsync(principal, cancellationToken) + : await _actions.SearchAsync(text, principal, 8, cancellationToken); + } + catch (Exception ex) + { + Logging.LogException(ex, "System action search failed."); + } + } + + if (text.Length == 0) + { + result.Total = 0; + return Finish(result, watch); + } + + var types = AllowedTypes(request.EntityTypes, principal); + var dropped = 0; + var authorized = new List(); + var indexTotal = 0; + var truncated = false; + var windowCoveredAll = false; + + if (types.Count > 0) + { + if (!_global.IsAvailable) + { + result.Degraded = true; + result.DegradedReason = "The search index is not available yet."; + await EnsureStateAsync(principal.DepartmentId, cancellationToken); + } + else + { + GlobalSearchResult indexResult; + try + { + indexResult = await _global.SearchAsync(principal.DepartmentId, new GlobalSearchQuery + { + Text = text, + EntityTypes = types, + ViewerUserId = principal.UserId, + IncludeAdminOnly = principal.IsDepartmentAdmin, + Prefix = request.Prefix, + Skip = 0, + Take = CandidateWindow + }, cancellationToken); + } + catch (Exception ex) + { + Logging.LogException(ex, "Global search query failed."); + indexResult = new GlobalSearchResult { Available = false }; + } + + if (!indexResult.Available) + { + result.Degraded = true; + result.DegradedReason = "The search index is not available yet."; + await EnsureStateAsync(principal.DepartmentId, cancellationToken); + } + else + { + indexTotal = indexResult.Total; + truncated = indexResult.Truncated; + windowCoveredAll = indexResult.Hits.Count >= indexResult.Total; + var need = Math.Max(0, request.Skip) + Math.Max(1, request.Take); + foreach (var hit in indexResult.Hits) + { + cancellationToken.ThrowIfCancellationRequested(); + if (authorized.Count >= need && windowCoveredAll == false) + break; + if (await AuthorizeAsync(hit, principal)) + authorized.Add(Map(hit)); + else + dropped++; + } + } + } + } + + var recordHits = new List(); + int? recordsTotal = 0; + if (request.IncludeRecords && !request.Prefix && WantsType(request.EntityTypes, SearchEntityTypes.Record)) + { + try + { + (recordHits, recordsTotal) = await FederateRecordsAsync(text, principal, cancellationToken); + } + catch (Exception ex) + { + Logging.LogException(ex, "Records federation failed; returning the other families."); + recordsTotal = null; + } + } + + var skip = Math.Max(0, request.Skip); + var take = Math.Max(1, Math.Min(100, request.Take)); + var page = authorized.Skip(skip).Take(take).ToList(); + if (page.Count < take && skip == 0) + page.AddRange(recordHits.Take(take - page.Count)); + result.Hits = page; + result.Truncated = truncated; + + // Totals only when they can be proven from authorized results (plan 2026-08-15 correction). + if (dropped == 0 && recordsTotal.HasValue) + result.Total = (windowCoveredAll ? authorized.Count : indexTotal) + recordsTotal.Value; + else + result.Total = null; + + return Finish(result, watch); + } + + private static UnifiedSearchResult Finish(UnifiedSearchResult result, Stopwatch watch) + { + result.QueryTimeMs = (int)watch.ElapsedMilliseconds; + return result; + } + + private static bool WantsType(List requested, string type) + { + return requested == null || requested.Count == 0 || requested.Any(t => string.Equals(t, type, StringComparison.OrdinalIgnoreCase)); + } + + /// Families the caller may search at all: requested ∩ claim-allowed ∩ module-enabled. + private static List AllowedTypes(List requested, SearchPrincipal principal) + { + var allowed = new List(); + void Add(string type, string resource, string module = null) + { + if (!WantsType(requested, type)) + return; + if (!principal.IsDepartmentAdmin && !principal.HasResourceClaim(resource, "View")) + return; + if (!principal.ModuleEnabled(module)) + return; + allowed.Add(type); + } + + Add(SearchEntityTypes.Call, "Call"); + Add(SearchEntityTypes.Unit, "Unit"); + Add(SearchEntityTypes.Personnel, "Personnel"); + Add(SearchEntityTypes.Contact, "Contacts"); + Add(SearchEntityTypes.Message, "Messages", SystemActionModules.Messaging); + Add(SearchEntityTypes.Document, "Documents", SystemActionModules.Documents); + Add(SearchEntityTypes.Note, "Notes", SystemActionModules.Notes); + return allowed; + } + + private async Task AuthorizeAsync(GlobalSearchHit hit, SearchPrincipal principal) + { + try + { + switch (hit.EntityType) + { + case SearchEntityTypes.Call: + return int.TryParse(hit.EntityId, out var callId) && await _authorization.CanUserViewCallAsync(principal.UserId, callId); + case SearchEntityTypes.Unit: + return int.TryParse(hit.EntityId, out var unitId) && await _authorization.CanUserViewUnitViaMatrixAsync(unitId, principal.UserId, principal.DepartmentId); + case SearchEntityTypes.Personnel: + return !string.IsNullOrWhiteSpace(hit.EntityId) && await _authorization.CanUserViewPersonViaMatrixAsync(hit.EntityId, principal.UserId, principal.DepartmentId); + case SearchEntityTypes.Message: + return int.TryParse(hit.EntityId, out var messageId) && await _authorization.CanUserViewMessageAsync(principal.UserId, messageId); + case SearchEntityTypes.Contact: + return principal.IsDepartmentAdmin || principal.HasResourceClaim("Contacts", "View"); + case SearchEntityTypes.Document: + return principal.IsDepartmentAdmin || principal.HasResourceClaim("Documents", "View"); + case SearchEntityTypes.Note: + return principal.IsDepartmentAdmin || principal.HasResourceClaim("Notes", "View"); + default: + return false; + } + } + catch (Exception ex) + { + // Fail closed: an authorization error drops the hit and suppresses the total. + Logging.LogException(ex, $"Search hit authorization failed for {hit.EntityType} {hit.EntityId}."); + return false; + } + } + + private static UnifiedSearchHit Map(GlobalSearchHit hit) + { + IDictionary metadata = new Dictionary(); + if (!string.IsNullOrWhiteSpace(hit.MetadataJson)) + { + try { metadata = JsonConvert.DeserializeObject>(hit.MetadataJson) ?? metadata; } + catch { /* stored by us; a parse failure only loses badges */ } + } + + return new UnifiedSearchHit + { + EntityType = hit.EntityType, + EntityId = hit.EntityId, + Title = hit.Title, + Summary = hit.Summary, + Url = hit.Url, + Score = hit.Score, + OccurredOn = hit.OccurredOnTicks > 0 ? new DateTime(hit.OccurredOnTicks, DateTimeKind.Utc) : (DateTime?)null, + Category = hit.Category, + Status = hit.Status, + Metadata = metadata + }; + } + + private async Task<(List hits, int? total)> FederateRecordsAsync(string text, SearchPrincipal principal, CancellationToken cancellationToken) + { + var hits = new List(); + if (!principal.IsDepartmentAdmin && !principal.HasResourceClaim("Record", "View")) + return (hits, 0); + if (_recordsSearch == null || !_recordsSearch.IsAvailable) + return (hits, 0); + + var module = await _recordsCutover.GetModuleStateAsync(principal.DepartmentId); + if (module == null || !module.FlagEnabled || !module.Activated) + return (hits, 0); + if (!await _recordsAuthorization.IsActiveMemberAsync(principal.UserId, principal.DepartmentId)) + return (hits, 0); + + List visibleGroups = null; + if (await _recordsAuthorization.IsGroupScopedAsync(principal.DepartmentId)) + visibleGroups = await _recordsAuthorization.GetVisibleGroupIdsAsync(principal.UserId, principal.DepartmentId) ?? new List(); + + var search = await _recordsSearch.SearchAsync(principal.DepartmentId, new RecordsSearchRequest + { + Text = text, + VisibleGroupIds = visibleGroups, + ViewerUserId = principal.UserId, + Take = 20 + }, cancellationToken); + if (search == null || !search.Available) + return (hits, 0); + + var recordSource = ((int)RmsSearchSourceType.Record).ToString(); + var ids = search.Hits.Where(h => h.SourceType == recordSource && !string.IsNullOrWhiteSpace(h.SourceId)).Select(h => h.SourceId).Distinct().ToList(); + var loaded = (await _records.GetProjectionsByIdsAsync(principal.DepartmentId, ids) ?? new List()) + .ToDictionary(p => p.RmsRecordSearchProjectionId, StringComparer.OrdinalIgnoreCase); + + var dropped = 0; + foreach (var hit in search.Hits) + { + cancellationToken.ThrowIfCancellationRequested(); + if (!loaded.TryGetValue(hit.SourceId ?? string.Empty, out var projection) || !await _recordsAuthorization.CanUserViewRecordAsync(principal.UserId, hit.SourceId, principal.DepartmentId)) + { + dropped++; + continue; + } + + hits.Add(new UnifiedSearchHit + { + EntityType = SearchEntityTypes.Record, + EntityId = projection.RmsRecordSearchProjectionId, + Title = string.IsNullOrWhiteSpace(projection.RecordNumber) ? (projection.DraftReference ?? projection.DisplaySummary ?? "Record") : projection.RecordNumber, + Summary = projection.DisplaySummary, + Url = $"/User/Records/Edit?id={Uri.EscapeDataString(projection.RmsRecordSearchProjectionId)}", + Score = hit.Score, + OccurredOn = projection.OccurredOn ?? projection.RecordCreatedOn, + Category = projection.DefinitionKey, + Status = projection.State.ToString(), + Metadata = new Dictionary + { + ["DefinitionKey"] = projection.DefinitionKey ?? string.Empty, + ["State"] = projection.State.ToString(), + ["CallId"] = projection.CallId?.ToString() ?? string.Empty + } + }); + } + + return (hits, dropped == 0 ? search.Total : (int?)null); + } + + private async Task EnsureStateAsync(int departmentId, CancellationToken cancellationToken) + { + try + { + var existing = await _states.GetAsync(SearchIndexNames.Global, departmentId); + if (existing != null) + return; + var now = DateTime.UtcNow; + await _states.SaveOrUpdateAsync(new SearchIndexState + { + IndexName = SearchIndexNames.Global, + DepartmentId = departmentId, + SchemaVersion = GlobalSearchGeneration.SchemaVersion, + Generation = GlobalSearchGeneration.Compute(0, 0), + State = (int)SearchIndexBuildState.RebuildRequested, + RebuildRequestedOn = now, + CreatedOn = now, + ModifiedOn = now + }, cancellationToken, true); + } + catch (Exception ex) + { + Logging.LogException(ex, $"Could not create the search index state row for department {departmentId}."); + } + } + + private async Task FlagOnAsync(int departmentId) + { + try { return await _featureToggles.IsEnabledAsync(FeatureFlagKeys.SearchUnified, departmentId); } + catch (Exception ex) { Logging.LogException(ex); return false; } + } + } +} diff --git a/Core/Resgrid.Services/ServicesModule.cs b/Core/Resgrid.Services/ServicesModule.cs index 7e849305a..4527d50b1 100644 --- a/Core/Resgrid.Services/ServicesModule.cs +++ b/Core/Resgrid.Services/ServicesModule.cs @@ -281,6 +281,12 @@ protected override void Load(ContainerBuilder builder) builder.RegisterType().As().InstancePerLifetimeScope(); builder.RegisterType().As().InstancePerLifetimeScope(); builder.RegisterType().As().InstancePerLifetimeScope(); + // Unified Search (plan R4 Phase 1b/2, worker 70): projections written from the entity services, the global + // index sweep, the unified endpoint and the system-functionality catalog. + builder.RegisterType().As().InstancePerLifetimeScope(); + builder.RegisterType().As().InstancePerLifetimeScope(); + builder.RegisterType().As().InstancePerLifetimeScope(); + builder.RegisterType().As().InstancePerLifetimeScope(); // RMS-3: worker 42 (due-state evaluation, trigger 112 + notification 32) and worker 43 (retention, // legal hold, attachment purge and the Pending-attachment rescan). builder.RegisterType().As().InstancePerLifetimeScope(); diff --git a/Core/Resgrid.Services/UnitsService.cs b/Core/Resgrid.Services/UnitsService.cs index 493019629..3ff39320c 100644 --- a/Core/Resgrid.Services/UnitsService.cs +++ b/Core/Resgrid.Services/UnitsService.cs @@ -8,6 +8,7 @@ using Resgrid.Model.Events; using Resgrid.Model.Providers; using Resgrid.Model.Repositories; +using Resgrid.Model.Search; using Resgrid.Model.Services; namespace Resgrid.Services @@ -40,6 +41,8 @@ public class UnitsService : IUnitsService // Lazy: the Records cutover guard (RMS plan section 4.1) is consulted only on a legacy UnitLog write. private readonly Lazy _recordsCutoverService; + private readonly Lazy _searchProjections; + public UnitsService(IUnitsRepository unitsRepository, IUnitStatesRepository unitStatesRepository, IUnitLogsRepository unitLogsRepository, IUnitTypesRepository unitTypesRepository, ISubscriptionsService subscriptionsService, IUnitRolesRepository unitRolesRepository, IUnitStateRoleRepository unitStateRoleRepository, IUserStateService userStateService, @@ -47,7 +50,7 @@ public UnitsService(IUnitsRepository unitsRepository, IUnitStatesRepository unit IUnitLocationsDocRepository unitLocationsDocRepository, Lazy unitLocationsMongoRepository, IUnitActiveRolesRepository unitActiveRolesRepository, IDepartmentGroupsService departmentGroupsService, ILimitsService limitsService, IPersonnelRolesService personnelRolesService, - Lazy protectedWriteService, Lazy recordsCutoverService, IInventoryStore inventoryStore = null, Resgrid.Model.Repositories.Queries.IUnitOfWork inventoryUnitOfWork = null) + Lazy protectedWriteService, Lazy recordsCutoverService, IInventoryStore inventoryStore = null, Resgrid.Model.Repositories.Queries.IUnitOfWork inventoryUnitOfWork = null, Lazy searchProjections = null) { _recordsCutoverService = recordsCutoverService; if ((inventoryStore == null) != (inventoryUnitOfWork == null)) @@ -71,6 +74,7 @@ public UnitsService(IUnitsRepository unitsRepository, IUnitStatesRepository unit _limitsService = limitsService; _personnelRolesService = personnelRolesService; _protectedWriteService = protectedWriteService; + _searchProjections = searchProjections; } public async Task> GetAllAsync() @@ -94,6 +98,7 @@ public async Task> GetAllAsync() // from it and a re-stationed unit is filed under the wrong group until it is rebuilt. SendUnitVisibilityRefresh(saved.DepartmentId); + if (_searchProjections != null) await _searchProjections.Value.ProjectUnitAsync(saved, cancellationToken); return saved; } @@ -211,6 +216,7 @@ public async Task GetUnitByIdAsync(int unitId) await _unitActiveRolesRepository.DeleteActiveRolesByUnitIdAsync(unit.UnitId, cancellationToken); await _unitsRepository.DeleteAsync(unit, cancellationToken); + if (_searchProjections != null) await _searchProjections.Value.RemoveAsync(unit.DepartmentId, SearchEntityTypes.Unit, unit.UnitId.ToString(), cancellationToken); await _limitsService.InvalidateDepartmentsEntityLimitsCache(unit.DepartmentId); _eventAggregator.SendMessage(new DepartmentSettingsUpdateEvent() { DepartmentId = unit.DepartmentId }); diff --git a/Core/Resgrid.Services/UserProfileService.cs b/Core/Resgrid.Services/UserProfileService.cs index 3de4abcd0..982040c47 100644 --- a/Core/Resgrid.Services/UserProfileService.cs +++ b/Core/Resgrid.Services/UserProfileService.cs @@ -21,12 +21,15 @@ public class UserProfileService : IUserProfileService private readonly ICacheProvider _cacheProvider; private readonly IChatbotIdentityRepository _chatbotIdentityRepository; + private readonly Lazy _searchProjections; + public UserProfileService(IUserProfilesRepository userProfileRepository, ICacheProvider cacheProvider, - IChatbotIdentityRepository chatbotIdentityRepository) + IChatbotIdentityRepository chatbotIdentityRepository, Lazy searchProjections = null) { _userProfileRepository = userProfileRepository; _cacheProvider = cacheProvider; _chatbotIdentityRepository = chatbotIdentityRepository; + _searchProjections = searchProjections; } public async Task GetProfileByUserIdAsync(string userId, bool bypassCache = false) @@ -137,6 +140,7 @@ public async Task> GetAllProfilesForDepartmentIn ClearUserProfileFromCache(savedProfile.UserId); ClearAllUserProfilesFromCache(DepartmentId); + if (_searchProjections != null && DepartmentId > 0) await _searchProjections.Value.ProjectPersonnelAsync(DepartmentId, savedProfile, null, null, cancellationToken); return savedProfile; } diff --git a/Providers/Resgrid.Providers.Migrations/Migrations/M0208_AddUnifiedSearch.cs b/Providers/Resgrid.Providers.Migrations/Migrations/M0208_AddUnifiedSearch.cs new file mode 100644 index 000000000..1a6329f5d --- /dev/null +++ b/Providers/Resgrid.Providers.Migrations/Migrations/M0208_AddUnifiedSearch.cs @@ -0,0 +1,105 @@ +using FluentMigrator; + +namespace Resgrid.Providers.Migrations.Migrations +{ + /// + /// Unified Search Phase 1b/2 (plan R4, R5; registry §4F, next physical number under the no-gaps rule): + /// SearchProjections is the safe, rebuildable search row per entity for the global index (one table with an + /// EntityType discriminator; only allowlisted fields, cataloged columns only where protection is not enforced, + /// never an envelope); SearchIndexStates tracks the per-index, per-department generation key + /// (schemaVersion, protectedCatalogVersion, policyEpoch), checkpoint and admin rebuild requests for the shared + /// host; SearchIndexLeases is the single-writer publish lease for the object store. Seeds the Search.Unified + /// feature flag off. Existence-guarded for safe retry. + /// + [Migration(208)] + public class M0208_AddUnifiedSearch : Migration + { + public override void Up() + { + if (!Schema.Table("SearchProjections").Exists()) + { + Create.Table("SearchProjections") + .WithColumn("SearchProjectionId").AsString(36).NotNullable().PrimaryKey() + .WithColumn("DepartmentId").AsInt32().NotNullable() + .WithColumn("EntityType").AsString(30).NotNullable() + .WithColumn("EntityId").AsString(128).NotNullable() + .WithColumn("Title").AsString(400).Nullable() + .WithColumn("Summary").AsString(1000).Nullable() + .WithColumn("SearchText").AsString(int.MaxValue).Nullable() + .WithColumn("Keywords").AsString(400).Nullable() + .WithColumn("Category").AsString(100).Nullable() + .WithColumn("Status").AsString(50).Nullable() + .WithColumn("Priority").AsInt32().Nullable() + .WithColumn("GroupId").AsInt32().Nullable() + .WithColumn("OwnerUserId").AsString(128).Nullable() + .WithColumn("ParticipantUserIds").AsString(int.MaxValue).Nullable() + .WithColumn("IsAdminOnly").AsBoolean().NotNullable().WithDefaultValue(false) + .WithColumn("IsActive").AsBoolean().NotNullable().WithDefaultValue(true) + .WithColumn("OccurredOn").AsDateTime2().NotNullable() + .WithColumn("Url").AsString(400).Nullable() + .WithColumn("MetadataJson").AsString(int.MaxValue).Nullable() + .WithColumn("ProtectedCatalogVersion").AsInt32().NotNullable().WithDefaultValue(0) + .WithColumn("PolicyEpoch").AsInt64().NotNullable().WithDefaultValue(0L) + .WithColumn("IncludesProtectedText").AsBoolean().NotNullable().WithDefaultValue(false) + .WithColumn("CreatedOn").AsDateTime2().NotNullable() + .WithColumn("ModifiedOn").AsDateTime2().NotNullable() + .WithColumn("RowVersion").AsInt64().NotNullable().WithDefaultValue(1L) + .WithColumn("DeletedOn").AsDateTime2().Nullable(); + + Execute.Sql("CREATE UNIQUE NONCLUSTERED INDEX UX_SearchProjections_Department_Entity ON SearchProjections (DepartmentId, EntityType, EntityId);"); + Create.Index("IX_SearchProjections_Department_Modified").OnTable("SearchProjections") + .OnColumn("DepartmentId").Ascending().OnColumn("ModifiedOn").Ascending(); + Create.Index("IX_SearchProjections_Department_Type_Modified").OnTable("SearchProjections") + .OnColumn("DepartmentId").Ascending().OnColumn("EntityType").Ascending().OnColumn("ModifiedOn").Ascending(); + } + + if (!Schema.Table("SearchIndexStates").Exists()) + { + Create.Table("SearchIndexStates") + .WithColumn("SearchIndexStateId").AsInt32().NotNullable().PrimaryKey().Identity() + .WithColumn("IndexName").AsString(50).NotNullable() + .WithColumn("DepartmentId").AsInt32().NotNullable() + .WithColumn("SchemaVersion").AsInt32().NotNullable().WithDefaultValue(1) + .WithColumn("ProtectedCatalogVersion").AsInt32().NotNullable().WithDefaultValue(0) + .WithColumn("PolicyEpoch").AsInt64().NotNullable().WithDefaultValue(0L) + .WithColumn("Generation").AsString(100).NotNullable() + .WithColumn("State").AsInt32().NotNullable().WithDefaultValue(0) + .WithColumn("DocumentCount").AsInt32().NotNullable().WithDefaultValue(0) + .WithColumn("LastRebuiltOn").AsDateTime2().Nullable() + .WithColumn("LastIndexedModifiedOn").AsDateTime2().Nullable() + .WithColumn("RebuildRequestedOn").AsDateTime2().Nullable() + .WithColumn("CreatedOn").AsDateTime2().NotNullable() + .WithColumn("ModifiedOn").AsDateTime2().NotNullable(); + + Execute.Sql("CREATE UNIQUE NONCLUSTERED INDEX UX_SearchIndexStates_Index_Department ON SearchIndexStates (IndexName, DepartmentId);"); + } + + if (!Schema.Table("SearchIndexLeases").Exists()) + { + Create.Table("SearchIndexLeases") + .WithColumn("IndexName").AsString(50).NotNullable().PrimaryKey() + .WithColumn("LeaseOwner").AsString(200).Nullable() + .WithColumn("LeaseExpiresOn").AsDateTime2().Nullable() + .WithColumn("LastPublishedRevision").AsString(64).Nullable() + .WithColumn("LastPublishedOn").AsDateTime2().Nullable() + .WithColumn("ModifiedOn").AsDateTime2().NotNullable(); + } + + Execute.Sql("IF NOT EXISTS (SELECT 1 FROM [FeatureFlags] WHERE [FlagKey] = 'Search.Unified') INSERT INTO [FeatureFlags] ([FlagKey], [Name], [Description], [Category], [IsEnabledGlobally]) VALUES ('Search.Unified', 'Unified Search', 'Cross-entity search (calls, units, personnel, contacts, messages, documents, notes, records) and the system-functionality command palette. Requires the search host (SearchConfig.Enabled) in every process. Seeded off.', 'Search', 0);"); + } + + public override void Down() + { + if (Schema.Table("SearchIndexLeases").Exists()) + Delete.Table("SearchIndexLeases"); + + if (Schema.Table("SearchIndexStates").Exists()) + Delete.Table("SearchIndexStates"); + + if (Schema.Table("SearchProjections").Exists()) + Delete.Table("SearchProjections"); + + // The flag row is operator-owned once seeded (overrides, targeting); Up tolerates its presence. + } + } +} diff --git a/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0208_AddUnifiedSearchPg.cs b/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0208_AddUnifiedSearchPg.cs new file mode 100644 index 000000000..7ba378273 --- /dev/null +++ b/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0208_AddUnifiedSearchPg.cs @@ -0,0 +1,98 @@ +using FluentMigrator; + +namespace Resgrid.Providers.MigrationsPg.Migrations +{ + /// + /// Unified Search Phase 1b/2 (plan R4, R5; registry §4F): searchprojections, searchindexstates, searchindexleases + /// and the Search.Unified flag seed. PostgreSQL twin of the SQL Server migration. Existence-guarded for safe retry. + /// + [Migration(208)] + public class M0208_AddUnifiedSearchPg : Migration + { + public override void Up() + { + if (!Schema.Table("searchprojections").Exists()) + { + Create.Table("searchprojections") + .WithColumn("searchprojectionid").AsCustom("citext").NotNullable().PrimaryKey() + .WithColumn("departmentid").AsInt32().NotNullable() + .WithColumn("entitytype").AsCustom("citext").NotNullable() + .WithColumn("entityid").AsCustom("citext").NotNullable() + .WithColumn("title").AsCustom("citext").Nullable() + .WithColumn("summary").AsCustom("citext").Nullable() + .WithColumn("searchtext").AsCustom("citext").Nullable() + .WithColumn("keywords").AsCustom("citext").Nullable() + .WithColumn("category").AsCustom("citext").Nullable() + .WithColumn("status").AsCustom("citext").Nullable() + .WithColumn("priority").AsInt32().Nullable() + .WithColumn("groupid").AsInt32().Nullable() + .WithColumn("owneruserid").AsCustom("citext").Nullable() + .WithColumn("participantuserids").AsCustom("citext").Nullable() + .WithColumn("isadminonly").AsBoolean().NotNullable().WithDefaultValue(false) + .WithColumn("isactive").AsBoolean().NotNullable().WithDefaultValue(true) + .WithColumn("occurredon").AsDateTime2().NotNullable() + .WithColumn("url").AsCustom("citext").Nullable() + .WithColumn("metadatajson").AsCustom("citext").Nullable() + .WithColumn("protectedcatalogversion").AsInt32().NotNullable().WithDefaultValue(0) + .WithColumn("policyepoch").AsInt64().NotNullable().WithDefaultValue(0L) + .WithColumn("includesprotectedtext").AsBoolean().NotNullable().WithDefaultValue(false) + .WithColumn("createdon").AsDateTime2().NotNullable() + .WithColumn("modifiedon").AsDateTime2().NotNullable() + .WithColumn("rowversion").AsInt64().NotNullable().WithDefaultValue(1L) + .WithColumn("deletedon").AsDateTime2().Nullable(); + + Execute.Sql("CREATE UNIQUE INDEX IF NOT EXISTS ux_searchprojections_department_entity ON searchprojections (departmentid, entitytype, entityid);"); + Execute.Sql("CREATE INDEX IF NOT EXISTS ix_searchprojections_department_modified ON searchprojections (departmentid, modifiedon);"); + Execute.Sql("CREATE INDEX IF NOT EXISTS ix_searchprojections_department_type_modified ON searchprojections (departmentid, entitytype, modifiedon);"); + } + + if (!Schema.Table("searchindexstates").Exists()) + { + Create.Table("searchindexstates") + .WithColumn("searchindexstateid").AsInt32().NotNullable().PrimaryKey().Identity() + .WithColumn("indexname").AsCustom("citext").NotNullable() + .WithColumn("departmentid").AsInt32().NotNullable() + .WithColumn("schemaversion").AsInt32().NotNullable().WithDefaultValue(1) + .WithColumn("protectedcatalogversion").AsInt32().NotNullable().WithDefaultValue(0) + .WithColumn("policyepoch").AsInt64().NotNullable().WithDefaultValue(0L) + .WithColumn("generation").AsCustom("citext").NotNullable() + .WithColumn("state").AsInt32().NotNullable().WithDefaultValue(0) + .WithColumn("documentcount").AsInt32().NotNullable().WithDefaultValue(0) + .WithColumn("lastrebuilton").AsDateTime2().Nullable() + .WithColumn("lastindexedmodifiedon").AsDateTime2().Nullable() + .WithColumn("rebuildrequestedon").AsDateTime2().Nullable() + .WithColumn("createdon").AsDateTime2().NotNullable() + .WithColumn("modifiedon").AsDateTime2().NotNullable(); + + Execute.Sql("CREATE UNIQUE INDEX IF NOT EXISTS ux_searchindexstates_index_department ON searchindexstates (indexname, departmentid);"); + } + + if (!Schema.Table("searchindexleases").Exists()) + { + Create.Table("searchindexleases") + .WithColumn("indexname").AsCustom("citext").NotNullable().PrimaryKey() + .WithColumn("leaseowner").AsCustom("citext").Nullable() + .WithColumn("leaseexpireson").AsDateTime2().Nullable() + .WithColumn("lastpublishedrevision").AsCustom("citext").Nullable() + .WithColumn("lastpublishedon").AsDateTime2().Nullable() + .WithColumn("modifiedon").AsDateTime2().NotNullable(); + } + + Execute.Sql("INSERT INTO featureflags (flagkey, name, description, category, isenabledglobally) SELECT 'Search.Unified', 'Unified Search', 'Cross-entity search (calls, units, personnel, contacts, messages, documents, notes, records) and the system-functionality command palette. Requires the search host (SearchConfig.Enabled) in every process. Seeded off.', 'Search', false WHERE NOT EXISTS (SELECT 1 FROM featureflags WHERE flagkey = 'Search.Unified');"); + } + + public override void Down() + { + if (Schema.Table("searchindexleases").Exists()) + Delete.Table("searchindexleases"); + + if (Schema.Table("searchindexstates").Exists()) + Delete.Table("searchindexstates"); + + if (Schema.Table("searchprojections").Exists()) + Delete.Table("searchprojections"); + + // Flag row preserved, as in the SQL Server twin. + } + } +} diff --git a/Repositories/Resgrid.Repositories.DataRepository/Modules/DataModule.cs b/Repositories/Resgrid.Repositories.DataRepository/Modules/DataModule.cs index 68d376fd1..0a0733024 100644 --- a/Repositories/Resgrid.Repositories.DataRepository/Modules/DataModule.cs +++ b/Repositories/Resgrid.Repositories.DataRepository/Modules/DataModule.cs @@ -319,6 +319,10 @@ protected override void Load(ContainerBuilder builder) builder.RegisterType().As().InstancePerLifetimeScope(); builder.RegisterType().As().InstancePerLifetimeScope(); builder.RegisterType().As().InstancePerLifetimeScope(); + // Unified Search (plan R4 Phase 1b/2, M0208): projections, per-index state and the single-writer publish lease. + builder.RegisterType().As().InstancePerLifetimeScope(); + builder.RegisterType().As().InstancePerLifetimeScope(); + builder.RegisterType().As().InstancePerLifetimeScope(); builder.RegisterType().As().InstancePerLifetimeScope(); builder.RegisterType().As().InstancePerLifetimeScope(); builder.RegisterType().As().InstancePerLifetimeScope(); diff --git a/Repositories/Resgrid.Repositories.DataRepository/SearchRepositories.cs b/Repositories/Resgrid.Repositories.DataRepository/SearchRepositories.cs new file mode 100644 index 000000000..3584503a2 --- /dev/null +++ b/Repositories/Resgrid.Repositories.DataRepository/SearchRepositories.cs @@ -0,0 +1,212 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Dapper; +using Resgrid.Model.Repositories; +using Resgrid.Model.Repositories.Connection; +using Resgrid.Model.Repositories.Queries; +using Resgrid.Model.Search; +using Resgrid.Repositories.DataRepository.Configs; + +namespace Resgrid.Repositories.DataRepository +{ + /// Unified Search projections (plan R2.3): the safe row per entity that feeds the global index. + public class SearchProjectionsRepository : RmsRepositoryBase, ISearchProjectionsRepository + { + public SearchProjectionsRepository(IConnectionProvider connectionProvider, SqlConfiguration sqlConfiguration, IUnitOfWork unitOfWork, IQueryFactory queryFactory) + : base(connectionProvider, sqlConfiguration, unitOfWork, queryFactory) { } + + public Task GetAsync(int departmentId, string entityType, string entityId) + { + return QueryFirstOrDefaultAsync( + $"SELECT * FROM {Tbl("SearchProjections")} WHERE {Col("DepartmentId")} = {P}DepartmentId AND {Col("EntityType")} = {P}EntityType AND {Col("EntityId")} = {P}EntityId", + new { DepartmentId = departmentId, EntityType = entityType, EntityId = entityId }); + } + + public async Task UpsertAsync(SearchProjection projection, CancellationToken cancellationToken = default) + { + if (projection == null) throw new ArgumentNullException(nameof(projection)); + if (projection.DepartmentId <= 0 || string.IsNullOrWhiteSpace(projection.EntityType) || string.IsNullOrWhiteSpace(projection.EntityId)) + throw new ArgumentException("A search projection needs a department, entity type and entity id."); + + var now = DateTime.UtcNow; + var existing = await GetAsync(projection.DepartmentId, projection.EntityType, projection.EntityId); + if (existing == null) + { + projection.SearchProjectionId = string.IsNullOrWhiteSpace(projection.SearchProjectionId) ? Guid.NewGuid().ToString() : projection.SearchProjectionId; + projection.CreatedOn = now; + projection.ModifiedOn = now; + projection.RowVersion = 1; + projection.DeletedOn = null; + try + { + return await InsertAsync(projection, cancellationToken, true); + } + catch (Exception) + { + // Two writers raced on the unique (DepartmentId, EntityType, EntityId) index; fall through to update. + existing = await GetAsync(projection.DepartmentId, projection.EntityType, projection.EntityId); + if (existing == null) throw; + } + } + + projection.SearchProjectionId = existing.SearchProjectionId; + projection.CreatedOn = existing.CreatedOn; + projection.ModifiedOn = now; + projection.RowVersion = existing.RowVersion + 1; + projection.DeletedOn = null; + return await UpdateAsync(projection, cancellationToken, true); + } + + public async Task SoftDeleteAsync(int departmentId, string entityType, string entityId, CancellationToken cancellationToken = default) + { + var now = DatabaseTimestamp(DateTime.UtcNow); + var rows = await ExecuteAsync( + $"UPDATE {Tbl("SearchProjections")} SET {Col("DeletedOn")} = {P}Now, {Col("ModifiedOn")} = {P}Now, {Col("RowVersion")} = {Col("RowVersion")} + 1 " + + $"WHERE {Col("DepartmentId")} = {P}DepartmentId AND {Col("EntityType")} = {P}EntityType AND {Col("EntityId")} = {P}EntityId AND {Col("DeletedOn")} IS NULL", + new { Now = now, DepartmentId = departmentId, EntityType = entityType, EntityId = entityId }, cancellationToken); + return rows > 0; + } + + public Task SoftDeleteStaleAsync(int departmentId, string entityType, DateTime notTouchedSince, CancellationToken cancellationToken = default) + { + var now = DatabaseTimestamp(DateTime.UtcNow); + return ExecuteAsync( + $"UPDATE {Tbl("SearchProjections")} SET {Col("DeletedOn")} = {P}Now, {Col("ModifiedOn")} = {P}Now, {Col("RowVersion")} = {Col("RowVersion")} + 1 " + + $"WHERE {Col("DepartmentId")} = {P}DepartmentId AND {Col("EntityType")} = {P}EntityType AND {Col("DeletedOn")} IS NULL AND {Col("ModifiedOn")} < {P}Since", + new { Now = now, DepartmentId = departmentId, EntityType = entityType, Since = DatabaseTimestamp(notTouchedSince) }, cancellationToken); + } + + public Task> GetModifiedSinceAsync(int departmentId, DateTime? since, int take, string sinceId = null) + { + var parameters = new DynamicParameters(); + parameters.Add("DepartmentId", departmentId); + parameters.Add("Skip", 0); + parameters.Add("Take", take <= 0 ? 500 : Math.Min(take, 5000)); + var sinceClause = string.Empty; + if (since.HasValue) + { + // The predicate has to match the ordering, tie-breaker included, or rows that share the cursor's + // timestamp and sort after it are dropped on the next page. + if (string.IsNullOrWhiteSpace(sinceId)) + { + sinceClause = $" AND {Col("ModifiedOn")} > {P}Since"; + } + else + { + sinceClause = $" AND ({Col("ModifiedOn")} > {P}Since OR ({Col("ModifiedOn")} = {P}Since AND {Col("SearchProjectionId")} > {P}SinceId))"; + parameters.Add("SinceId", sinceId); + } + + parameters.Add("Since", DatabaseTimestamp(since.Value), System.Data.DbType.DateTime2); + } + + return QueryAsync( + $"SELECT * FROM {Tbl("SearchProjections")} WHERE {Col("DepartmentId")} = {P}DepartmentId{sinceClause} ORDER BY {Col("ModifiedOn")} ASC, {Col("SearchProjectionId")} ASC {Paging()}", + parameters); + } + + public Task> GetLivePageAsync(int departmentId, int skip, int take) + { + var parameters = new DynamicParameters(); + parameters.Add("DepartmentId", departmentId); + parameters.Add("Skip", Math.Max(0, skip)); + parameters.Add("Take", take <= 0 ? 500 : Math.Min(take, 5000)); + return QueryAsync( + $"SELECT * FROM {Tbl("SearchProjections")} WHERE {Col("DepartmentId")} = {P}DepartmentId AND {Col("DeletedOn")} IS NULL ORDER BY {Col("SearchProjectionId")} ASC {Paging()}", + parameters); + } + + public async Task> GetByIdsAsync(int departmentId, IEnumerable projectionIds) + { + var rows = new List(); + foreach (var ids in (projectionIds ?? Enumerable.Empty()).Where(id => !string.IsNullOrWhiteSpace(id)).Distinct().Chunk(1000)) + { + var parameters = new DynamicParameters(); + parameters.Add("DepartmentId", departmentId); + parameters.Add("Ids", IsPostgres ? (object)ids : ids.ToList()); + rows.AddRange(await QueryAsync( + $"SELECT * FROM {Tbl("SearchProjections")} WHERE {Col("DepartmentId")} = {P}DepartmentId AND {Col("DeletedOn")} IS NULL AND {InList("SearchProjectionId", "Ids")}", + parameters)); + } + return rows; + } + + public Task HardDeleteDepartmentAsync(int departmentId, CancellationToken cancellationToken = default) + { + return ExecuteAsync($"DELETE FROM {Tbl("SearchProjections")} WHERE {Col("DepartmentId")} = {P}DepartmentId", new { DepartmentId = departmentId }, cancellationToken); + } + } + + public class SearchIndexStatesRepository : RmsRepositoryBase, ISearchIndexStatesRepository + { + public SearchIndexStatesRepository(IConnectionProvider connectionProvider, SqlConfiguration sqlConfiguration, IUnitOfWork unitOfWork, IQueryFactory queryFactory) + : base(connectionProvider, sqlConfiguration, unitOfWork, queryFactory) { } + + public Task GetAsync(string indexName, int departmentId) + { + return QueryFirstOrDefaultAsync( + $"SELECT * FROM {Tbl("SearchIndexStates")} WHERE {Col("IndexName")} = {P}IndexName AND {Col("DepartmentId")} = {P}DepartmentId", + new { IndexName = indexName, DepartmentId = departmentId }); + } + + public Task> GetAllForIndexAsync(string indexName) + { + return QueryAsync( + $"SELECT * FROM {Tbl("SearchIndexStates")} WHERE {Col("IndexName")} = {P}IndexName ORDER BY {Col("DepartmentId")} ASC", + new { IndexName = indexName }); + } + } + + /// The single-writer publish lease (plan R7 writer sequence step 2). One row per index name, compare-and-set. + public class SearchIndexLeasesRepository : RmsRepositoryBase, ISearchIndexLeasesRepository + { + public SearchIndexLeasesRepository(IConnectionProvider connectionProvider, SqlConfiguration sqlConfiguration, IUnitOfWork unitOfWork, IQueryFactory queryFactory) + : base(connectionProvider, sqlConfiguration, unitOfWork, queryFactory) { } + + public async Task TryAcquireAsync(string indexName, string owner, TimeSpan duration, DateTime utcNow, CancellationToken cancellationToken = default) + { + var now = DatabaseTimestamp(utcNow); + var until = DatabaseTimestamp(utcNow.Add(duration)); + var updated = await ExecuteAsync( + $"UPDATE {Tbl("SearchIndexLeases")} SET {Col("LeaseOwner")} = {P}Owner, {Col("LeaseExpiresOn")} = {P}Until, {Col("ModifiedOn")} = {P}Now " + + $"WHERE {Col("IndexName")} = {P}IndexName AND ({Col("LeaseOwner")} IS NULL OR {Col("LeaseExpiresOn")} IS NULL OR {Col("LeaseExpiresOn")} < {P}Now OR {Col("LeaseOwner")} = {P}Owner)", + new { Owner = owner, Until = until, Now = now, IndexName = indexName }, cancellationToken); + if (updated == 1) + return true; + + var exists = await ScalarAsync($"SELECT COUNT(1) FROM {Tbl("SearchIndexLeases")} WHERE {Col("IndexName")} = {P}IndexName", new { IndexName = indexName }, cancellationToken); + if (exists > 0) + return false; + + try + { + await ExecuteAsync( + $"INSERT INTO {Tbl("SearchIndexLeases")} ({Cols("IndexName", "LeaseOwner", "LeaseExpiresOn", "ModifiedOn")}) VALUES ({P}IndexName, {P}Owner, {P}Until, {P}Now)", + new { IndexName = indexName, Owner = owner, Until = until, Now = now }, cancellationToken); + return true; + } + catch (Exception) + { + // Lost the insert race; the other writer holds it. + return false; + } + } + + public Task ReleaseAsync(string indexName, string owner, CancellationToken cancellationToken = default) + { + return ExecuteAsync( + $"UPDATE {Tbl("SearchIndexLeases")} SET {Col("LeaseOwner")} = NULL, {Col("LeaseExpiresOn")} = NULL, {Col("ModifiedOn")} = {P}Now WHERE {Col("IndexName")} = {P}IndexName AND {Col("LeaseOwner")} = {P}Owner", + new { Now = DatabaseTimestamp(DateTime.UtcNow), IndexName = indexName, Owner = owner }, cancellationToken); + } + + public Task RecordPublishedAsync(string indexName, string owner, string revision, DateTime utcNow, CancellationToken cancellationToken = default) + { + return ExecuteAsync( + $"UPDATE {Tbl("SearchIndexLeases")} SET {Col("LastPublishedRevision")} = {P}Revision, {Col("LastPublishedOn")} = {P}Now, {Col("ModifiedOn")} = {P}Now WHERE {Col("IndexName")} = {P}IndexName AND {Col("LeaseOwner")} = {P}Owner", + new { Revision = revision, Now = DatabaseTimestamp(utcNow), IndexName = indexName, Owner = owner }, cancellationToken); + } + } +} diff --git a/Tests/Resgrid.Tests/Search/GlobalSearchTests.cs b/Tests/Resgrid.Tests/Search/GlobalSearchTests.cs new file mode 100644 index 000000000..49934825c --- /dev/null +++ b/Tests/Resgrid.Tests/Search/GlobalSearchTests.cs @@ -0,0 +1,165 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Lucene.Net.Store; +using NUnit.Framework; +using Resgrid.Config; +using Resgrid.Model.Search; +using Resgrid.Model.Services; +using Resgrid.Search; + +namespace Resgrid.Tests.Search +{ + /// + /// The global index end to end on an in-memory directory (Unified Search plan R4 Phase 2): department isolation, + /// message scope inside the query, admin-only filtering, prefix typeahead, exact identifiers and health. + /// + [TestFixture] + public class GlobalSearchTests + { + private LuceneGlobalIndexHost _host; + private LuceneGlobalSearchIndexer _indexer; + private LuceneGlobalSearchService _search; + + [SetUp] + public async Task SetUp() + { + SearchConfig.Enabled = true; + _host = new LuceneGlobalIndexHost(new RAMDirectory(), ownsDirectory: true); + _indexer = new LuceneGlobalSearchIndexer(_host); + _search = new LuceneGlobalSearchService(_host); + + await _indexer.IndexAsync(new[] + { + Projection(1, SearchEntityTypes.Call, "10", "Structure Fire - 123 Main St", keywords: "2026-000123 INC-77", summary: "Residential structure fire", category: "Fire", status: "Active", occurred: new DateTime(2026, 5, 1)), + Projection(1, SearchEntityTypes.Unit, "5", "Engine 2", keywords: "E2 1FTSW21P", category: "Engine", occurred: new DateTime(2026, 4, 1)), + Projection(1, SearchEntityTypes.Personnel, "u1", "Jane Doe", keywords: "1042", occurred: new DateTime(2026, 3, 1)), + Projection(1, SearchEntityTypes.Message, "300", "Shift swap Saturday", owner: "u2", participants: "u1,u3", occurred: new DateTime(2026, 6, 1)), + Projection(1, SearchEntityTypes.Note, "7", "Engine bay memo", adminOnly: true, occurred: new DateTime(2026, 2, 1)), + Projection(2, SearchEntityTypes.Call, "11", "Structure Fire - 9 Oak Ave", keywords: "2026-000009", occurred: new DateTime(2026, 5, 2)) + }, "1.25.3"); + await _indexer.CommitAsync(); + } + + [TearDown] + public void TearDown() + { + _host.Dispose(); + SearchConfig.Enabled = false; + } + + [Test] + public async Task Department_filter_is_always_injected() + { + var one = await _search.SearchAsync(1, new GlobalSearchQuery { Text = "structure fire", ViewerUserId = "u1" }); + var two = await _search.SearchAsync(2, new GlobalSearchQuery { Text = "structure fire", ViewerUserId = "u1" }); + + one.Hits.Select(h => h.EntityId).Should().BeEquivalentTo(new[] { "10" }); + two.Hits.Select(h => h.EntityId).Should().BeEquivalentTo(new[] { "11" }); + } + + [Test] + public async Task Messages_are_visible_only_to_sender_or_recipient() + { + var recipient = await _search.SearchAsync(1, new GlobalSearchQuery { Text = "shift swap", ViewerUserId = "u1" }); + var sender = await _search.SearchAsync(1, new GlobalSearchQuery { Text = "shift swap", ViewerUserId = "u2" }); + var stranger = await _search.SearchAsync(1, new GlobalSearchQuery { Text = "shift swap", ViewerUserId = "u9" }); + + recipient.Hits.Should().ContainSingle(h => h.EntityType == SearchEntityTypes.Message); + sender.Hits.Should().ContainSingle(h => h.EntityType == SearchEntityTypes.Message); + stranger.Hits.Should().BeEmpty(); + stranger.Total.Should().Be(0, "counts must not disclose messages the viewer cannot open"); + } + + [Test] + public async Task Admin_only_rows_are_hidden_unless_the_viewer_is_an_admin() + { + var member = await _search.SearchAsync(1, new GlobalSearchQuery { Text = "engine", ViewerUserId = "u1" }); + var admin = await _search.SearchAsync(1, new GlobalSearchQuery { Text = "engine", ViewerUserId = "u1", IncludeAdminOnly = true }); + + member.Hits.Select(h => h.EntityType).Should().BeEquivalentTo(new[] { SearchEntityTypes.Unit }); + admin.Hits.Select(h => h.EntityType).Should().BeEquivalentTo(new[] { SearchEntityTypes.Unit, SearchEntityTypes.Note }); + } + + [Test] + public async Task Prefix_mode_matches_titles_and_identifiers() + { + var eng = await _search.SearchAsync(1, new GlobalSearchQuery { Text = "eng", ViewerUserId = "u1", Prefix = true }); + var number = await _search.SearchAsync(1, new GlobalSearchQuery { Text = "2026-0001", ViewerUserId = "u1", Prefix = true }); + var name = await _search.SearchAsync(1, new GlobalSearchQuery { Text = "jan do", ViewerUserId = "u1", Prefix = true }); + + eng.Hits.Select(h => h.EntityId).Should().Contain("5"); + number.Hits.Select(h => h.EntityId).Should().BeEquivalentTo(new[] { "10" }); + name.Hits.Select(h => h.EntityId).Should().BeEquivalentTo(new[] { "u1" }); + } + + [Test] + public async Task Exact_identifier_ranks_first_and_partial_last_token_still_matches() + { + var exact = await _search.SearchAsync(1, new GlobalSearchQuery { Text = "INC-77", ViewerUserId = "u1" }); + var partial = await _search.SearchAsync(1, new GlobalSearchQuery { Text = "structure fi", ViewerUserId = "u1" }); + + exact.Hits.First().EntityId.Should().Be("10"); + partial.Hits.Select(h => h.EntityId).Should().Contain("10"); + } + + [Test] + public async Task Entity_type_filter_and_date_sort_without_text() + { + var units = await _search.SearchAsync(1, new GlobalSearchQuery { Text = "e", ViewerUserId = "u1", Prefix = true, EntityTypes = new List { SearchEntityTypes.Unit } }); + var newest = await _search.SearchAsync(1, new GlobalSearchQuery { ViewerUserId = "u1", IncludeAdminOnly = true }); + + units.Hits.Should().OnlyContain(h => h.EntityType == SearchEntityTypes.Unit); + newest.Hits.First().EntityType.Should().Be(SearchEntityTypes.Message, "no text sorts by OccurredOn descending"); + } + + [Test] + public async Task Query_input_is_escaped_and_field_selectors_never_reach_the_parser() + { + var result = await _search.SearchAsync(1, new GlobalSearchQuery { Text = "DepartmentId:2 OR Title:*", ViewerUserId = "u1" }); + result.Available.Should().BeTrue(); + result.Hits.Should().BeEmpty(); + } + + [Test] + public async Task Disabled_host_reports_unavailable_and_health() + { + var health = await _search.GetHealthAsync(); + health.Online.Should().BeTrue(); + health.DocumentCount.Should().Be(6); + health.IndexName.Should().Be(SearchIndexNames.Global); + + SearchConfig.Enabled = false; + var result = await _search.SearchAsync(1, new GlobalSearchQuery { Text = "fire" }); + result.Available.Should().BeFalse(); + } + + internal static SearchProjection Projection(int departmentId, string type, string id, string title, string keywords = null, string summary = null, + string category = null, string status = null, string owner = null, string participants = null, bool adminOnly = false, DateTime? occurred = null) + { + return new SearchProjection + { + SearchProjectionId = Guid.NewGuid().ToString(), + DepartmentId = departmentId, + EntityType = type, + EntityId = id, + Title = title, + Keywords = keywords, + Summary = summary, + Category = category, + Status = status, + OwnerUserId = owner, + ParticipantUserIds = participants, + IsAdminOnly = adminOnly, + IsActive = true, + OccurredOn = occurred ?? DateTime.UtcNow, + Url = "/User/" + type, + CreatedOn = DateTime.UtcNow, + ModifiedOn = DateTime.UtcNow, + RowVersion = 1 + }; + } + } +} diff --git a/Tests/Resgrid.Tests/Search/SearchIndexStoreSyncTests.cs b/Tests/Resgrid.Tests/Search/SearchIndexStoreSyncTests.cs new file mode 100644 index 000000000..297bd287b --- /dev/null +++ b/Tests/Resgrid.Tests/Search/SearchIndexStoreSyncTests.cs @@ -0,0 +1,237 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using Lucene.Net.Store; +using NUnit.Framework; +using Resgrid.Config; +using Resgrid.Model.Providers; +using Resgrid.Model.Search; +using Resgrid.Model.Services; +using Resgrid.Search; + +namespace Resgrid.Tests.Search +{ + /// + /// The object-store publish/pull cycle (Unified Search plan R7) against an in-memory store with S3 conditional-PUT + /// semantics: a writer publishes after commit, a reader in another directory pulls and serves, superseded objects + /// are pruned, a new writer pulls before it opens, and a manifest published elsewhere makes the next publish fail. + /// + [TestFixture] + public class SearchIndexStoreSyncTests + { + private readonly List _dirs = new List(); + private readonly List _hosts = new List(); + + [SetUp] + public void SetUp() + { + SearchConfig.Enabled = true; + } + + [TearDown] + public void TearDown() + { + foreach (var host in _hosts) { try { host.Dispose(); } catch { } } + _hosts.Clear(); + foreach (var dir in _dirs) { try { System.IO.Directory.Delete(dir, true); } catch { } } + _dirs.Clear(); + SearchConfig.Enabled = false; + } + + private string TempDir() + { + var dir = Path.Combine(Path.GetTempPath(), "rg-search-" + Guid.NewGuid().ToString("N")); + System.IO.Directory.CreateDirectory(dir); + _dirs.Add(dir); + return dir; + } + + private LuceneGlobalIndexHost Host(string dir, ISearchIndexStore store) + { + var host = new LuceneGlobalIndexHost(FSDirectory.Open(dir), true, store); + _hosts.Add(host); + return host; + } + + [Test] + public async Task Writer_publishes_reader_pulls_and_superseded_objects_are_pruned() + { + var store = new InMemorySearchIndexStore(); + var writer = Host(TempDir(), store); + var indexer = new LuceneGlobalSearchIndexer(writer); + + await indexer.IndexAsync(new[] { GlobalSearchTests.Projection(1, SearchEntityTypes.Call, "1", "Brush fire on Ridge Rd") }, "1.0.0"); + await indexer.CommitAsync(); + + store.Manifests.Should().ContainKey(SearchIndexNames.Global); + var first = store.Manifests[SearchIndexNames.Global]; + first.Files.Select(f => f.Name).Should().Contain(n => n.StartsWith("segments_")); + store.Objects[SearchIndexNames.Global].Keys.Should().BeEquivalentTo(first.Files.Select(f => f.Name)); + + var reader = Host(TempDir(), store); + (await reader.PullAsync()).Should().BeTrue(); + var search = new LuceneGlobalSearchService(reader); + (await search.SearchAsync(1, new GlobalSearchQuery { Text = "brush" })).Hits.Should().ContainSingle(); + reader.LastSyncedRevision.Should().Be(first.Revision); + + await indexer.IndexAsync(new[] { GlobalSearchTests.Projection(1, SearchEntityTypes.Unit, "2", "Brush 41") }, "1.0.0"); + await indexer.ExpungeDeletesAsync(); // force-merge + commit + publish + prune + + var second = store.Manifests[SearchIndexNames.Global]; + second.Revision.Should().NotBe(first.Revision); + store.Objects[SearchIndexNames.Global].Keys.Should().BeEquivalentTo(second.Files.Select(f => f.Name), "objects no longer referenced by the manifest are deleted"); + + (await reader.PullAsync()).Should().BeTrue(); + (await search.SearchAsync(1, new GlobalSearchQuery { Text = "brush" })).Hits.Should().HaveCount(2); + (await reader.PullAsync()).Should().BeFalse("nothing new to pull"); + System.IO.Directory.EnumerateFiles(reader.IndexPath).Select(Path.GetFileName).Where(n => n != "write.lock").Should().BeEquivalentTo(second.Files.Select(f => f.Name), "stale local files are removed after a pull"); + } + + [Test] + public async Task A_new_writer_pulls_the_published_index_before_opening() + { + var store = new InMemorySearchIndexStore(); + var first = Host(TempDir(), store); + var indexer = new LuceneGlobalSearchIndexer(first); + await indexer.IndexAsync(new[] { GlobalSearchTests.Projection(1, SearchEntityTypes.Note, "9", "Hydrant flow test") }, "1.0.0"); + await indexer.CommitAsync(); + first.Dispose(); + + var replacement = Host(TempDir(), store); + var replacementIndexer = new LuceneGlobalSearchIndexer(replacement); + await replacementIndexer.IndexAsync(new[] { GlobalSearchTests.Projection(1, SearchEntityTypes.Note, "10", "Hydrant map") }, "1.0.0"); + await replacementIndexer.CommitAsync(); + + var search = new LuceneGlobalSearchService(replacement); + (await search.SearchAsync(1, new GlobalSearchQuery { Text = "hydrant" })).Hits.Should().HaveCount(2, "the replacement writer started from the published revision, not an empty directory"); + } + + [Test] + public async Task Publish_fails_when_another_writer_published_first() + { + var store = new InMemorySearchIndexStore(); + var writer = Host(TempDir(), store); + var indexer = new LuceneGlobalSearchIndexer(writer); + await indexer.IndexAsync(new[] { GlobalSearchTests.Projection(1, SearchEntityTypes.Call, "1", "First") }, "1.0.0"); + await indexer.CommitAsync(); + + // Another writer swaps the manifest underneath us. + var current = store.Manifests[SearchIndexNames.Global]; + await store.PutManifestAsync(SearchIndexNames.Global, new SearchIndexManifest { IndexName = SearchIndexNames.Global, Revision = "elsewhere", SegmentsGeneration = current.SegmentsGeneration, Files = current.Files }, current.ETag); + + await indexer.IndexAsync(new[] { GlobalSearchTests.Projection(1, SearchEntityTypes.Call, "2", "Second") }, "1.0.0"); + Func act = () => indexer.CommitAsync(); + await act.Should().ThrowAsync(); + store.Manifests[SearchIndexNames.Global].Revision.Should().Be("elsewhere", "the losing writer never overwrites the other writer's manifest"); + } + + [Test] + public async Task Without_a_store_commit_is_local_only() + { + var writer = Host(TempDir(), null); + var indexer = new LuceneGlobalSearchIndexer(writer); + await indexer.IndexAsync(new[] { GlobalSearchTests.Projection(1, SearchEntityTypes.Call, "1", "Local only") }, "1.0.0"); + await indexer.CommitAsync(); + writer.StoreEnabled.Should().BeFalse(); + writer.LastSyncedRevision.Should().BeNull(); + } + } + + /// S3 semantics in memory: immutable objects, one manifest per index, If-None-Match:* / If-Match on the manifest. + public sealed class InMemorySearchIndexStore : ISearchIndexStore + { + private readonly object _sync = new object(); + private int _etag; + + public Dictionary> Objects { get; } = new Dictionary>(); + public Dictionary Manifests { get; } = new Dictionary(); + + public bool Enabled => true; + + public Task GetManifestAsync(string indexName, CancellationToken cancellationToken = default) + { + lock (_sync) + { + return Task.FromResult(Manifests.TryGetValue(indexName, out var m) ? Clone(m) : null); + } + } + + public Task> ListFilesAsync(string indexName, CancellationToken cancellationToken = default) + { + lock (_sync) + { + return Task.FromResult(Objects.TryGetValue(indexName, out var files) ? new HashSet(files.Keys) : new HashSet()); + } + } + + public Task UploadFileAsync(string indexName, string fileName, string localPath, CancellationToken cancellationToken = default) + { + var bytes = File.ReadAllBytes(localPath); + lock (_sync) + { + if (!Objects.TryGetValue(indexName, out var files)) + Objects[indexName] = files = new Dictionary(); + files[fileName] = bytes; + } + return Task.CompletedTask; + } + + public Task DownloadFileAsync(string indexName, string fileName, string localPath, CancellationToken cancellationToken = default) + { + byte[] bytes; + lock (_sync) + { + if (!Objects.TryGetValue(indexName, out var files) || !files.TryGetValue(fileName, out bytes)) + throw new FileNotFoundException(fileName); + } + File.WriteAllBytes(localPath, bytes); + return Task.CompletedTask; + } + + public Task DeleteFilesAsync(string indexName, IEnumerable fileNames, CancellationToken cancellationToken = default) + { + lock (_sync) + { + if (Objects.TryGetValue(indexName, out var files)) + foreach (var name in fileNames) files.Remove(name); + } + return Task.CompletedTask; + } + + public Task PutManifestAsync(string indexName, SearchIndexManifest manifest, string expectedETag, CancellationToken cancellationToken = default) + { + lock (_sync) + { + Manifests.TryGetValue(indexName, out var current); + if (expectedETag == null && current != null) + throw new SearchIndexManifestConflictException(indexName, "manifest exists"); + if (expectedETag != null && (current == null || current.ETag != expectedETag)) + throw new SearchIndexManifestConflictException(indexName, "etag mismatch"); + var stored = Clone(manifest); + stored.ETag = "\"" + (++_etag) + "\""; + Manifests[indexName] = stored; + manifest.ETag = stored.ETag; + return Task.FromResult(manifest); + } + } + + private static SearchIndexManifest Clone(SearchIndexManifest m) + { + return new SearchIndexManifest + { + IndexName = m.IndexName, + Revision = m.Revision, + SegmentsGeneration = m.SegmentsGeneration, + SchemaVersion = m.SchemaVersion, + PublishedOnUtc = m.PublishedOnUtc, + PublishedBy = m.PublishedBy, + Files = (m.Files ?? new List()).Select(f => new SearchIndexManifestFile { Name = f.Name, Length = f.Length }).ToList(), + ETag = m.ETag + }; + } + } +} diff --git a/Tests/Resgrid.Tests/Search/SearchProjectionServiceTests.cs b/Tests/Resgrid.Tests/Search/SearchProjectionServiceTests.cs new file mode 100644 index 000000000..14a3db889 --- /dev/null +++ b/Tests/Resgrid.Tests/Search/SearchProjectionServiceTests.cs @@ -0,0 +1,154 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using Moq; +using NUnit.Framework; +using Resgrid.Model; +using Resgrid.Model.Repositories; +using Resgrid.Model.Search; +using Resgrid.Model.Services; +using Resgrid.Services.Search; + +namespace Resgrid.Tests.Search +{ + /// + /// The projection allowlist (plan R2.15, R3): cataloged columns only where protection is not enforced, envelopes + /// and the redaction placeholder never, hooks never throw, and the profile hook keeps group/active state it cannot know. + /// + [TestFixture] + public class SearchProjectionServiceTests + { + private Mock _repo; + private Mock _protection; + private SearchProjectionService _service; + private bool _enforced; + + [SetUp] + public void SetUp() + { + _enforced = false; + _repo = new Mock(); + _repo.Setup(r => r.UpsertAsync(It.IsAny(), It.IsAny())).ReturnsAsync((SearchProjection p, CancellationToken _) => p); + _protection = new Mock(); + _protection.Setup(p => p.IsProtectionEnforcedAsync(It.IsAny())).ReturnsAsync(() => _enforced); + _protection.Setup(p => p.GetPinnedCatalogVersionAsync(It.IsAny())).ReturnsAsync(25); + _protection.Setup(p => p.GetPolicyByDepartmentIdAsync(It.IsAny(), It.IsAny())).ReturnsAsync(new DepartmentDataProtectionPolicy { PolicyEpoch = 3 }); + _service = new SearchProjectionService(_repo.Object, _protection.Object); + } + + private static Call Call() => new Call + { + CallId = 42, DepartmentId = 7, Number = "2026-000042", Name = "House Fire", NatureOfCall = "Smoke showing", Type = "Fire", + Address = "123 Main St", IncidentNumber = "INC-9", Priority = 3, State = (int)CallStates.Active, LoggedOn = new DateTime(2026, 5, 1) + }; + + [Test] + public async Task Unprotected_department_projects_name_address_and_identifiers() + { + var p = await _service.BuildCallAsync(Call()); + + p.Title.Should().Be("House Fire"); + p.SearchText.Should().Contain("123 Main St"); + p.Keywords.Should().Contain("2026-000042").And.Contain("INC-9"); + p.Status.Should().Be("Active"); + p.Priority.Should().Be(3); + p.IncludesProtectedText.Should().BeTrue(); + p.ProtectedCatalogVersion.Should().Be(25); + p.PolicyEpoch.Should().Be(3); + p.Url.Should().Be("/User/Dispatch/ViewCall?callId=42"); + } + + [Test] + public async Task Enforced_department_projects_only_system_fields() + { + _enforced = true; + var p = await _service.BuildCallAsync(Call()); + + p.Title.Should().Be("Call 2026-000042"); + p.SearchText.Should().BeNull(); + p.Summary.Should().BeNull(); + p.Keywords.Should().Be("2026-000042"); + p.IncludesProtectedText.Should().BeFalse(); + } + + [Test] + public async Task Envelopes_and_the_redaction_placeholder_are_never_projected() + { + var call = Call(); + call.Name = ProtectedDataEnvelope.Prefix + "AAAA"; + call.Address = ProtectedDataEnvelope.RedactionValue; + + var p = await _service.BuildCallAsync(call); + + p.Title.Should().Be("Call 2026-000042"); + (p.SearchText ?? string.Empty).Should().NotContain(ProtectedDataEnvelope.Prefix).And.NotContain(ProtectedDataEnvelope.RedactionValue); + } + + [Test] + public async Task Contacts_are_not_projected_at_all_under_enforcement() + { + _enforced = true; + var p = await _service.BuildContactAsync(new Contact { ContactId = "c1", DepartmentId = 7, FirstName = "Ada", LastName = "Lovelace", CompanyName = "Analytical" }); + p.Should().BeNull(); + + _enforced = false; + var open = await _service.BuildContactAsync(new Contact { ContactId = "c1", DepartmentId = 7, FirstName = "Ada", LastName = "Lovelace", CompanyName = "Analytical", CellPhoneNumber = "(555) 010-2020" }); + open.Title.Should().Be("Ada Lovelace"); + open.Keywords.Should().Contain("5550102020"); + } + + [Test] + public async Task Notes_are_projected_regardless_with_html_stripped() + { + _enforced = true; + var p = await _service.BuildNoteAsync(new Note { NoteId = 3, DepartmentId = 7, Title = "Bay doors", Body = "

Door two sticks

", IsAdminOnly = true, AddedOn = new DateTime(2026, 1, 1) }); + + p.Title.Should().Be("Bay doors"); + p.SearchText.Should().Be("Door two sticks"); + p.IsAdminOnly.Should().BeTrue(); + } + + [Test] + public async Task Messages_carry_sender_and_recipients_for_query_scoping() + { + var p = await _service.BuildMessageAsync(new Message + { + MessageId = 5, DepartmentId = 7, Subject = "Trade", Body = "Can anyone cover Saturday?", SendingUserId = "u2", SentOn = new DateTime(2026, 6, 1), + MessageRecipients = new System.Collections.Generic.List { new MessageRecipient { UserId = "u1" }, new MessageRecipient { UserId = "u3", IsDeleted = true } } + }); + + p.OwnerUserId.Should().Be("u2"); + p.ParticipantUserIds.Should().Be("u1"); + p.Title.Should().Be("Trade"); + } + + [Test] + public async Task Hooks_never_throw_and_soft_delete_when_nothing_is_indexable() + { + _repo.Setup(r => r.UpsertAsync(It.IsAny(), It.IsAny())).ThrowsAsync(new InvalidOperationException("db down")); + Func act = () => _service.ProjectCallAsync(Call()); + await act.Should().NotThrowAsync(); + + var deleted = Call(); + deleted.IsDeleted = true; + await _service.ProjectCallAsync(deleted); + _repo.Verify(r => r.SoftDeleteAsync(7, SearchEntityTypes.Call, "42", It.IsAny()), Times.Once); + } + + [Test] + public async Task Profile_hook_keeps_group_and_active_state_it_does_not_know() + { + _repo.Setup(r => r.GetAsync(7, SearchEntityTypes.Personnel, "u1")).ReturnsAsync(new SearchProjection { GroupId = 4, IsActive = false }); + SearchProjection stored = null; + _repo.Setup(r => r.UpsertAsync(It.IsAny(), It.IsAny())).Callback((SearchProjection p, CancellationToken _) => stored = p).ReturnsAsync((SearchProjection p, CancellationToken _) => p); + + await _service.ProjectPersonnelAsync(7, new UserProfile { UserId = "u1", FirstName = "Jane", LastName = "Doe" }, null, null); + + stored.Should().NotBeNull(); + stored.GroupId.Should().Be(4); + stored.IsActive.Should().BeFalse(); + stored.Title.Should().Be("Jane Doe"); + } + } +} diff --git a/Tests/Resgrid.Tests/Search/SystemActionsServiceTests.cs b/Tests/Resgrid.Tests/Search/SystemActionsServiceTests.cs new file mode 100644 index 000000000..40b94137d --- /dev/null +++ b/Tests/Resgrid.Tests/Search/SystemActionsServiceTests.cs @@ -0,0 +1,112 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Moq; +using NUnit.Framework; +using Resgrid.Model; +using Resgrid.Model.Search; +using Resgrid.Model.Services; +using Resgrid.Services.Search; + +namespace Resgrid.Tests.Search +{ + /// System functionality search: claim, admin, module and flag gates are applied before scoring; scoring handles slash commands, prefixes and one-edit typos. + [TestFixture] + public class SystemActionsServiceTests + { + private Mock _flags; + private SystemActionsService _service; + private HashSet _enabledFlags; + private HashSet _claims; + private HashSet _disabledModules; + + [SetUp] + public void SetUp() + { + _enabledFlags = new HashSet(); + _claims = new HashSet(); + _disabledModules = new HashSet(); + _flags = new Mock(); + _flags.Setup(f => f.IsEnabledAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync((string key, int dept, bool def, IDictionary ctx) => _enabledFlags.Contains(key)); + _service = new SystemActionsService(_flags.Object); + } + + private SearchPrincipal Principal(bool admin = false) => new SearchPrincipal + { + UserId = "u1", + DepartmentId = 7, + IsDepartmentAdmin = admin, + HasClaim = (resource, action) => _claims.Contains(resource + ":" + action), + IsModuleEnabled = module => !_disabledModules.Contains(module) + }; + + [Test] + public async Task New_call_needs_the_create_claim() + { + _claims.Add("Call:View"); + var without = await _service.SearchAsync("new call", Principal()); + without.Select(h => h.Key).Should().NotContain("new-call").And.Contain("calls"); + + _claims.Add("Call:Create"); + var with = await _service.SearchAsync("new call", Principal()); + with.First().Key.Should().Be("new-call"); + with.First().Url.Should().EndWith("/User/Dispatch/NewCall"); + } + + [Test] + public async Task Slash_commands_prefixes_and_typos_resolve() + { + _claims.Add("Call:View"); + _claims.Add("Personnel:View"); + (await _service.SearchAsync("/calls", Principal())).First().Key.Should().Be("calls"); + (await _service.SearchAsync("pers", Principal())).First().Key.Should().Be("personnel"); + (await _service.SearchAsync("personel", Principal())).Select(h => h.Key).Should().Contain("personnel"); + (await _service.SearchAsync("xyzzy", Principal())).Should().BeEmpty(); + } + + [Test] + public async Task Feature_flag_and_module_gates_apply() + { + _claims.Add("Messages:View"); + (await _service.SearchAsync("chat", Principal())).Should().BeEmpty("Chat.System is off"); + _enabledFlags.Add(FeatureFlagKeys.ChatSystem); + (await _service.SearchAsync("chat", Principal())).Select(h => h.Key).Should().Contain("chat"); + + (await _service.SearchAsync("inbox", Principal())).Select(h => h.Key).Should().Contain("inbox"); + _disabledModules.Add(SystemActionModules.Messaging); + (await _service.SearchAsync("inbox", Principal())).Should().BeEmpty("the messaging module is disabled for the department"); + } + + [Test] + public async Task Logs_disappear_when_records_is_on_and_admin_entries_need_admin() + { + _claims.Add("Log:View"); + (await _service.SearchAsync("logs", Principal())).Select(h => h.Key).Should().Contain("logs"); + _enabledFlags.Add(FeatureFlagKeys.RecordsSystem); + (await _service.SearchAsync("logs", Principal())).Select(h => h.Key).Should().NotContain("logs"); + + (await _service.SearchAsync("department settings", Principal())).Should().BeEmpty(); + (await _service.SearchAsync("department settings", Principal(admin: true))).First().Key.Should().Be("department-settings"); + } + + [Test] + public async Task Admins_see_every_claim_gated_entry_and_list_returns_them_unscored() + { + var list = await _service.ListAsync(Principal(admin: true)); + list.Select(l => l.Key).Should().Contain(new[] { "calls", "new-call", "personnel", "units", "contacts" }); + list.Should().OnlyContain(l => l.Score == 0f); + list.Should().OnlyContain(l => l.Url.Contains("/User/")); + } + + [Test] + public void Within_one_edit_is_symmetric_and_bounded() + { + SystemActionsService.WithinOneEdit("personel", "personnel").Should().BeTrue(); + SystemActionsService.WithinOneEdit("personnel", "personel").Should().BeTrue(); + SystemActionsService.WithinOneEdit("unit", "units").Should().BeTrue(); + SystemActionsService.WithinOneEdit("calls", "chats").Should().BeFalse(); + } + } +} diff --git a/Tests/Resgrid.Tests/Search/UnifiedSearchServiceTests.cs b/Tests/Resgrid.Tests/Search/UnifiedSearchServiceTests.cs new file mode 100644 index 000000000..424a76fc5 --- /dev/null +++ b/Tests/Resgrid.Tests/Search/UnifiedSearchServiceTests.cs @@ -0,0 +1,131 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using Moq; +using NUnit.Framework; +using Resgrid.Model; +using Resgrid.Model.Repositories; +using Resgrid.Model.Search; +using Resgrid.Model.Services; +using Resgrid.Services.Search; + +namespace Resgrid.Tests.Search +{ + /// The unified orchestrator: flag gate, claim-based family filter, per-hit authorization with total suppression, lazy state-row activation. + [TestFixture] + public class UnifiedSearchServiceTests + { + private Mock _global; + private Mock _actions; + private Mock _flags; + private Mock _auth; + private Mock _states; + private Mock _recordsSearch; + private UnifiedSearchService _service; + private bool _flagOn; + private GlobalSearchQuery _lastQuery; + + [SetUp] + public void SetUp() + { + _flagOn = true; + _global = new Mock(); + _global.SetupGet(g => g.IsAvailable).Returns(true); + _global.Setup(g => g.SearchAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((int d, GlobalSearchQuery q, CancellationToken _) => _lastQuery = q) + .ReturnsAsync(() => new GlobalSearchResult + { + Total = 2, + Hits = new List + { + new GlobalSearchHit { EntityType = SearchEntityTypes.Call, EntityId = "1", Title = "One", Score = 2f }, + new GlobalSearchHit { EntityType = SearchEntityTypes.Call, EntityId = "2", Title = "Two", Score = 1f } + } + }); + _actions = new Mock(); + _actions.Setup(a => a.SearchAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(new List { new SystemActionHit { Key = "calls" } }); + _actions.Setup(a => a.ListAsync(It.IsAny(), It.IsAny())).ReturnsAsync(new List()); + _flags = new Mock(); + _flags.Setup(f => f.IsEnabledAsync(FeatureFlagKeys.SearchUnified, 7, It.IsAny(), It.IsAny>())).ReturnsAsync(() => _flagOn); + _auth = new Mock(); + _auth.Setup(a => a.CanUserViewCallAsync("u1", It.IsAny())).ReturnsAsync(true); + _states = new Mock(); + _recordsSearch = new Mock(); + _recordsSearch.SetupGet(r => r.IsAvailable).Returns(false); + + _service = new UnifiedSearchService(_global.Object, _actions.Object, _flags.Object, _auth.Object, _states.Object, _recordsSearch.Object, + new Mock().Object, new Mock().Object, new Mock().Object); + } + + private static SearchPrincipal Principal(params string[] claims) => new SearchPrincipal + { + UserId = "u1", + DepartmentId = 7, + HasClaim = (r, a) => claims.Contains(r + ":" + a) + }; + + [Test] + public async Task Flag_off_is_unavailable_and_returns_nothing() + { + _flagOn = false; + var result = await _service.SearchAsync(new UnifiedSearchRequest { Text = "one" }, Principal("Call:View")); + result.Available.Should().BeFalse(); + result.Hits.Should().BeEmpty(); + result.Actions.Should().BeEmpty(); + } + + [Test] + public async Task All_hits_authorized_keeps_the_total() + { + var result = await _service.SearchAsync(new UnifiedSearchRequest { Text = "one" }, Principal("Call:View")); + result.Available.Should().BeTrue(); + result.Hits.Should().HaveCount(2); + result.Total.Should().Be(2); + result.Actions.Should().ContainSingle(a => a.Key == "calls"); + } + + [Test] + public async Task A_dropped_hit_suppresses_the_total() + { + _auth.Setup(a => a.CanUserViewCallAsync("u1", 2)).ReturnsAsync(false); + var result = await _service.SearchAsync(new UnifiedSearchRequest { Text = "one" }, Principal("Call:View")); + result.Hits.Select(h => h.EntityId).Should().BeEquivalentTo(new[] { "1" }); + result.Total.Should().BeNull(); + } + + [Test] + public async Task Claims_decide_which_families_reach_the_index() + { + await _service.SearchAsync(new UnifiedSearchRequest { Text = "one" }, Principal("Call:View", "Notes:View")); + _lastQuery.EntityTypes.Should().BeEquivalentTo(new[] { SearchEntityTypes.Call, SearchEntityTypes.Note }); + _lastQuery.ViewerUserId.Should().Be("u1"); + _lastQuery.IncludeAdminOnly.Should().BeFalse(); + + _lastQuery = null; + var none = await _service.SearchAsync(new UnifiedSearchRequest { Text = "one" }, Principal()); + _lastQuery.Should().BeNull("no family is searchable without a view claim"); + none.Hits.Should().BeEmpty(); + } + + [Test] + public async Task Index_unavailable_degrades_and_activates_the_department_lazily() + { + _global.SetupGet(g => g.IsAvailable).Returns(false); + _states.Setup(s => s.GetAsync(SearchIndexNames.Global, 7)).ReturnsAsync((SearchIndexState)null); + SearchIndexState saved = null; + _states.Setup(s => s.SaveOrUpdateAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((SearchIndexState s, CancellationToken _, bool __) => saved = s).ReturnsAsync((SearchIndexState s, CancellationToken _, bool __) => s); + + var result = await _service.SearchAsync(new UnifiedSearchRequest { Text = "one" }, Principal("Call:View")); + + result.Available.Should().BeTrue(); + result.Degraded.Should().BeTrue(); + result.Actions.Should().NotBeEmpty("system functionality still resolves while the index is building"); + saved.Should().NotBeNull(); + saved.State.Should().Be((int)SearchIndexBuildState.RebuildRequested); + saved.IndexName.Should().Be(SearchIndexNames.Global); + } + } +} diff --git a/Tests/Resgrid.Tests/Web/workspace-wizard.test.cjs b/Tests/Resgrid.Tests/Web/workspace-wizard.test.cjs index 0b27f027c..0f592f6f1 100644 --- a/Tests/Resgrid.Tests/Web/workspace-wizard.test.cjs +++ b/Tests/Resgrid.Tests/Web/workspace-wizard.test.cjs @@ -60,6 +60,41 @@ const markup = ` `; +// A wizard inside a Bootstrap modal, with the stand in for jQuery's delegated event binding +// that the script uses to hear shown.bs.modal, and the workspace stylesheet rules that hide +// and show the modal message. +const modalMarkup = ` + + + +`; + (async () => { const browser = await chromium.launch(require('./browser-launch.cjs').launchOptions()); try { @@ -139,7 +174,44 @@ const markup = ` }); assert.equal(result, 'ok'); - console.log('Workspace wizard step navigation, review and submit guard passed.'); + + // Reopening a modal wizard starts it clean: the error the last attempt left on a step is + // gone, and the modal's message is hidden in a way notify() can undo by adding .rgw-visible + // (an inline display:none would outrank that class and swallow every later message). + const modalPage = await browser.newPage(); + await modalPage.setContent(modalMarkup); + await modalPage.addScriptTag({ content: script }); + const modalResult = await modalPage.evaluate(() => { + function check(condition, message) { if (!condition) throw new Error(message); } + + const modal = document.getElementById('dialog'); + const form = modal.querySelector('form'); + const steps = Array.from(form.querySelectorAll('.rgw-step')); + const message = modal.querySelector('.rgw-modal-message'); + + form.querySelector('.rgw-step-next').click(); + check(!steps[0].hidden && steps[0].querySelector('.rgw-step-error').hidden === false, 'The incomplete step did not report an error.'); + + message.textContent = 'Saved'; + message.className = 'alert rgw-modal-message rgw-visible alert-info'; + message.hidden = false; + + modal.dispatchEvent(new Event('shown.bs.modal', { bubbles: true })); + + check(!steps[0].hidden, 'Reopening did not return to the first step.'); + check(steps[0].querySelector('.rgw-step-error').hidden === true, 'Reopening left the previous error on the step.'); + check(!form.querySelector('.rgw-invalid'), 'Reopening left a field marked invalid.'); + check(message.hidden === true && !message.classList.contains('rgw-visible'), 'Reopening did not hide the previous message.'); + check(getComputedStyle(message).display === 'none', 'The previous message was still visible after reopening.'); + + message.className = 'alert rgw-modal-message rgw-visible alert-danger'; + message.hidden = false; + check(getComputedStyle(message).display === 'block', 'A message shown after reopening stayed hidden.'); + return 'ok'; + }); + + assert.equal(modalResult, 'ok'); + console.log('Workspace wizard step navigation, review, submit guard and modal reopen passed.'); } finally { await browser.close(); } diff --git a/Tools/Resgrid.Console/Commands/FeatureFlagsCommand.cs b/Tools/Resgrid.Console/Commands/FeatureFlagsCommand.cs index 25c3e0c5b..eddea89f6 100644 --- a/Tools/Resgrid.Console/Commands/FeatureFlagsCommand.cs +++ b/Tools/Resgrid.Console/Commands/FeatureFlagsCommand.cs @@ -308,14 +308,13 @@ private async Task SetAsync(string[] args, string key, int? department var value = GetValue(args, "Value"); var reason = GetValue(args, "Reason") ?? "Set from the Resgrid Console"; - try - { - await featureToggleService.SetDepartmentOverrideAsync(key, departmentId.Value, enabled, value, reason, expiresOn, userId, cancellationToken); - } - catch (InvalidOperationException) - { + // Check the flag up front rather than mapping InvalidOperationException to "not found": + // the service also throws that for a transaction that is already open, and a + // persistence failure should surface as the error it is, not as a missing flag. + if (await featureToggleService.GetFlagByKeyAsync(key, bypassCache: true) == null) return FlagNotFound(key); - } + + await featureToggleService.SetDepartmentOverrideAsync(key, departmentId.Value, enabled, value, reason, expiresOn, userId, cancellationToken); Write($"Override for '{key}' on department {departmentId.Value} ({department.Name}) is now {State(enabled)}{Expiry(expiresOn)}."); await WriteEvaluationAsync(key, departmentId.Value, department.Name); diff --git a/Web/Resgrid.Web.Services/Controllers/v4/HealthController.cs b/Web/Resgrid.Web.Services/Controllers/v4/HealthController.cs index ef1edaf59..981249e99 100644 --- a/Web/Resgrid.Web.Services/Controllers/v4/HealthController.cs +++ b/Web/Resgrid.Web.Services/Controllers/v4/HealthController.cs @@ -19,10 +19,14 @@ public class HealthController : V4AuthenticatedApiControllerbase { #region Members and Constructors private readonly IHealthService _healthService; + private readonly IGlobalSearchService _globalSearch; + private readonly IRecordsSearchService _recordsSearch; - public HealthController(IHealthService healthService) + public HealthController(IHealthService healthService, IGlobalSearchService globalSearch, IRecordsSearchService recordsSearch) { _healthService = healthService; + _globalSearch = globalSearch; + _recordsSearch = recordsSearch; } #endregion Members and Constructors @@ -44,6 +48,24 @@ public async Task GetCurrent() result.Data.SiteId = "0"; result.Data.CacheOnline = _healthService.IsCacheProviderConnected(); + // Unified Search plan R2.11: host state for this process. Never fails the health call. + try + { + result.Data.SearchEnabled = Config.SearchConfig.Enabled; + if (Config.SearchConfig.Enabled) + { + var global = await _globalSearch.GetHealthAsync(); + var records = await _recordsSearch.GetHealthAsync(); + result.Data.SearchOnline = global.Online; + result.Data.SearchIndexDocCount = global.DocumentCount + records.DocumentCount; + } + } + catch (System.Exception ex) + { + Resgrid.Framework.Logging.LogException(ex, "Search health could not be read."); + result.Data.SearchOnline = false; + } + var dbTime = await _healthService.GetDatabaseTimestamp(); if (!string.IsNullOrWhiteSpace(dbTime)) diff --git a/Web/Resgrid.Web.Services/Controllers/v4/SearchController.cs b/Web/Resgrid.Web.Services/Controllers/v4/SearchController.cs new file mode 100644 index 000000000..ad5525072 --- /dev/null +++ b/Web/Resgrid.Web.Services/Controllers/v4/SearchController.cs @@ -0,0 +1,265 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Resgrid.Framework; +using Resgrid.Model; +using Resgrid.Model.Repositories; +using Resgrid.Model.Search; +using Resgrid.Model.Services; +using Resgrid.Providers.Claims; +using Resgrid.Web.Services.Helpers; +using Resgrid.Web.Services.Models.v4.Search; +using Resgrid.Web.ServicesCore.Helpers; + +namespace Resgrid.Web.Services.Controllers.v4 +{ + /// + /// Unified Search (plan R4 Phase 2/3): one endpoint over the global index (calls, units, personnel, contacts, + /// messages, documents, notes), the RMS records index, and the system-functionality catalog. The department and + /// the caller's identity come from the token, never from the request. Each hit is re-checked against the entity's + /// own authorization rule; totals are null when any hit was dropped. Typeahead is the same query in prefix mode. + /// + [Route("api/v{VersionId:apiVersion}/[controller]")] + [ApiVersion("4.0")] + [ApiExplorerSettings(GroupName = "v4")] + public class SearchController : V4AuthenticatedApiControllerbase + { + private const int MaxTake = 100; + + private readonly IUnifiedSearchService _unifiedSearch; + private readonly IGlobalSearchService _globalSearch; + private readonly IRecordsSearchService _recordsSearch; + private readonly ISearchIndexMaintenanceService _maintenance; + private readonly ISearchIndexStatesRepository _states; + private readonly IDepartmentSettingsService _departmentSettings; + private readonly IFeatureToggleService _featureToggles; + + public SearchController(IUnifiedSearchService unifiedSearch, IGlobalSearchService globalSearch, IRecordsSearchService recordsSearch, + ISearchIndexMaintenanceService maintenance, ISearchIndexStatesRepository states, IDepartmentSettingsService departmentSettings, + IFeatureToggleService featureToggles) + { + _unifiedSearch = unifiedSearch; + _globalSearch = globalSearch; + _recordsSearch = recordsSearch; + _maintenance = maintenance; + _states = states; + _departmentSettings = departmentSettings; + _featureToggles = featureToggles; + } + + /// + /// Full search. is a comma-separated list of entity types (Call, Unit, Personnel, + /// Contact, Message, Document, Note, Record, Action); omit for all. + /// + [HttpGet("Search")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status400BadRequest)] + [ProducesResponseType(StatusCodes.Status404NotFound)] + [Authorize(Policy = ResgridResources.Department_View)] + public async Task> Search(string query, string types = null, int skip = 0, int take = 20, CancellationToken cancellationToken = default) + { + if (!await _featureToggles.IsEnabledAsync(FeatureFlagKeys.SearchUnified, DepartmentId)) + return NotFound(); + if (string.IsNullOrWhiteSpace(query)) + return BadRequest(); + + var requestedTypes = ParseTypes(types); + var unified = await _unifiedSearch.SearchAsync(new UnifiedSearchRequest + { + Text = query, + EntityTypes = requestedTypes, + Skip = Math.Max(0, skip), + Take = Math.Max(1, Math.Min(MaxTake, take)), + IncludeActions = requestedTypes == null || requestedTypes.Any(t => string.Equals(t, SearchEntityTypes.Action, StringComparison.OrdinalIgnoreCase)), + IncludeRecords = requestedTypes == null || requestedTypes.Any(t => string.Equals(t, SearchEntityTypes.Record, StringComparison.OrdinalIgnoreCase)), + Prefix = false + }, await BuildPrincipalAsync(), cancellationToken); + + return Ok(Map(unified, skip, take)); + } + + /// Typeahead: prefix search on titles and identifiers, no records federation, small result list. + [HttpGet("Typeahead")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status404NotFound)] + [Authorize(Policy = ResgridResources.Department_View)] + public async Task> Typeahead(string query, string types = null, int take = 8, CancellationToken cancellationToken = default) + { + if (!await _featureToggles.IsEnabledAsync(FeatureFlagKeys.SearchUnified, DepartmentId)) + return NotFound(); + + var requestedTypes = ParseTypes(types); + var unified = await _unifiedSearch.SearchAsync(new UnifiedSearchRequest + { + Text = query ?? string.Empty, + EntityTypes = requestedTypes, + Skip = 0, + Take = Math.Max(1, Math.Min(25, take)), + IncludeActions = requestedTypes == null || requestedTypes.Any(t => string.Equals(t, SearchEntityTypes.Action, StringComparison.OrdinalIgnoreCase)), + IncludeRecords = false, + Prefix = true + }, await BuildPrincipalAsync(), cancellationToken); + + return Ok(Map(unified, 0, take)); + } + + /// Department admins: flag the department's global index for a full projection + index rebuild on the next sweep (plan R4 Phase 3). + [HttpPost("Rebuild")] + [ProducesResponseType(StatusCodes.Status202Accepted)] + [ProducesResponseType(StatusCodes.Status403Forbidden)] + [ProducesResponseType(StatusCodes.Status404NotFound)] + [Authorize(Policy = ResgridResources.Department_Update)] + public async Task> Rebuild(CancellationToken cancellationToken) + { + if (!await _featureToggles.IsEnabledAsync(FeatureFlagKeys.SearchUnified, DepartmentId)) + return NotFound(); + if (!ClaimsAuthorizationHelper.IsUserDepartmentAdmin()) + return Forbid(); + + var state = await _maintenance.RequestRebuildAsync(DepartmentId, cancellationToken); + Logging.LogInfo($"Search index rebuild requested for department {DepartmentId} by {UserId}."); + + var result = new SearchRebuildResult + { + Data = new SearchRebuildData + { + DepartmentId = DepartmentId, + State = ((SearchIndexBuildState)state.State).ToString(), + RequestedOn = state.RebuildRequestedOn + }, + PageSize = 1, + Status = ResponseHelper.Success + }; + ResponseHelper.PopulateV4ResponseData(result); + return Accepted(result); + } + + /// Department admins: host and index health plus this department's index state. + [HttpGet("Health")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status403Forbidden)] + [Authorize(Policy = ResgridResources.Department_Update)] + public async Task> Health() + { + if (!ClaimsAuthorizationHelper.IsUserDepartmentAdmin()) + return Forbid(); + + var global = await _globalSearch.GetHealthAsync(); + var records = await _recordsSearch.GetHealthAsync(); + var state = await _states.GetAsync(SearchIndexNames.Global, DepartmentId); + + var result = new SearchHealthResult + { + Data = new SearchHealthData + { + Enabled = global.Enabled, + GlobalOnline = global.Online, + GlobalDocumentCount = global.DocumentCount, + RecordsOnline = records.Online, + RecordsDocumentCount = records.DocumentCount, + StoreEnabled = global.StoreEnabled, + LastSyncedRevision = global.LastSyncedRevision, + LastSyncedOnUtc = global.LastSyncedOnUtc, + DepartmentIndexState = state == null ? "None" : ((SearchIndexBuildState)state.State).ToString(), + DepartmentDocumentCount = state?.DocumentCount ?? 0, + DepartmentLastRebuiltOn = state?.LastRebuiltOn + }, + PageSize = 1, + Status = ResponseHelper.Success + }; + ResponseHelper.PopulateV4ResponseData(result); + return Ok(result); + } + + private static List ParseTypes(string types) + { + if (string.IsNullOrWhiteSpace(types)) + return null; + var list = types.Split(new[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries).Select(t => t.Trim()).Where(t => t.Length > 0).Distinct(StringComparer.OrdinalIgnoreCase).ToList(); + return list.Count == 0 ? null : list; + } + + private async Task BuildPrincipalAsync() + { + var user = HttpContext?.User; + DepartmentModuleSettings modules = null; + try { modules = await _departmentSettings.GetDepartmentModuleSettingsAsync(DepartmentId); } + catch (Exception ex) { Logging.LogException(ex); } + + return new SearchPrincipal + { + UserId = UserId, + DepartmentId = DepartmentId, + IsDepartmentAdmin = ClaimsAuthorizationHelper.IsUserDepartmentAdmin(), + HasClaim = (resource, action) => user != null && user.HasClaim(resource, action), + IsModuleEnabled = module => + { + if (modules == null) return true; + switch (module) + { + case SystemActionModules.Messaging: return !modules.MessagingDisabled; + case SystemActionModules.Mapping: return !modules.MappingDisabled; + case SystemActionModules.Shifts: return !modules.ShiftsDisabled; + case SystemActionModules.Logs: return !modules.LogsDisabled; + case SystemActionModules.Reports: return !modules.ReportsDisabled; + case SystemActionModules.Documents: return !modules.DocumentsDisabled; + case SystemActionModules.Calendar: return !modules.CalendarDisabled; + case SystemActionModules.Notes: return !modules.NotesDisabled; + case SystemActionModules.Training: return !modules.TrainingDisabled; + case SystemActionModules.Inventory: return !modules.InventoryDisabled; + case SystemActionModules.Maintenance: return !modules.MaintenanceDisabled; + default: return true; + } + } + }; + } + + private static SearchResult Map(UnifiedSearchResult unified, int skip, int take) + { + var result = new SearchResult + { + Data = new SearchResultData + { + Available = unified.Available, + Degraded = unified.Degraded, + DegradedReason = unified.DegradedReason, + TotalCount = unified.Total, + Truncated = unified.Truncated, + QueryTimeMs = unified.QueryTimeMs, + Results = unified.Hits.Select(h => new SearchHitData + { + EntityType = h.EntityType, + EntityId = h.EntityId, + Title = h.Title, + Summary = h.Summary, + Url = h.Url, + Score = h.Score, + OccurredOn = h.OccurredOn, + Category = h.Category, + Status = h.Status, + Metadata = h.Metadata == null ? new Dictionary() : new Dictionary(h.Metadata) + }).ToList(), + Actions = unified.Actions.Select(a => new SearchActionData + { + Key = a.Key, + Title = a.Title, + Description = a.Description, + Category = a.Category, + Url = a.Url, + Score = a.Score + }).ToList() + }, + Page = take > 0 ? skip / take : 0, + PageSize = unified.Hits.Count, + Status = unified.Available ? ResponseHelper.Success : ResponseHelper.NotFound + }; + ResponseHelper.PopulateV4ResponseData(result); + return result; + } + } +} diff --git a/Web/Resgrid.Web.Services/Models/v4/Health/HealthResult.cs b/Web/Resgrid.Web.Services/Models/v4/Health/HealthResult.cs index 9f3c97b2a..ff1541c67 100644 --- a/Web/Resgrid.Web.Services/Models/v4/Health/HealthResult.cs +++ b/Web/Resgrid.Web.Services/Models/v4/Health/HealthResult.cs @@ -48,5 +48,14 @@ public class HealthResultData /// Can the API services talk to the cache /// public bool CacheOnline { get; set; } + + /// Search host enabled in this process (SearchConfig.Enabled). + public bool SearchEnabled { get; set; } + + /// A local reader is open for the global index (after a pull or a write). + public bool SearchOnline { get; set; } + + /// Documents in this process's copy of the global and records indexes. + public int SearchIndexDocCount { get; set; } } } diff --git a/Web/Resgrid.Web.Services/Models/v4/Search/SearchApiModels.cs b/Web/Resgrid.Web.Services/Models/v4/Search/SearchApiModels.cs new file mode 100644 index 000000000..3b135660d --- /dev/null +++ b/Web/Resgrid.Web.Services/Models/v4/Search/SearchApiModels.cs @@ -0,0 +1,90 @@ +using System; +using System.Collections.Generic; + +namespace Resgrid.Web.Services.Models.v4.Search +{ + /// Unified search response (plan R4 Phase 2). Shape follows Records/Search: availability and degradation are explicit, totals are null when they cannot be proven. + public class SearchResult : StandardApiResponseV4Base + { + public SearchResultData Data { get; set; } = new SearchResultData(); + } + + public class SearchResultData + { + /// False when Search.Unified is off for the department. + public bool Available { get; set; } + + /// True when the index could not serve; actions still return. + public bool Degraded { get; set; } + + public string DegradedReason { get; set; } + + public List Results { get; set; } = new List(); + + public List Actions { get; set; } = new List(); + + /// Authorized total, or null when a hit was dropped by per-entity authorization. + public int? TotalCount { get; set; } + + public bool Truncated { get; set; } + + public int QueryTimeMs { get; set; } + } + + public class SearchHitData + { + public string EntityType { get; set; } + public string EntityId { get; set; } + public string Title { get; set; } + public string Summary { get; set; } + /// Relative web path; clients with native screens switch on EntityType/EntityId instead. + public string Url { get; set; } + public float Score { get; set; } + public DateTime? OccurredOn { get; set; } + public string Category { get; set; } + public string Status { get; set; } + public Dictionary Metadata { get; set; } = new Dictionary(); + } + + public class SearchActionData + { + public string Key { get; set; } + public string Title { get; set; } + public string Description { get; set; } + public string Category { get; set; } + public string Url { get; set; } + public float Score { get; set; } + } + + public class SearchRebuildResult : StandardApiResponseV4Base + { + public SearchRebuildData Data { get; set; } = new SearchRebuildData(); + } + + public class SearchRebuildData + { + public int DepartmentId { get; set; } + public string State { get; set; } + public DateTime? RequestedOn { get; set; } + } + + public class SearchHealthResult : StandardApiResponseV4Base + { + public SearchHealthData Data { get; set; } = new SearchHealthData(); + } + + public class SearchHealthData + { + public bool Enabled { get; set; } + public bool GlobalOnline { get; set; } + public int GlobalDocumentCount { get; set; } + public bool RecordsOnline { get; set; } + public int RecordsDocumentCount { get; set; } + public bool StoreEnabled { get; set; } + public string LastSyncedRevision { get; set; } + public DateTime? LastSyncedOnUtc { get; set; } + public string DepartmentIndexState { get; set; } + public int DepartmentDocumentCount { get; set; } + public DateTime? DepartmentLastRebuiltOn { get; set; } + } +} diff --git a/Web/Resgrid.Web.Services/Resgrid.Web.Services.xml b/Web/Resgrid.Web.Services/Resgrid.Web.Services.xml index 65a8440c3..d582dc4be 100644 --- a/Web/Resgrid.Web.Services/Resgrid.Web.Services.xml +++ b/Web/Resgrid.Web.Services/Resgrid.Web.Services.xml @@ -3496,6 +3496,29 @@ Minimal SCIM 2.0 User resource for inbound provisioning requests. + + + Unified Search (plan R4 Phase 2/3): one endpoint over the global index (calls, units, personnel, contacts, + messages, documents, notes), the RMS records index, and the system-functionality catalog. The department and + the caller's identity come from the token, never from the request. Each hit is re-checked against the entity's + own authorization rule; totals are null when any hit was dropped. Typeahead is the same query in prefix mode. + + + + + Full search. is a comma-separated list of entity types (Call, Unit, Personnel, + Contact, Message, Document, Note, Record, Action); omit for all. + + + + Typeahead: prefix search on titles and identifiers, no records federation, small result list. + + + Department admins: flag the department's global index for a full projection + index rebuild on the next sweep (plan R4 Phase 3). + + + Department admins: host and index health plus this department's index state. + Call Priorities, for example Low, Medium, High. Call Priorities can be system provided ones or custom for a department @@ -11583,6 +11606,15 @@ Can the API services talk to the cache + + Search host enabled in this process (SearchConfig.Enabled). + + + A local reader is open for the global index (after a pull or a write). + + + Documents in this process's copy of the global and records indexes. + Input to establish command on a call. @@ -13621,6 +13653,21 @@ Response Data + + Unified search response (plan R4 Phase 2). Shape follows Records/Search: availability and degradation are explicit, totals are null when they cannot be proven. + + + False when Search.Unified is off for the department. + + + True when the index could not serve; actions still return. + + + Authorized total, or null when a hit was dropped by per-entity authorization. + + + Relative web path; clients with native screens switch on EntityType/EntityId instead. + Response Data diff --git a/Web/Resgrid.Web/Areas/User/Controllers/ChecklistsSchedulingController.cs b/Web/Resgrid.Web/Areas/User/Controllers/ChecklistsSchedulingController.cs index 707c103e3..04786080d 100644 --- a/Web/Resgrid.Web/Areas/User/Controllers/ChecklistsSchedulingController.cs +++ b/Web/Resgrid.Web/Areas/User/Controllers/ChecklistsSchedulingController.cs @@ -13,7 +13,11 @@ namespace Resgrid.Web.Areas.User.Controllers public partial class ChecklistsController { [HttpGet, Authorize(Policy = ResgridResources.Checklist_Update)] - public async Task Schedules(string id, int page = 0) => View("Schedules", new ChecklistSchedulesView { DefinitionId = id, Schedules = await _checklists.SchedulesAsync(Actor, id, page), Page = page, CanEdit = await ChecklistsEnabledAsync() }); + public async Task Schedules(string id, int page = 0) + { + var rows = await _checklists.SchedulesAsync(Actor, id, page, includeNext: true); + return View("Schedules", new ChecklistSchedulesView { DefinitionId = id, Schedules = rows.Take(50).ToList(), Page = page, HasMore = rows.Count > 50, CanEdit = await ChecklistsEnabledAsync() }); + } [HttpGet, Authorize(Policy = ResgridResources.Checklist_Update)] public async Task EditSchedule(string id = null, string definitionId = null) { diff --git a/Web/Resgrid.Web/Areas/User/Controllers/SearchController.cs b/Web/Resgrid.Web/Areas/User/Controllers/SearchController.cs index 6818eb07a..f684f9814 100644 --- a/Web/Resgrid.Web/Areas/User/Controllers/SearchController.cs +++ b/Web/Resgrid.Web/Areas/User/Controllers/SearchController.cs @@ -1,182 +1,152 @@ -using Microsoft.AspNetCore.Mvc; -using Resgrid.Model.Services; +using System.Collections.Generic; +using System.Linq; +using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Newtonsoft.Json; +using Resgrid.Framework; +using Resgrid.Model.Search; +using Resgrid.Model.Services; using Resgrid.Providers.Claims; -using System.Collections.Generic; +using Resgrid.Web.Helpers; using Resgrid.WebCore.Areas.User.Models.Search; -using Newtonsoft.Json; -using System.Linq; namespace Resgrid.Web.Areas.User.Controllers { + /// + /// The command palette behind the top search box (Unified Search plan R3 "Action" family + R4 Phase 2). System + /// functionality always comes from the catalog, filtered by the caller's claims, module switches and feature flags; + /// entity hits come from the unified endpoint when Search.Unified is on for the department, each re-checked against + /// the entity's own authorization rule before it is returned. + /// [Area("User")] public class SearchController : SecureBaseController { - private readonly Model.Services.IAuthorizationService _authorizationService; - private readonly IDepartmentSettingsService _departmentSettingsService; + private readonly IUnifiedSearchService _unifiedSearch; + private readonly ISystemActionsService _systemActions; - public SearchController(Model.Services.IAuthorizationService authorizationService, IDepartmentSettingsService departmentSettingsService) + public SearchController(IUnifiedSearchService unifiedSearch, ISystemActionsService systemActions) { - _authorizationService = authorizationService; - _departmentSettingsService = departmentSettingsService; + _unifiedSearch = unifiedSearch; + _systemActions = systemActions; } [HttpGet] [Authorize(Policy = ResgridResources.Department_View)] - public async Task GetSearchResults(string query) + public async Task GetSearchResults(string query, CancellationToken cancellationToken) { - List allActions = new List(); - List results = null; - - allActions.Add(new SearchResultJson - { - Label = "/Calls", - Summary = "View Calls and Dispatches", - Url = Config.SystemBehaviorConfig.ResgridBaseUrl + "/User/Dispatch/Dashboard" - }); + var principal = BuildPrincipal(); + var text = (query ?? string.Empty).Trim(); + var items = new List(); - allActions.Add(new SearchResultJson + UnifiedSearchResult unified = null; + try { - Label = "/Personnel", - Summary = "View People (Personnel)", - Url = Config.SystemBehaviorConfig.ResgridBaseUrl + "/User/Personnel" - }); - - allActions.Add(new SearchResultJson - { - Label = "/Units", - Summary = "View Units (Teams or Apparatuses)", - Url = Config.SystemBehaviorConfig.ResgridBaseUrl + "/User/Units" - }); - - allActions.Add(new SearchResultJson - { - Label = "/Mapping", - Summary = "Large Map View which allows filtering and layers", - Url = Config.SystemBehaviorConfig.ResgridBaseUrl + "/User/Mapping" - }); - - allActions.Add(new SearchResultJson - { - Label = "/Shifts", - Summary = "Shifts (Signup, Recurring, Workshift)", - Url = Config.SystemBehaviorConfig.ResgridBaseUrl + "/User/Shifts" - }); - - allActions.Add(new SearchResultJson - { - Label = "/Logs", - Summary = "Logs for activity in the department (Run, Training, Work, Meetings, Callbacks)", - Url = Config.SystemBehaviorConfig.ResgridBaseUrl + "/User/Logs" - }); - - allActions.Add(new SearchResultJson - { - Label = "/NewLog", - Summary = "Create a new Log (i.e. Run Report, Training Log)", - Url = Config.SystemBehaviorConfig.ResgridBaseUrl + "/User/Logs/NewLog" - }); - - allActions.Add(new SearchResultJson - { - Label = "/Reports", - Summary = "Generate reports based on data in the Department", - Url = Config.SystemBehaviorConfig.ResgridBaseUrl + "/User/Reports" - }); - - allActions.Add(new SearchResultJson - { - Label = "/Calendar", - Summary = "Calendar where you can schedule and signup to events, trainings", - Url = Config.SystemBehaviorConfig.ResgridBaseUrl + "/User/Calendar" - }); - - allActions.Add(new SearchResultJson - { - Label = "/Notes", - Summary = "Department notes which are small bits of information", - Url = Config.SystemBehaviorConfig.ResgridBaseUrl + "/User/Notes" - }); - - allActions.Add(new SearchResultJson - { - Label = "/Documents", - Summary = "Upload and Share documents (like pdfs, word docs, excel)", - Url = Config.SystemBehaviorConfig.ResgridBaseUrl + "/User/Documents" - }); - - allActions.Add(new SearchResultJson + unified = await _unifiedSearch.SearchAsync(new UnifiedSearchRequest + { + Text = text, + Take = 10, + Prefix = true, + IncludeActions = true, + IncludeRecords = false + }, principal, cancellationToken); + } + catch (System.Exception ex) { - Label = "/Trainings", - Summary = "Trainings, Study Guides Procedures for people to review", - Url = Config.SystemBehaviorConfig.ResgridBaseUrl + "/User/Trainings" - }); + Logging.LogException(ex, "Unified search failed for the command palette; returning system actions only."); + } - allActions.Add(new SearchResultJson + List actions; + if (unified != null && unified.Available) { - Label = "/Inventory", - Summary = "Inventory for your Stations and Units", - Url = Config.SystemBehaviorConfig.ResgridBaseUrl + "/User/Inventory" - }); - - allActions.Add(new SearchResultJson + actions = unified.Actions; + } + else { - Label = "/Inbox", - Summary = "View your Messages Inbox", - Url = Config.SystemBehaviorConfig.ResgridBaseUrl + "/User/Messages/Inbox" - }); + // Flag off or search unavailable: the palette still finds system functionality. + actions = text.Length == 0 + ? await _systemActions.ListAsync(principal, cancellationToken) + : await _systemActions.SearchAsync(text, principal, 8, cancellationToken); + } - allActions.Add(new SearchResultJson + items.AddRange((actions ?? new List()).Select(a => new SearchResultJson { - Label = "/Profile", - Summary = "View and Edit your own User Profile", - Url = Config.SystemBehaviorConfig.ResgridBaseUrl + "/User/Home/EditUserProfile?UserId=" + UserId - }); + Label = a.Title, + Summary = a.Description, + Url = a.Url, + Group = "Actions", + Type = a.Category + })); - if (await _authorizationService.CanUserCreateCallAsync(UserId, DepartmentId)) + if (unified != null && unified.Available) { - allActions.Add(new SearchResultJson + items.AddRange(unified.Hits.Select(h => new SearchResultJson { - Label = "/NewCall", - Summary = "Create and Dispatch a new Call", - Url = Config.SystemBehaviorConfig.ResgridBaseUrl + "/User/Dispatch/NewCall" - }); + Label = h.Title, + Summary = string.IsNullOrWhiteSpace(h.Summary) ? Badge(h) : h.Summary, + Url = string.IsNullOrWhiteSpace(h.Url) ? null : (h.Url.StartsWith("http") ? h.Url : Config.SystemBehaviorConfig.ResgridBaseUrl + h.Url), + Group = Plural(h.EntityType), + Type = h.EntityType + })); } - allActions.Add(new SearchResultJson - { - Label = "/ArchivedCalls", - Summary = "View Archived Calls (old Calls)", - Url = Config.SystemBehaviorConfig.ResgridBaseUrl + "/User/Dispatch/ArchivedCalls" - }); + return Content(JsonConvert.SerializeObject(items), "application/json"); + } - if (await _authorizationService.CanUserAddNewUserAsync(DepartmentId, UserId)) - { - allActions.Add(new SearchResultJson + private SearchPrincipal BuildPrincipal() + { + var user = HttpContext?.User; + return new SearchPrincipal + { + UserId = UserId, + DepartmentId = DepartmentId, + IsDepartmentAdmin = ClaimsAuthorizationHelper.IsUserDepartmentAdmin(), + HasClaim = (resource, action) => user != null && user.HasClaim(resource, action), + IsModuleEnabled = module => { - Label = "/AddPerson", - Summary = "Manually Create a User Account", - Url = Config.SystemBehaviorConfig.ResgridBaseUrl + "/User/Personnel/AddPerson" - }); + switch (module) + { + case SystemActionModules.Messaging: return SettingsHelper.IsMessagingEnabled(); + case SystemActionModules.Mapping: return SettingsHelper.IsMappingEnabled(); + case SystemActionModules.Shifts: return SettingsHelper.IsShiftsEnabled(); + case SystemActionModules.Logs: return SettingsHelper.IsLogsEnabled(); + case SystemActionModules.Reports: return SettingsHelper.IsReportsEnabled(); + case SystemActionModules.Documents: return SettingsHelper.IsDocumentsEnabled(); + case SystemActionModules.Calendar: return SettingsHelper.IsCalendarEnabled(); + case SystemActionModules.Notes: return SettingsHelper.IsNotesEnabled(); + case SystemActionModules.Training: return SettingsHelper.IsTrainingEnabled(); + case SystemActionModules.Inventory: return SettingsHelper.IsInventoryEnabled(); + case SystemActionModules.Maintenance: return SettingsHelper.IsMaintenanceEnabled(); + default: return true; + } + } + }; + } - allActions.Add(new SearchResultJson - { - Label = "/ManageInvites", - Summary = "Send Email Invites for users to create their own Accounts", - Url = Config.SystemBehaviorConfig.ResgridBaseUrl + "/User/Department/Invites" - }); - } + private static string Badge(UnifiedSearchHit hit) + { + var parts = new List(); + if (!string.IsNullOrWhiteSpace(hit.Category)) parts.Add(hit.Category); + if (!string.IsNullOrWhiteSpace(hit.Status)) parts.Add(hit.Status); + if (hit.OccurredOn.HasValue) parts.Add(hit.OccurredOn.Value.ToString("yyyy-MM-dd")); + return string.Join(" · ", parts); + } - if (string.IsNullOrWhiteSpace(query)) - results = allActions; - else - { - var querySet = query.Trim().ToLower(); - results = allActions.Where(x => x.Label.ToLower().Contains(querySet) || x.Summary.ToLower().Contains(querySet)).ToList(); + private static string Plural(string entityType) + { + switch (entityType) + { + case SearchEntityTypes.Call: return "Calls"; + case SearchEntityTypes.Unit: return "Units"; + case SearchEntityTypes.Personnel: return "Personnel"; + case SearchEntityTypes.Contact: return "Contacts"; + case SearchEntityTypes.Message: return "Messages"; + case SearchEntityTypes.Document: return "Documents"; + case SearchEntityTypes.Note: return "Notes"; + case SearchEntityTypes.Record: return "Records"; + default: return entityType; } - - return Content(JsonConvert.SerializeObject(results), "application/json");// Json(results); } } } diff --git a/Web/Resgrid.Web/Areas/User/Models/Checklists/ChecklistScheduleViews.cs b/Web/Resgrid.Web/Areas/User/Models/Checklists/ChecklistScheduleViews.cs index 730650f1a..2330fc1aa 100644 --- a/Web/Resgrid.Web/Areas/User/Models/Checklists/ChecklistScheduleViews.cs +++ b/Web/Resgrid.Web/Areas/User/Models/Checklists/ChecklistScheduleViews.cs @@ -3,7 +3,7 @@ namespace Resgrid.Web.Areas.User.Models.Checklists { - public class ChecklistSchedulesView { public string DefinitionId { get; set; } public List Schedules { get; set; } public int Page { get; set; } public bool CanEdit { get; set; } } + public class ChecklistSchedulesView { public string DefinitionId { get; set; } public List Schedules { get; set; } public int Page { get; set; } public bool HasMore { get; set; } public bool CanEdit { get; set; } } public class ChecklistScheduleEditView { public ChecklistScheduleInput Input { get; set; } diff --git a/Web/Resgrid.Web/Areas/User/Models/Search/SearchResultJson.cs b/Web/Resgrid.Web/Areas/User/Models/Search/SearchResultJson.cs index e99d2a944..4327136fe 100644 --- a/Web/Resgrid.Web/Areas/User/Models/Search/SearchResultJson.cs +++ b/Web/Resgrid.Web/Areas/User/Models/Search/SearchResultJson.cs @@ -1,16 +1,25 @@ -using Newtonsoft.Json; +using Newtonsoft.Json; namespace Resgrid.WebCore.Areas.User.Models.Search { + /// One command-palette row. Label/summary/url are the contract the layout script already renders; group and type drive the section headers. public class SearchResultJson { - [JsonProperty(PropertyName = "label")] + [JsonProperty("label")] public string Label { get; set; } - [JsonProperty(PropertyName = "summary")] + [JsonProperty("summary")] public string Summary { get; set; } - [JsonProperty(PropertyName = "url")] + [JsonProperty("url")] public string Url { get; set; } + + /// "Actions" for system functionality, otherwise the plural entity family ("Calls", "Units", ...). + [JsonProperty("group")] + public string Group { get; set; } + + /// Action category or SearchEntityTypes value. + [JsonProperty("type")] + public string Type { get; set; } } } diff --git a/Web/Resgrid.Web/Areas/User/Views/Checklists/Schedules.cshtml b/Web/Resgrid.Web/Areas/User/Views/Checklists/Schedules.cshtml index 448386fab..55578bc68 100644 --- a/Web/Resgrid.Web/Areas/User/Views/Checklists/Schedules.cshtml +++ b/Web/Resgrid.Web/Areas/User/Views/Checklists/Schedules.cshtml @@ -63,14 +63,20 @@ } -
- @if (Model.Page > 0) - { - @localizer["Previous"] - } - @localizer["Next"] - @(Model.Page + 1) -
+ @if (Model.Page > 0 || Model.HasMore) + { +
+ @if (Model.Page > 0) + { + @localizer["Previous"] + } + @if (Model.HasMore) + { + @localizer["Next"] + } + @(Model.Page + 1) +
+ } diff --git a/Web/Resgrid.Web/Areas/User/Views/Checklists/_Tabs.cshtml b/Web/Resgrid.Web/Areas/User/Views/Checklists/_Tabs.cshtml index 3457a2a86..3e8808acf 100644 --- a/Web/Resgrid.Web/Areas/User/Views/Checklists/_Tabs.cshtml +++ b/Web/Resgrid.Web/Areas/User/Views/Checklists/_Tabs.cshtml @@ -22,7 +22,7 @@ tabs.Add(("Reminders", "fa-bell-o", localizer["ReminderSettings"].Value)); } } -
    +
      @foreach (var tab in tabs) {
    • diff --git a/Web/Resgrid.Web/Areas/User/Views/RecordEvidence/Select.cshtml b/Web/Resgrid.Web/Areas/User/Views/RecordEvidence/Select.cshtml index b9a53c30b..92b04758d 100644 --- a/Web/Resgrid.Web/Areas/User/Views/RecordEvidence/Select.cshtml +++ b/Web/Resgrid.Web/Areas/User/Views/RecordEvidence/Select.cshtml @@ -16,6 +16,7 @@ ViewData["ParentText"] = localizer["Evidence"].Value; ViewData["ParentController"] = "RecordEvidence"; ViewData["ParentAction"] = "Index"; + ViewData["ParentRouteValues"] = new Dictionary { ["recordId"] = context.RecordId, ["recordKind"] = context.RecordKind.ToString() }; string SourceGuidance() => selection.SourceKind switch { diff --git a/Web/Resgrid.Web/Areas/User/Views/Records/Edit.cshtml b/Web/Resgrid.Web/Areas/User/Views/Records/Edit.cshtml index 0d0f00b80..cfa23e339 100644 --- a/Web/Resgrid.Web/Areas/User/Views/Records/Edit.cshtml +++ b/Web/Resgrid.Web/Areas/User/Views/Records/Edit.cshtml @@ -228,11 +228,11 @@
      @localizer["Participants"]
      - +
      @localizer["Participants"]@localizer["Unit"]Role
      @for (var i = 0; i < Model.ParticipantRows.Count; i++) { var participant = Model.ParticipantRows[i]; var name = Model.Personnel.FirstOrDefault(p => p.Value == participant.UserId)?.Text ?? participant.UserId; - - + @@ -315,7 +320,7 @@ @localizer["Status" + (WorkOrderStatus)a.NewStatus.Value] } - +
      @localizer["Participants"]@localizer["Unit"]@localizer["Role"]
      @l.UserId@UserName(l.UserId) @l.WorkDate.ToString("d") @l.Content.Hours @l.Content.RatePerHour @c.Currency@a.UserId@UserName(a.UserId) @if (!string.IsNullOrEmpty(a.Note)) { @@ -323,7 +328,7 @@ } @if (a.AssignedToUserId != null || a.AssignedToRoleId.HasValue) { - @localizer["AssignedTo"]: @(a.AssignedToUserId ?? a.AssignedToRoleId.ToString()) + @localizer["AssignedTo"]: @(a.AssignedToUserId != null ? UserName(a.AssignedToUserId) : RoleName(a.AssignedToRoleId)) } @if (a.RevisedDueOn.HasValue) { diff --git a/Web/Resgrid.Web/Areas/User/Views/WorkOrders/EditRecurrence.cshtml b/Web/Resgrid.Web/Areas/User/Views/WorkOrders/EditRecurrence.cshtml index 1cfebbc2f..80b93df74 100644 --- a/Web/Resgrid.Web/Areas/User/Views/WorkOrders/EditRecurrence.cshtml +++ b/Web/Resgrid.Web/Areas/User/Views/WorkOrders/EditRecurrence.cshtml @@ -70,7 +70,7 @@
      - +
      diff --git a/Web/Resgrid.Web/Areas/User/Views/WorkOrders/Policy.cshtml b/Web/Resgrid.Web/Areas/User/Views/WorkOrders/Policy.cshtml index 0a88d985e..e0d382242 100644 --- a/Web/Resgrid.Web/Areas/User/Views/WorkOrders/Policy.cshtml +++ b/Web/Resgrid.Web/Areas/User/Views/WorkOrders/Policy.cshtml @@ -7,7 +7,7 @@ ViewData["Tab"] = "Policy"; ViewData["Title"] = localizer["OperationsPolicy"].Value; ViewData["Subtitle"] = localizer["PolicyHelp"].Value; - ViewData["CanWrite"] = true; + ViewData["CanWrite"] = Model.CanWrite; } @await Html.PartialAsync("_Shell") diff --git a/Web/Resgrid.Web/Areas/User/Views/WorkOrders/_OrderFields.cshtml b/Web/Resgrid.Web/Areas/User/Views/WorkOrders/_OrderFields.cshtml index 4e7e87fb3..3529b0bd9 100644 --- a/Web/Resgrid.Web/Areas/User/Views/WorkOrders/_OrderFields.cshtml +++ b/Web/Resgrid.Web/Areas/User/Views/WorkOrders/_OrderFields.cshtml @@ -48,7 +48,19 @@ {
      - + @if (Model.Revision > 0) + { + @* A saved order changes its due date through a deferral, and the service rejects an + edit whose DueOn differs from the stored value by even a tick. datetime-local + cannot carry that precision (the browser blanks anything past milliseconds), so + the exact value posts from a hidden field and the visible one is display only. *@ + + + } + else + { + + } @localizer["HelpFieldDueOn"]
      } diff --git a/Web/Resgrid.Web/Areas/User/Views/WorkOrders/_Tabs.cshtml b/Web/Resgrid.Web/Areas/User/Views/WorkOrders/_Tabs.cshtml index 921015f2d..fc95d0e4d 100644 --- a/Web/Resgrid.Web/Areas/User/Views/WorkOrders/_Tabs.cshtml +++ b/Web/Resgrid.Web/Areas/User/Views/WorkOrders/_Tabs.cshtml @@ -21,7 +21,7 @@ tabs.Add(("Policy", "fa-sliders", localizer["OperationsPolicy"].Value)); } } -
        +
          @foreach (var tab in tabs) {
        • diff --git a/Web/Resgrid.Web/wwwroot/js/app/common/workspace/resgrid.common.workspace.js b/Web/Resgrid.Web/wwwroot/js/app/common/workspace/resgrid.common.workspace.js index 6ce54b25c..75a25c818 100644 --- a/Web/Resgrid.Web/wwwroot/js/app/common/workspace/resgrid.common.workspace.js +++ b/Web/Resgrid.Web/wwwroot/js/app/common/workspace/resgrid.common.workspace.js @@ -223,10 +223,16 @@ var host = this; host.querySelectorAll('form.rgw-wizard').forEach(function (form) { delete form.dataset.wizardStarted; + steps(form).forEach(clearErrors); show(form, 0); }); + // Hide through the same class notify() shows with; an inline display would + // outrank .rgw-visible and keep every later message in this modal invisible. var message = host.querySelector('.rgw-modal-message'); - if (message) { message.style.display = 'none'; } + if (message) { + message.classList.remove('rgw-visible'); + message.hidden = true; + } var focusable = host.querySelector('.rgw-step:not([hidden]) input:not([type=hidden]), .rgw-step:not([hidden]) select, .modal-body input:not([type=hidden]), .modal-body select'); if (focusable) { focusable.focus(); } }); diff --git a/Web/Resgrid.Web/wwwroot/js/app/internal/resgrid.user.js b/Web/Resgrid.Web/wwwroot/js/app/internal/resgrid.user.js index 23cd7d883..553d666a4 100644 --- a/Web/Resgrid.Web/wwwroot/js/app/internal/resgrid.user.js +++ b/Web/Resgrid.Web/wwwroot/js/app/internal/resgrid.user.js @@ -45,29 +45,40 @@ var resgrid; autocomplete({ container: '#autocomplete', placeholder: 'Search Resgrid', - getSources() { - return [{ - getItems({ query }) { - const queryClean = query.replace(/ |-/g, '') - if (!!queryClean) { - return fetch(resgrid.absoluteBaseUrl + `/User/Search/GetSearchResults?query=${queryClean}`) - .then(res => res.json()) - .catch(console.log()) - } - }, - onSelect: function (event) { - window.location.assign(event.item.url); - }, - templates: { - item({ item, html }) { - return html` - ` - }, - }, - }]; + openOnFocus: true, + getSources({ query }) { + const q = (query || '').trim(); + // One round trip serves both sections: system functionality ("Actions") and entity hits. + return fetch(resgrid.absoluteBaseUrl + '/User/Search/GetSearchResults?query=' + encodeURIComponent(q)) + .then(res => res.json()) + .then(items => { + const groups = []; + (items || []).forEach(item => { + const name = item.group || 'Results'; + let g = groups.find(x => x.name === name); + if (!g) { g = { name: name, items: [] }; groups.push(g); } + g.items.push(item); + }); + return groups.map(g => ({ + sourceId: 'search-' + g.name.toLowerCase(), + getItems() { return g.items; }, + getItemUrl({ item }) { return item.url; }, + onSelect(event) { if (event.item && event.item.url) { window.location.assign(event.item.url); } }, + templates: { + header({ html }) { + return html`
          ${g.name === 'Actions' ? 'Go to' : g.name}
          `; + }, + item({ item, html }) { + return html` + `; + }, + }, + })); + }) + .catch(() => []); }, }); } diff --git a/Workers/Resgrid.Workers.Console/Commands/SearchIndexCommand.cs b/Workers/Resgrid.Workers.Console/Commands/SearchIndexCommand.cs new file mode 100644 index 000000000..4dc7bd3c8 --- /dev/null +++ b/Workers/Resgrid.Workers.Console/Commands/SearchIndexCommand.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; +using Quidjibo.Commands; + +namespace Resgrid.Workers.Console.Commands +{ + /// Worker ID 70 (registry §4F, Unified Search): global search index maintenance sweep (plan R4 Phase 2). Same process as 44, the sole IndexWriter owner. + public sealed class SearchIndexCommand : IQuidjiboCommand + { + public SearchIndexCommand(int id) + { + Id = id; + } + + public int Id { get; } + public Guid? CorrelationId { get; set; } + public Dictionary Metadata { get; set; } + } +} diff --git a/Workers/Resgrid.Workers.Console/Program.cs b/Workers/Resgrid.Workers.Console/Program.cs index 15f6ea8d3..fd5a76783 100644 --- a/Workers/Resgrid.Workers.Console/Program.cs +++ b/Workers/Resgrid.Workers.Console/Program.cs @@ -506,6 +506,14 @@ await Client.ScheduleAsync("Records Search Index", Cron.MinuteIntervals(1), stoppingToken); + // Worker ID 70 (Identifier Allocation Registry section 4F, Unified Search): global search index maintenance. Same + // single-writer process as 44; no-op while SearchConfig.Enabled is off; departments enter lazily via their state row. + _logger.Log(LogLevel.Information, "Scheduling Search Index"); + await Client.ScheduleAsync("Search Index", + new Commands.SearchIndexCommand(70), + Cron.MinuteIntervals(1), + stoppingToken); + // Worker ID 41 (Identifier Allocation Registry section 3.3, RMS-2): NERIS submission sweep. Claims due // submissions with a lease, talks to the destination outside any transaction, no-op while NerisConfig.Enabled is off. _logger.Log(LogLevel.Information, "Scheduling Records Submission"); diff --git a/Workers/Resgrid.Workers.Console/Tasks/SearchIndexTask.cs b/Workers/Resgrid.Workers.Console/Tasks/SearchIndexTask.cs new file mode 100644 index 000000000..1b3331b6b --- /dev/null +++ b/Workers/Resgrid.Workers.Console/Tasks/SearchIndexTask.cs @@ -0,0 +1,45 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Quidjibo.Handlers; +using Quidjibo.Misc; +using Resgrid.Workers.Console.Commands; +using Resgrid.Workers.Framework.Logic; + +namespace Resgrid.Workers.Console.Tasks +{ + public sealed class SearchIndexTask : IQuidjiboHandler + { + private readonly ILogger _logger; + + public SearchIndexTask(ILogger logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string Name => "Search Index"; + public int Priority => 1; + + public async Task ProcessAsync(SearchIndexCommand command, IQuidjiboProgress progress, CancellationToken cancellationToken) + { + progress?.Report(1, $"Starting the {Name} Task"); + + try + { + var logic = new SearchIndexLogic(); + var result = await logic.Process(cancellationToken); + + if (!result.Item1) + throw new InvalidOperationException(result.Item2); + + _logger.LogInformation("SearchIndex::{Summary}", result.Item2); + progress?.Report(100, $"Finishing the {Name} Task"); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + } + } +} diff --git a/Workers/Resgrid.Workers.Framework/Logic/SearchIndexLogic.cs b/Workers/Resgrid.Workers.Framework/Logic/SearchIndexLogic.cs new file mode 100644 index 000000000..0b1e03f60 --- /dev/null +++ b/Workers/Resgrid.Workers.Framework/Logic/SearchIndexLogic.cs @@ -0,0 +1,41 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Autofac; +using Resgrid.Framework; +using Resgrid.Model.Services; + +namespace Resgrid.Workers.Framework.Logic +{ + /// + /// Worker command 70: the global search index maintenance sweep (Unified Search plan R4 Phase 2). The worker process + /// is the only holder of the index writer; this logic just drives ISearchIndexMaintenanceService, which no-ops + /// while SearchConfig.Enabled is off. + /// + public sealed class SearchIndexLogic + { + public async Task> Process(CancellationToken cancellationToken) + { + try + { + using var scope = Bootstrapper.GetKernel().BeginLifetimeScope(); + var maintenance = scope.Resolve(); + var result = await maintenance.SweepAsync(cancellationToken); + + if (result.Errors > 0) + Logging.LogError($"Global search index sweep finished with {result.Errors} error(s): {result.Message}"); + + return new Tuple(result.Errors == 0, result.Message); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + Logging.LogException(ex); + return new Tuple(false, ex.ToString()); + } + } + } +}