using System; using System.Collections.Generic; using System.Data; using System.Globalization; using System.IO; using System.Linq; using System.Text; using ClosedXML.Excel; using Ease.Core.DataAccess; using Ease.Core.Model; using Ease.Core.Utility; using HRM.BO; using Microsoft.Extensions.Caching.Memory; namespace HRM.DA { /// /// The one service behind every virtual-scrolling grid (api/virtualGrid). It looks the grid up in /// and serves it in one of two ways: /// /// - List grid (): the grid's existing service returns a /// List of business objects; that list is kept in server memory for a few minutes and every scroll, /// sort, search and Excel export is answered from it. /// - SQL grid (): SQL Server counts, sorts, searches and returns /// one OFFSET/FETCH window per request (VirtualGridDA). /// /// Shared work for both: access check, column validation, paging limits, sort whitelist, the Excel file. /// public class VirtualGridService : ServiceTemplate, IVirtualGridService { private const int ExportCommandTimeoutSeconds = 600; private const int MaxSearchLength = 200; private const string ExcelContentType = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; // List grids: loaded lists, per user + grid + filters. Size = rows, so at most ~1,000,000 rows are // held in total; the least recently used lists are dropped first. private static readonly MemoryCache ListCache = new MemoryCache(new MemoryCacheOptions { SizeLimit = 1000000 }); private static readonly TimeSpan ListCacheSliding = TimeSpan.FromMinutes(10); private static readonly TimeSpan ListCacheAbsolute = TimeSpan.FromMinutes(30); #region Public API public VirtualGridLayout GetLayout(string gridKey, VirtualGridContext context) { VirtualGridDefinition grid = Open(gridKey, context); List columns = GetValidatedColumns(grid, context); VirtualGridColumn defaultSort = DefaultSortColumn(grid, columns); var layout = new VirtualGridLayout { GridKey = grid.Key, Title = grid.Title, DefaultSortField = defaultSort?.Field, DefaultSortDir = NormalizeDir(grid.DefaultSortDir), MaxPageSize = grid.MaxPageSize, }; layout.Columns.AddRange(columns.Select(c => new VirtualGridColumnInfo { Field = c.Field, Title = c.Title, Width = c.Width, Type = c.Type, Sortable = grid.CanSort(c), })); return layout; } public VirtualGridPage GetPage(VirtualGridRequest request, VirtualGridContext context) { if (request == null) throw new ArgumentException("Invalid request."); VirtualGridDefinition grid = Open(request.GridKey, context); List columns = GetValidatedColumns(grid, context); int skip = Math.Max(0, request.Skip); int take = Math.Min(Math.Max(1, request.Take), grid.MaxPageSize); var listSource = grid as IVirtualGridListSource; if (listSource != null) { CachedList list = GetList(grid, listSource, columns, request, context); List matches = Search(list, request.Search); List ordered = Sort(grid, columns, list, matches, request); return new VirtualGridPage { Total = ordered.Count, TotalWithoutSearch = list.Rows.Count, Data = ordered.Skip(skip).Take(take).Select(i => list.Rows[i]).ToList(), }; } var sqlGrid = (VirtualGridSqlDefinition)grid; VirtualGridQuery query = BuildQuery(sqlGrid, columns, request, context); var page = new VirtualGridPage(); TransactionContext tc = null; try { tc = TransactionContext.Begin(); page.Total = VirtualGridDA.Count(tc, query, true); page.TotalWithoutSearch = query.SearchPattern == null ? page.Total : VirtualGridDA.Count(tc, query, false); if (page.Total > skip) page.Data = ReadRows(VirtualGridDA.GetRows(tc, query, skip, take)); tc.End(); } catch (Exception e) { #region Handle Exception if (tc != null) tc.HandleError(); ExceptionLog.Write(e); throw new ServiceException(e.Message, e); #endregion } // After the transaction: definitions may call other services that open their own. FinishRows(sqlGrid, page.Data, context); return page; } public VirtualGridFile ExportExcel(VirtualGridExportRequest request, VirtualGridContext context) { if (request == null) throw new ArgumentException("Invalid request."); VirtualGridDefinition grid = Open(request.GridKey, context); List columns = GetValidatedColumns(grid, context); List exportColumns = ResolveExportColumns(columns, request.Fields); List> rows; string filterDescription; var listSource = grid as IVirtualGridListSource; if (listSource != null) { CachedList list = GetList(grid, listSource, columns, request, context); List ordered = Sort(grid, columns, list, Search(list, request.Search), request); CheckExportLimit(grid, ordered.Count); rows = ordered.Select(i => list.Rows[i]).ToList(); filterDescription = listSource.DescribeFilters(new VirtualGridFilters(request.Filters)); } else { var sqlGrid = (VirtualGridSqlDefinition)grid; VirtualGridQuery query = BuildQuery(sqlGrid, columns, request, context); TransactionContext tc = null; try { tc = TransactionContext.Begin(); tc.CommandTimeOut = ExportCommandTimeoutSeconds; CheckExportLimit(grid, VirtualGridDA.Count(tc, query, true)); rows = ReadRows(VirtualGridDA.GetRows(tc, query, 0, null)); tc.End(); } catch (ArgumentException) { if (tc != null) tc.HandleError(); throw; } catch (Exception e) { #region Handle Exception if (tc != null) tc.HandleError(); ExceptionLog.Write(e); throw new ServiceException(e.Message, e); #endregion } FinishRows(sqlGrid, rows, context); filterDescription = query.Source.FilterDescription; } return new VirtualGridFile { FileName = grid.ExcelFileName, ContentType = ExcelContentType, Content = BuildWorkbook(grid, exportColumns, rows, filterDescription, request.Search), }; } #endregion #region Shared helpers /// Finds the grid and checks the user may use it. Unknown key → ArgumentException. private static VirtualGridDefinition Open(string gridKey, VirtualGridContext context) { if (context == null) throw new UnauthorizedAccessException("Not signed in."); VirtualGridDefinition grid = VirtualGridRegistry.Create(gridKey); if (!(grid is IVirtualGridListSource) && !(grid is VirtualGridSqlDefinition)) throw new InvalidOperationException("Virtual grid '" + grid.Key + "' must derive from VirtualGridListDefinition or VirtualGridSqlDefinition."); if (context.UserType == EnumUserType.Employee && !grid.AllowSelfServiceUsers) throw new UnauthorizedAccessException("You do not have access to " + grid.Title + "."); return grid; } /// The definition's columns, checked so a bad definition fails loudly. private static List GetValidatedColumns(VirtualGridDefinition grid, VirtualGridContext context) { List columns = grid.GetColumns(context) ?? new List(); var seen = new HashSet(StringComparer.OrdinalIgnoreCase); foreach (VirtualGridColumn c in columns) { if (!VirtualGridDefinition.IsIdentifier(c.Field) || c.Field.StartsWith("_") || !seen.Add(c.Field)) throw new InvalidOperationException("Virtual grid '" + grid.Key + "' has an invalid or duplicate column field: " + c.Field); } if (columns.Count == 0) throw new InvalidOperationException("Virtual grid '" + grid.Key + "' has no columns."); return columns; } private static VirtualGridColumn DefaultSortColumn(VirtualGridDefinition grid, List columns) { VirtualGridColumn byName = columns.FirstOrDefault(c => grid.CanSort(c) && string.Equals(c.Field, grid.DefaultSortField, StringComparison.OrdinalIgnoreCase)); return byName ?? columns.FirstOrDefault(c => grid.CanSort(c)); } /// The column to sort by and its direction: a sortable column the user picked, else the grid's default. private static VirtualGridColumn ResolveSort(VirtualGridDefinition grid, List columns, VirtualGridRequest request, out bool descending) { VirtualGridColumn picked = columns.FirstOrDefault(c => grid.CanSort(c) && string.Equals(c.Field, request.SortField, StringComparison.OrdinalIgnoreCase)); descending = NormalizeDir(picked != null ? request.SortDir : grid.DefaultSortDir) == "desc"; return picked ?? DefaultSortColumn(grid, columns); } private static string NormalizeDir(string dir) => string.Equals(dir, "desc", StringComparison.OrdinalIgnoreCase) ? "desc" : "asc"; private static string CleanSearch(string search) { string term = (search ?? string.Empty).Trim(); return term.Length > MaxSearchLength ? term.Substring(0, MaxSearchLength) : term; } private static void CheckExportLimit(VirtualGridDefinition grid, int total) { if (total > grid.ExportRowLimit) throw new ArgumentException("There are " + total.ToString("N0", CultureInfo.InvariantCulture) + " rows; Excel export is limited to " + grid.ExportRowLimit.ToString("N0", CultureInfo.InvariantCulture) + ". Please narrow the filters or the search."); } /// /// A value as the screen shows it (dates "dd MMM yyyy", money "1,234.56"), so the search box finds /// what the user sees. Mirrors the formatting in <app-virtual-grid>. /// private static string DisplayText(object value, EnumVirtualGridColumnType type) { if (value == null) return string.Empty; switch (type) { case EnumVirtualGridColumnType.Date: if (value is DateTime d) return d.ToString("dd MMM yyyy", CultureInfo.InvariantCulture); break; case EnumVirtualGridColumnType.DateTime: if (value is DateTime dt) return dt.ToString("dd MMM yyyy HH:mm", CultureInfo.InvariantCulture); break; case EnumVirtualGridColumnType.Money: if (value is IConvertible && !(value is string)) return Convert.ToDecimal(value, CultureInfo.InvariantCulture).ToString("N2", CultureInfo.InvariantCulture); break; } return Convert.ToString(value, CultureInfo.InvariantCulture); } #endregion #region List grids (existing service + in-memory paging) /// A loaded list plus, per row, the text the search box looks in. private sealed class CachedList { public List> Rows; public List SearchText; } /// /// The grid's list for this user and these filters: from the cache, or — on Preview (Refresh) or /// when the cache has expired — freshly from the grid's service. /// private static CachedList GetList(VirtualGridDefinition grid, IVirtualGridListSource source, List columns, VirtualGridRequest request, VirtualGridContext context) { string key = CacheKey(grid, columns, request, context); CachedList list; if (!request.Refresh && ListCache.TryGetValue(key, out list)) return list; List> rows = source.LoadRows(new VirtualGridFilters(request.Filters), context, columns); List searchable = columns.Where(c => grid.CanSearch(c)).ToList(); var searchText = new List(rows.Count); var sb = new StringBuilder(); foreach (Dictionary row in rows) { sb.Clear(); foreach (VirtualGridColumn c in searchable) { object value; row.TryGetValue(c.Field, out value); sb.Append(DisplayText(value, c.Type)).Append('\u0001'); // separator: no match across two columns } searchText.Add(sb.ToString()); } list = new CachedList { Rows = rows, SearchText = searchText }; ListCache.Set(key, list, new MemoryCacheEntryOptions { Size = Math.Max(1, rows.Count), SlidingExpiration = ListCacheSliding, AbsoluteExpirationRelativeToNow = ListCacheAbsolute, }); return list; } /// One list per user, payroll type, grid and filter values (and column set, in case configuration changes). private static string CacheKey(VirtualGridDefinition grid, List columns, VirtualGridRequest request, VirtualGridContext context) { var sb = new StringBuilder(); sb.Append(grid.Key.ToLowerInvariant()) .Append('|').Append(context.UserID) .Append('|').Append(context.PayrollTypeID) .Append('|').Append((int)context.UserType) .Append('|').Append(string.Join(",", columns.Select(c => c.Field))); foreach (KeyValuePair f in (request.Filters ?? new Dictionary()) .OrderBy(f => f.Key, StringComparer.OrdinalIgnoreCase)) sb.Append('|').Append(f.Key.ToLowerInvariant()).Append('=').Append(f.Value); return sb.ToString(); } /// Indexes of the rows whose shown text contains the search (case-insensitive); all rows when there is none. private static List Search(CachedList list, string search) { string term = CleanSearch(search); var result = new List(term.Length == 0 ? list.Rows.Count : 64); for (int i = 0; i < list.Rows.Count; i++) { if (term.Length == 0 || list.SearchText[i].IndexOf(term, StringComparison.OrdinalIgnoreCase) >= 0) result.Add(i); } return result; } /// Sorts row indexes by the chosen column (stable, so equal values keep the service's order). private static List Sort(VirtualGridDefinition grid, List columns, CachedList list, List indexes, VirtualGridRequest request) { bool descending; VirtualGridColumn sortColumn = ResolveSort(grid, columns, request, out descending); if (sortColumn == null) return indexes; string field = sortColumn.Field; Func keyOf = i => { object value; list.Rows[i].TryGetValue(field, out value); return value; }; return (descending ? indexes.OrderByDescending(keyOf, ValueComparer.Instance) : indexes.OrderBy(keyOf, ValueComparer.Instance)) .ToList(); } /// Compares cell values: empty first, numbers as numbers, dates as dates, text case-insensitively. private sealed class ValueComparer : IComparer { public static readonly ValueComparer Instance = new ValueComparer(); public int Compare(object x, object y) { bool xEmpty = x == null || (x is string xs && xs.Length == 0); bool yEmpty = y == null || (y is string ys && ys.Length == 0); if (xEmpty || yEmpty) return xEmpty == yEmpty ? 0 : (xEmpty ? -1 : 1); if (x is string a && y is string b) return StringComparer.InvariantCultureIgnoreCase.Compare(a, b); if (IsNumber(x) && IsNumber(y)) return Convert.ToDecimal(x, CultureInfo.InvariantCulture).CompareTo(Convert.ToDecimal(y, CultureInfo.InvariantCulture)); if (x.GetType() == y.GetType() && x is IComparable comparable) return comparable.CompareTo(y); return StringComparer.InvariantCultureIgnoreCase.Compare( Convert.ToString(x, CultureInfo.InvariantCulture), Convert.ToString(y, CultureInfo.InvariantCulture)); } private static bool IsNumber(object v) => v is decimal || v is double || v is float || v is int || v is long || v is short || v is byte; } #endregion #region SQL grids (VirtualGridDA) private static VirtualGridQuery BuildQuery(VirtualGridSqlDefinition grid, List columns, VirtualGridRequest request, VirtualGridContext context) { VirtualGridSource source = grid.BuildSource(new VirtualGridFilters(request.Filters), context); if (source == null || string.IsNullOrWhiteSpace(source.FromWhere)) throw new InvalidOperationException("Virtual grid '" + grid.Key + "' returned no source."); var query = new VirtualGridQuery { Source = source, TieBreaker = grid.TieBreakerSql }; foreach (VirtualGridColumn c in columns.Where(c => !string.IsNullOrEmpty(c.Sql))) query.Select.Add(c.Sql + " AS [" + c.Field + "]"); foreach (string hidden in grid.HiddenSelect) query.Select.Add(hidden); // Sort: only a sortable column of this grid; anything else falls back to the default. bool descending; VirtualGridColumn sortColumn = ResolveSort(grid, columns, request, out descending); query.OrderBy = (sortColumn != null ? sortColumn.Sql : grid.TieBreakerSql) + (descending ? " DESC" : " ASC"); // Search: the user's text is a parameter; only LIKE wildcards are escaped. string term = CleanSearch(request.Search); if (term.Length > 0) { query.SearchPattern = "%" + term.Replace(@"\", @"\\").Replace("%", @"\%").Replace("_", @"\_").Replace("[", @"\[") + "%"; foreach (VirtualGridColumn c in columns.Where(c => grid.CanSearch(c))) query.SearchColumns.Add(new KeyValuePair(c.Sql, c.Type)); var extras = new VirtualGridSearchExtras(); grid.AddSearchExtras(term, extras, context); query.SearchPredicates.AddRange(extras.Predicates); } return query; } private static List> ReadRows(IDataReader reader) { var rows = new List>(); try { int count = reader.FieldCount; var names = new string[count]; for (int i = 0; i < count; i++) names[i] = reader.GetName(i); while (reader.Read()) { var row = new Dictionary(count, StringComparer.OrdinalIgnoreCase); for (int i = 0; i < count; i++) row[names[i]] = reader.IsDBNull(i) ? null : reader.GetValue(i); rows.Add(row); } } finally { reader.Close(); } return rows; } /// Lets the definition fill computed columns, then drops the hidden "_" helper values. private static void FinishRows(VirtualGridSqlDefinition grid, List> rows, VirtualGridContext context) { if (rows.Count == 0) return; grid.OnRowsLoaded(rows, context); foreach (Dictionary row in rows) { foreach (string key in row.Keys.Where(k => k.StartsWith("_")).ToList()) row.Remove(key); } } #endregion #region Excel /// The requested fields that are columns of this grid, in the requested order (all columns when none are given). private static List ResolveExportColumns(List columns, List fields) { if (fields == null || fields.Count == 0) return columns; var byField = columns.ToDictionary(c => c.Field, StringComparer.OrdinalIgnoreCase); var result = new List(); var seen = new HashSet(StringComparer.OrdinalIgnoreCase); foreach (string field in fields) { VirtualGridColumn c; if (field != null && byField.TryGetValue(field.Trim(), out c) && seen.Add(c.Field)) result.Add(c); } if (result.Count == 0) throw new ArgumentException("No columns to export."); return result; } private static byte[] BuildWorkbook(VirtualGridDefinition grid, List columns, List> rows, string filterDescription, string searchText) { const int headerRow = 5; using (var workbook = new XLWorkbook()) { string sheetName = grid.Title.Length > 31 ? grid.Title.Substring(0, 31) : grid.Title; foreach (char ch in new[] { ':', '\\', '/', '?', '*', '[', ']' }) sheetName = sheetName.Replace(ch, ' '); IXLWorksheet ws = workbook.Worksheets.Add(sheetName); ws.Cell(1, 1).Value = grid.Title; ws.Cell(1, 1).Style.Font.Bold = true; ws.Cell(1, 1).Style.Font.FontSize = 14; string line2 = filterDescription ?? string.Empty; if (!string.IsNullOrWhiteSpace(searchText)) line2 += (line2.Length > 0 ? " | " : "") + "Search: " + searchText.Trim(); ws.Cell(2, 1).Value = line2; ws.Cell(3, 1).Value = "Total: " + rows.Count.ToString("N0", CultureInfo.InvariantCulture); for (int c = 0; c < columns.Count; c++) { ws.Cell(headerRow, c + 1).Value = columns[c].Title; switch (columns[c].Type) { case EnumVirtualGridColumnType.Date: ws.Column(c + 1).Style.DateFormat.Format = "dd MMM yyyy"; break; case EnumVirtualGridColumnType.DateTime: ws.Column(c + 1).Style.DateFormat.Format = "dd MMM yyyy HH:mm"; break; case EnumVirtualGridColumnType.Money: ws.Column(c + 1).Style.NumberFormat.Format = "#,##0.00"; break; } } IXLRange header = ws.Range(headerRow, 1, headerRow, columns.Count); header.Style.Font.Bold = true; header.Style.Fill.BackgroundColor = XLColor.FromHtml("#E7ECF3"); header.Style.Border.OutsideBorder = XLBorderStyleValues.Thin; header.Style.Border.InsideBorder = XLBorderStyleValues.Thin; int r = headerRow + 1; foreach (Dictionary row in rows) { for (int c = 0; c < columns.Count; c++) { object value; if (!row.TryGetValue(columns[c].Field, out value) || value == null) continue; SetCell(ws.Cell(r, c + 1), value, columns[c].Type); } r++; } int lastRow = Math.Max(headerRow, r - 1); ws.Range(headerRow, 1, lastRow, columns.Count).SetAutoFilter(); ws.SheetView.FreezeRows(headerRow); // Measure the header and the first rows only: measuring every row is slow on large files. ws.Columns(1, columns.Count).AdjustToContents(headerRow, Math.Min(lastRow, headerRow + 300), 8, 60); using (var stream = new MemoryStream()) { workbook.SaveAs(stream); return stream.ToArray(); } } } private static void SetCell(IXLCell cell, object value, EnumVirtualGridColumnType type) { switch (type) { case EnumVirtualGridColumnType.Date: case EnumVirtualGridColumnType.DateTime: if (value is DateTime dt) { cell.Value = dt; return; } break; case EnumVirtualGridColumnType.Money: case EnumVirtualGridColumnType.Number: if (value is IConvertible && !(value is string) && !(value is DateTime)) { cell.Value = Convert.ToDecimal(value, CultureInfo.InvariantCulture); return; } break; } cell.Value = Convert.ToString(value, CultureInfo.InvariantCulture); // text: no formulas, keeps leading zeros } #endregion protected override T CreateObject(DataReader dr) { throw new NotSupportedException("VirtualGridService reads rows generically."); } } }