using System; using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; using ClosedXML.Excel; using Ease.Core.Model; using Ease.Core.Utility; using HRM.BO; namespace HRM.DA { /// /// Builds the downloadable Salary Master Data workbook from /// , validates an uploaded one layer by layer, and /// saves it through . It opens no transaction of its /// own: the save is that service's one all-or-nothing transaction. /// public class SalaryMasterBulkUploadService : ISalaryMasterBulkUploadService { #region Template public byte[] GetTemplate(DateTime salaryMonth) { try { List columns = SalaryMasterBulkUploadTemplate.GetColumns(); using (XLWorkbook workbook = new XLWorkbook()) { IXLWorksheet sheet = workbook.Worksheets.Add(SalaryMasterBulkUploadTemplate.SheetName); BuildMonthRow(sheet, salaryMonth); BuildHeadingRow(sheet, columns); ApplyDataArea(sheet, columns); // Keeps the month, the headings and Employee ID / Name in view while scrolling. sheet.SheetView.FreezeRows(SalaryMasterBulkUploadTemplate.HeadingRow); sheet.SheetView.FreezeColumns(2); using (MemoryStream stream = new MemoryStream()) { workbook.SaveAs(stream); return stream.ToArray(); } } } catch (Exception e) { #region Handle Exception ExceptionLog.Write(e); throw new ServiceException(e.Message, e); #endregion } } /// Row 1: "Salary Month" and the month itself, which the upload checks against the page. private void BuildMonthRow(IXLWorksheet sheet, DateTime salaryMonth) { int row = SalaryMasterBulkUploadTemplate.MonthRow; XLColor borderColor = XLColor.FromHtml(SalaryMasterBulkUploadTemplate.BorderColor); IXLCell label = sheet.Cell(row, 1); label.Value = SalaryMasterBulkUploadTemplate.MonthLabel; label.Style.Font.Bold = true; label.Style.Fill.BackgroundColor = XLColor.FromHtml(SalaryMasterBulkUploadTemplate.EmployeeHeaderColor); SetHeaderLayout(label.Style, borderColor); // B1 only, not merged into C1: the column freeze sits between B and C and would cut a merged cell in two. IXLCell month = sheet.Cell(row, 2); SetHeaderLayout(month.Style, borderColor); month.Value = new DateTime(salaryMonth.Year, salaryMonth.Month, 1); month.Style.NumberFormat.Format = "mmmm yyyy"; month.Style.Font.Bold = true; IXLCell note = sheet.Cell(row, 4); note.Value = "Upload this file for " + SalaryMasterBulkUploadTemplate.MonthText(salaryMonth) + ". Do not change the month or the headings; enter the data from row " + SalaryMasterBulkUploadTemplate.FirstDataRow + "."; note.Style.Font.Italic = true; note.Style.Font.FontColor = XLColor.FromHtml("#6B7280"); sheet.Row(row).Height = 24; } private void BuildHeadingRow(IXLWorksheet sheet, List columns) { int row = SalaryMasterBulkUploadTemplate.HeadingRow; XLColor borderColor = XLColor.FromHtml(SalaryMasterBulkUploadTemplate.BorderColor); for (int i = 0; i < columns.Count; i++) { SalaryMasterBulkUploadColumn definition = columns[i]; IXLCell cell = sheet.Cell(row, i + 1); cell.Value = definition.Heading; cell.Style.Fill.BackgroundColor = XLColor.FromHtml(HeaderColor(definition)); cell.Style.Font.Bold = true; cell.Style.Font.FontSize = 10; cell.Style.Font.FontColor = XLColor.FromHtml(definition.IsMandatory ? SalaryMasterBulkUploadTemplate.MandatoryTextColor : SalaryMasterBulkUploadTemplate.OptionalTextColor); SetHeaderLayout(cell.Style, borderColor); sheet.Column(i + 1).Width = definition.Width; } sheet.Row(row).Height = 20; } private static string HeaderColor(SalaryMasterBulkUploadColumn column) { switch (column.ItemSide) { case HistorySalaryItemSide.Gross: return SalaryMasterBulkUploadTemplate.GrossHeaderColor; case HistorySalaryItemSide.Deduction: return SalaryMasterBulkUploadTemplate.DeductionHeaderColor; case HistorySalaryItemSide.Others: return SalaryMasterBulkUploadTemplate.OthersHeaderColor; default: return SalaryMasterBulkUploadTemplate.EmployeeHeaderColor; } } /// /// Text columns are formatted as text, so an Employee ID such as 00123 keeps its zeros; /// amount columns get a number format. /// private void ApplyDataArea(IXLWorksheet sheet, List columns) { int firstRow = SalaryMasterBulkUploadTemplate.FirstDataRow; int lastRow = firstRow + SalaryMasterBulkUploadTemplate.FormattedDataRows - 1; for (int i = 0; i < columns.Count; i++) { sheet.Range(firstRow, i + 1, lastRow, i + 1).Style.NumberFormat.Format = columns[i].IsAmount ? SalaryMasterBulkUploadTemplate.AmountFormat : "@"; if (columns[i].Key == SalaryMasterBulkUploadTemplate.IsConfirmKey) { IXLDataValidation validation = sheet.Range(firstRow, i + 1, lastRow, i + 1).CreateDataValidation(); validation.IgnoreBlanks = true; validation.InCellDropdown = true; validation.ErrorStyle = XLErrorStyle.Stop; validation.ErrorTitle = "Invalid value"; validation.ErrorMessage = "Pick Yes or No."; validation.List("\"" + SalaryMasterBulkUploadTemplate.ConfirmYes + "," + SalaryMasterBulkUploadTemplate.ConfirmNo + "\"", true); } } } /// Centred, wrapped, with a thin border. private void SetHeaderLayout(IXLStyle style, XLColor borderColor) { style.Alignment.Horizontal = XLAlignmentHorizontalValues.Center; style.Alignment.Vertical = XLAlignmentVerticalValues.Center; style.Alignment.WrapText = true; style.Border.OutsideBorder = XLBorderStyleValues.Thin; style.Border.OutsideBorderColor = borderColor; } #endregion #region Read upload /// /// Validates an upload layer by layer. All checks of a layer run and are reported together, /// but the next layer runs only when this one passed. The layers, in order: /// 1 the file is a readable Excel workbook with the template's sheet; /// 2 its salary month is the month picked on the page, and its headings are the template's; /// 3 it holds employee rows, and a payroll type is selected; /// 4 every row has every mandatory value (Employee ID to Is Confirm), and no Employee ID is in two rows; /// 5 every Employee ID is a live employee of the payroll type, under that Name; /// 6 Department, Designation, Location, Cost-Center and Grade exist in the payroll type's /// master data, and Is Confirm is Yes or No; /// 7 every amount is a number, and not negative; a blank amount is saved empty (NULL). /// Layers 4 to 7 check row by row, so they all run once layer 3 passed — one upload lists /// every problem of every row. Totals that do not add up are warnings only. /// The file can be saved only when every layer passed. /// public SalaryMasterBulkUploadReadResult ReadUpload(Stream stream, string fileName, long fileLength, DateTime salaryMonth, int payrollTypeID) { try { SalaryMasterBulkUploadReadResult result = new SalaryMasterBulkUploadReadResult(); result.FileName = fileName; result.SalaryMonth = GlobalFunctions.LastDateOfMonth(salaryMonth); EmployeeBulkUploadLayer fileLayer = result.AddLayer(1, "Excel file", "The upload is a readable Excel workbook (.xlsx) made from the template."); EmployeeBulkUploadLayer columnLayer = result.AddLayer(2, "Salary month and columns", "The file is for the salary month picked on the page, and row " + SalaryMasterBulkUploadTemplate.HeadingRow + " holds exactly the template's headings, in order."); EmployeeBulkUploadLayer rowLayer = result.AddLayer(3, "Rows", "The file holds employee rows, no more than one upload takes, and a payroll type is selected."); EmployeeBulkUploadLayer identityLayer = result.AddLayer(4, "Mandatory values", "Every row has Employee ID, Name, Department, Designation, Location, Cost-Center, Grade and " + "Is Confirm, and each Employee ID is in one row only."); EmployeeBulkUploadLayer employeeLayer = result.AddLayer(5, "Employees", "Every Employee ID is a live employee of the current payroll type, and its Name is that employee's name."); EmployeeBulkUploadLayer masterLayer = result.AddLayer(6, "Master data", "Department, Designation, Location, Cost-Center and Grade are names that exist in the system " + "for the current payroll type, and Is Confirm is Yes or No."); EmployeeBulkUploadLayer amountLayer = result.AddLayer(7, "Amounts", "Every amount is a number and not negative; a blank amount is saved empty."); using (UploadFile upload = new UploadFile()) { CheckFile(fileLayer, stream, fileName, fileLength, upload); bool run = fileLayer.Status == EnumBulkUploadCheckStatus.Passed; CheckMonthAndColumns(columnLayer, result, run ? upload : null); Gate(columnLayer, run, fileLayer); run = columnLayer.Status == EnumBulkUploadCheckStatus.Passed; if (run) { ReadRows(result, upload); } CheckRows(rowLayer, result, payrollTypeID, run); Gate(rowLayer, run, columnLayer); // Layers 4 to 7 check each row on its own, so they all run once the rows are // read — the problem list then shows every problem of the file at once. run = rowLayer.Status == EnumBulkUploadCheckStatus.Passed; CheckIdentity(identityLayer, result, run); Gate(identityLayer, run, rowLayer); CheckEmployees(employeeLayer, result, payrollTypeID, run); Gate(employeeLayer, run, rowLayer); CheckMasterData(masterLayer, result, payrollTypeID, run); Gate(masterLayer, run, rowLayer); CheckAmounts(amountLayer, result, run); Gate(amountLayer, run, rowLayer); } if (payrollTypeID > 0 && result.Rows.Count > 0) { // Read only: marks the employees whose row for the month a save would replace. MarkReplacements(result, payrollTypeID); } // A failed check about the file itself has no cell of its own; it goes into the // problem list too, so the list holds every problem. AddFileIssues(result); List columns = SalaryMasterBulkUploadTemplate.GetColumns(); result.Issues = result.Issues .OrderBy(x => x.RowNo) .ThenBy(x => x.Severity == SalaryMasterBulkUploadIssue.Error ? 0 : 1) .ThenBy(x => columns.FindIndex(c => c.Label == x.Column)) .ToList(); foreach (SalaryMasterBulkUploadRow row in result.Rows) { row.HasError = result.Issues.Any(x => x.RowNo == row.RowNo && x.Severity == SalaryMasterBulkUploadIssue.Error); row.HasWarning = result.Issues.Any(x => x.RowNo == row.RowNo && x.Severity == SalaryMasterBulkUploadIssue.Warning); } return result; } catch (Exception e) { #region Handle Exception ExceptionLog.Write(e); throw new ServiceException(e.Message, e); #endregion } } /// /// A layer that did not run because did not pass: its checks /// say so, naming that layer. /// private static void Gate(EmployeeBulkUploadLayer layer, bool ran, EmployeeBulkUploadLayer previous) { if (!ran) { layer.SkipRemaining("Not checked — runs after layer " + previous.LayerNo + " (" + previous.Name + ") passes."); } } /// The opened workbook. private class UploadFile : IDisposable { public MemoryStream Buffer { get; set; } public XLWorkbook Workbook { get; set; } public IXLWorksheet Sheet { get; set; } public void Dispose() { if (Workbook != null) { Workbook.Dispose(); } if (Buffer != null) { Buffer.Dispose(); } } } #region Layer 1 — Excel file private void CheckFile(EmployeeBulkUploadLayer layer, Stream stream, string fileName, long fileLength, UploadFile upload) { EmployeeBulkUploadCheck fileReceived = layer.AddCheck("FileReceived", "A file is uploaded"); EmployeeBulkUploadCheck fileType = layer.AddCheck("FileType", "The file is an Excel workbook (.xlsx)"); EmployeeBulkUploadCheck fileSize = layer.AddCheck("FileSize", "The file is not empty and is at most " + SalaryMasterBulkUploadTemplate.MaxUploadMegabytes + " MB"); EmployeeBulkUploadCheck fileOpens = layer.AddCheck("FileOpens", "The file opens as an Excel workbook"); EmployeeBulkUploadCheck sheetFound = layer.AddCheck("SheetFound", "The workbook has the \"" + SalaryMasterBulkUploadTemplate.SheetName + "\" sheet"); if (stream == null || string.IsNullOrWhiteSpace(fileName)) { fileReceived.Fail("No file reached the server.", null, "Choose the filled template and upload it again."); layer.SkipRemaining("Not checked — no file was uploaded."); return; } fileReceived.Pass(); if (!SalaryMasterBulkUploadTemplate.IsExcelFile(fileName)) { string extension = Path.GetExtension(fileName); fileType.Fail("\"" + fileName + "\" is " + (string.IsNullOrEmpty(extension) ? "a file without an extension" : "a " + extension + " file") + ", not an Excel workbook (.xlsx).", null, "Use the template downloaded from this page. If the file was saved in another format, " + "open it in Excel and save it as \"Excel Workbook (*.xlsx)\"."); } else { fileType.Pass(); } if (fileLength <= 0) { fileSize.Fail("The file is empty (0 KB).", null, "Fill in the template, save it and upload it again."); } else if (fileLength > SalaryMasterBulkUploadTemplate.MaxUploadMegabytes * 1024L * 1024L) { fileSize.Fail("The file is " + (fileLength / (1024.0 * 1024.0)).ToString("0.00") + " MB; the limit is " + SalaryMasterBulkUploadTemplate.MaxUploadMegabytes + " MB.", null, "Split the employees into several files and upload them one by one."); } else { fileSize.Pass(); } if (!fileType.IsPassed || !fileSize.IsPassed) { layer.SkipRemaining("Not checked — the file type or size check above failed."); return; } // ClosedXML needs a seekable stream; an upload stream may not be one. upload.Buffer = new MemoryStream(); stream.CopyTo(upload.Buffer); upload.Buffer.Position = 0; try { upload.Workbook = new XLWorkbook(upload.Buffer); } catch (Exception) { fileOpens.Fail("The file is named .xlsx, but its content is not a readable Excel workbook — " + "it may be another kind of file renamed to .xlsx, damaged, or password-protected.", null, "Open it in Excel, save it as \"Excel Workbook (*.xlsx)\" without a password and upload it again."); layer.SkipRemaining("Not checked — the file could not be opened."); return; } fileOpens.Pass(); IXLWorksheet sheet; if (!upload.Workbook.TryGetWorksheet(SalaryMasterBulkUploadTemplate.SheetName, out sheet)) { sheetFound.Fail("The workbook has no sheet named \"" + SalaryMasterBulkUploadTemplate.SheetName + "\".", upload.Workbook.Worksheets.Select(x => "Sheet in the file: \"" + x.Name + "\""), "Start from the template downloaded from this page and do not rename its sheet."); return; } sheetFound.Pass(); upload.Sheet = sheet; } #endregion #region Layer 2 — salary month and columns /// /// The month in B1 is the salary month picked on the page, and the heading row holds /// exactly the template's headings in order, with nothing after them. Runs only on a file /// that passed layer 1 ( is null otherwise). /// private void CheckMonthAndColumns(EmployeeBulkUploadLayer layer, SalaryMasterBulkUploadReadResult result, UploadFile upload) { string monthText = SalaryMasterBulkUploadTemplate.MonthText(result.SalaryMonth); EmployeeBulkUploadCheck monthCheck = layer.AddCheck("SalaryMonth", "Cell B" + SalaryMasterBulkUploadTemplate.MonthRow + " holds the salary month " + monthText); EmployeeBulkUploadCheck headingCheck = layer.AddCheck("Headings", "Row " + SalaryMasterBulkUploadTemplate.HeadingRow + " holds the template's " + SalaryMasterBulkUploadTemplate.GetColumns().Count + " headings, in order"); EmployeeBulkUploadCheck extraCheck = layer.AddCheck("NoExtraColumns", "There are no columns after \"" + SalaryMasterBulkUploadTemplate.GetColumns().Last().Label + "\""); if (upload == null) { return; } // Salary month IXLCell monthCell = upload.Sheet.Cell(SalaryMasterBulkUploadTemplate.MonthRow, 2); DateTime fileMonth; if (!TryReadMonth(monthCell, out fileMonth)) { monthCheck.Fail("Cell B" + SalaryMasterBulkUploadTemplate.MonthRow + " does not hold a month" + (monthCell.IsEmpty() ? " — it is empty." : " — it holds \"" + CellText(monthCell) + "\"."), null, "Download the template for " + monthText + " from this page and copy your data into it."); } else if (fileMonth.Year != result.SalaryMonth.Year || fileMonth.Month != result.SalaryMonth.Month) { monthCheck.Fail("The file is for " + SalaryMasterBulkUploadTemplate.MonthText(fileMonth) + ", but " + monthText + " is picked on the page.", null, "Pick " + SalaryMasterBulkUploadTemplate.MonthText(fileMonth) + " on the page, or download the template for " + monthText + " and copy your data into it."); } else { monthCheck.Pass(); } // Headings List columns = SalaryMasterBulkUploadTemplate.GetColumns(); List wrong = new List(); for (int i = 0; i < columns.Count; i++) { IXLCell cell = upload.Sheet.Cell(SalaryMasterBulkUploadTemplate.HeadingRow, i + 1); string found = CellText(cell); if (!string.Equals(NormalizeHeading(found), NormalizeHeading(columns[i].Label), StringComparison.OrdinalIgnoreCase)) { wrong.Add("Column " + ColumnLetter(i + 1) + ": expected \"" + columns[i].Label + "\", found " + (string.IsNullOrEmpty(found) ? "an empty cell" : "\"" + found + "\"")); } } if (wrong.Count == 0) { headingCheck.Pass(); } else { headingCheck.Fail(wrong.Count + " heading(s) in row " + SalaryMasterBulkUploadTemplate.HeadingRow + " are not the template's.", wrong, "Do not add, remove, move or rename columns. Download the template from this page and copy your data into it."); } IXLCell lastHeading = upload.Sheet.Row(SalaryMasterBulkUploadTemplate.HeadingRow).LastCellUsed(); int lastColumn = lastHeading == null ? 0 : lastHeading.Address.ColumnNumber; if (lastColumn > columns.Count) { List extra = new List(); for (int c = columns.Count + 1; c <= lastColumn; c++) { string text = CellText(upload.Sheet.Cell(SalaryMasterBulkUploadTemplate.HeadingRow, c)); if (!string.IsNullOrEmpty(text)) { extra.Add("Column " + ColumnLetter(c) + ": \"" + text + "\""); } } extraCheck.Fail("The file has column(s) that are not in the template.", extra, "Delete the extra columns, or copy your data into the template downloaded from this page."); } else { extraCheck.Pass(); } } /// The month a cell holds: a date (or its serial number), or text such as "October 2026". private bool TryReadMonth(IXLCell cell, out DateTime month) { month = DateTime.MinValue; if (cell.IsEmpty()) { return false; } XLCellValue value = cell.Value; if (value.IsDateTime) { month = value.GetDateTime(); return true; } if (value.IsNumber) { try { month = DateTime.FromOADate(value.GetNumber()); return true; } catch (ArgumentException) { return false; } } if (value.IsText) { string[] formats = { "MMMM yyyy", "MMM yyyy", "MMM-yyyy", "MMMM-yyyy", "MM/yyyy", "M/yyyy" }; return DateTime.TryParseExact(value.GetText().Trim(), formats, CultureInfo.InvariantCulture, DateTimeStyles.None, out month); } return false; } /// A heading without its mandatory mark and extra spaces, for comparing. private static string NormalizeHeading(string heading) { string text = (heading ?? string.Empty).Trim().TrimEnd('*').Trim(); return string.Join(" ", text.Split(new[] { ' ', '\t', '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries)); } #endregion #region Layer 3 — rows /// /// Every row from down that has /// at least one filled cell. Runs once the columns are known to be the template's. /// private void ReadRows(SalaryMasterBulkUploadReadResult result, UploadFile upload) { List columns = SalaryMasterBulkUploadTemplate.GetColumns(); IXLRow lastUsed = upload.Sheet.LastRowUsed(); int lastRow = lastUsed == null ? 0 : lastUsed.RowNumber(); for (int rowNo = SalaryMasterBulkUploadTemplate.FirstDataRow; rowNo <= lastRow; rowNo++) { SalaryMasterBulkUploadRow row = new SalaryMasterBulkUploadRow { RowNo = rowNo }; bool hasValue = false; for (int i = 0; i < columns.Count; i++) { IXLCell cell = upload.Sheet.Cell(rowNo, i + 1); SalaryMasterBulkUploadColumn column = columns[i]; string text = column.IsAmount && cell.Value.IsNumber ? cell.Value.GetNumber().ToString(CultureInfo.InvariantCulture) : CellText(cell); row.Values[column.Key] = text; if (!string.IsNullOrEmpty(text)) { hasValue = true; } } if (hasValue) { result.Rows.Add(row); } } } private void CheckRows(EmployeeBulkUploadLayer layer, SalaryMasterBulkUploadReadResult result, int payrollTypeID, bool run) { EmployeeBulkUploadCheck payrollType = layer.AddCheck("PayrollType", "A payroll type is selected"); EmployeeBulkUploadCheck hasRows = layer.AddCheck("HasRows", "The file holds at least one employee row from row " + SalaryMasterBulkUploadTemplate.FirstDataRow); EmployeeBulkUploadCheck rowLimit = layer.AddCheck("RowLimit", "The file holds at most " + SalaryMasterBulkUploadTemplate.MaxRows + " employee rows"); if (!run) { return; } if (payrollTypeID <= 0) { payrollType.Fail("No payroll type is selected, so the data cannot be saved to one.", null, "Select the payroll type, then upload the file again."); } else { payrollType.Pass(); } if (result.Rows.Count == 0) { hasRows.Fail("The file holds no employee rows.", null, "Enter one employee per row, from row " + SalaryMasterBulkUploadTemplate.FirstDataRow + "."); rowLimit.Skip("Not checked — the file holds no rows."); return; } hasRows.Pass(); if (result.Rows.Count > SalaryMasterBulkUploadTemplate.MaxRows) { rowLimit.Fail("The file holds " + result.Rows.Count + " employee rows; one upload takes at most " + SalaryMasterBulkUploadTemplate.MaxRows + ".", null, "Split the employees into several files and upload them one by one."); } else { rowLimit.Pass(); } } #endregion #region Layer 4 — mandatory values private void CheckIdentity(EmployeeBulkUploadLayer layer, SalaryMasterBulkUploadReadResult result, bool run) { SalaryMasterBulkUploadColumn employeeNoColumn = ColumnOf(SalaryMasterBulkUploadTemplate.EmployeeNoKey); List mandatory = SalaryMasterBulkUploadTemplate.GetColumns() .Where(x => x.IsMandatory).ToList(); // One check per mandatory column (key without "-", e.g. "CostCenterFilled"). Dictionary filled = new Dictionary(); foreach (SalaryMasterBulkUploadColumn column in mandatory) { filled[column.Key] = layer.AddCheck(column.Label.Replace("-", string.Empty).Replace(" ", string.Empty) + "Filled", "Every row has its " + column.Label); } EmployeeBulkUploadCheck unique = layer.AddCheck("EmployeeNoUnique", "Each " + employeeNoColumn.Label + " is in one row only"); if (!run) { return; } foreach (SalaryMasterBulkUploadColumn column in mandatory) { List emptyRows = new List(); foreach (SalaryMasterBulkUploadRow row in result.Rows) { if (string.IsNullOrWhiteSpace(ValueOf(row, column.Key))) { emptyRows.Add(row.RowNo); AddIssue(result, SalaryMasterBulkUploadIssue.Error, row, column, string.Empty, column.Label + " is empty. It is mandatory — every row needs it."); } } PassOrFail(filled[column.Key], emptyRows, "have no " + column.Label, column.Key == SalaryMasterBulkUploadTemplate.EmployeeNoKey ? "Fill in the " + column.Label + " of every row, or delete rows that are not employees." : "Fill in the " + column.Label + " of every row."); } List duplicates = new List(); foreach (IGrouping group in result.Rows .Where(x => !string.IsNullOrWhiteSpace(ValueOf(x, employeeNoColumn.Key))) .GroupBy(x => ValueOf(x, employeeNoColumn.Key).Trim(), StringComparer.OrdinalIgnoreCase) .Where(g => g.Count() > 1)) { List rowNos = group.Select(x => x.RowNo).ToList(); duplicates.Add(group.Key + " — rows " + string.Join(", ", rowNos)); foreach (SalaryMasterBulkUploadRow row in group) { AddIssue(result, SalaryMasterBulkUploadIssue.Error, row, employeeNoColumn, group.Key, employeeNoColumn.Label + " " + group.Key + " is in rows " + string.Join(", ", rowNos) + ". Keep one row per employee."); } } if (duplicates.Count == 0) { unique.Pass(); } else { unique.Fail(duplicates.Count + " " + employeeNoColumn.Label + "(s) are in more than one row.", duplicates, "Keep one row per employee: delete or merge the repeated rows."); } } #endregion #region Layer 5 — employees /// /// Every filled Employee ID is an employee of the current payroll type whose status is /// Live, and the row's Name is that employee's name (case and extra spaces ignored). A /// matching name is then kept as the system spells it. All employees are read in one go. /// private void CheckEmployees(EmployeeBulkUploadLayer layer, SalaryMasterBulkUploadReadResult result, int payrollTypeID, bool run) { SalaryMasterBulkUploadColumn employeeNoColumn = ColumnOf(SalaryMasterBulkUploadTemplate.EmployeeNoKey); SalaryMasterBulkUploadColumn nameColumn = ColumnOf(SalaryMasterBulkUploadTemplate.NameKey); EmployeeBulkUploadCheck exists = layer.AddCheck("EmployeeInPayrollType", "Every " + employeeNoColumn.Label + " is an employee of the current payroll type"); EmployeeBulkUploadCheck live = layer.AddCheck("EmployeeLive", "Every employee is live (active)"); EmployeeBulkUploadCheck nameMatches = layer.AddCheck("NameMatches", "Every " + nameColumn.Label + " is the name of the employee with that " + employeeNoColumn.Label); if (!run) { return; } List rows = result.Rows .Where(x => !string.IsNullOrWhiteSpace(ValueOf(x, employeeNoColumn.Key))).ToList(); List employees = new HistorySalaryMonthlyService() .GetEmployeesByNo(rows.Select(x => ValueOf(x, employeeNoColumn.Key).Trim()).ToList()); List missingRows = new List(); List notLiveRows = new List(); List wrongNameRows = new List(); foreach (SalaryMasterBulkUploadRow row in rows) { string employeeNo = ValueOf(row, employeeNoColumn.Key).Trim(); List found = employees .Where(x => string.Equals((x.EmployeeNo ?? string.Empty).Trim(), employeeNo, StringComparison.OrdinalIgnoreCase)) .ToList(); Employee employee = found.FirstOrDefault(x => x.PayrollTypeID == payrollTypeID); if (employee == null) { missingRows.Add(row.RowNo); AddIssue(result, SalaryMasterBulkUploadIssue.Error, row, employeeNoColumn, employeeNo, found.Count == 0 ? "There is no employee with " + employeeNoColumn.Label + " " + employeeNo + " in the system." : "Employee " + employeeNo + " belongs to another payroll type, not the current one."); continue; } if (employee.Status != EnumEmployeeStatus.Live) { notLiveRows.Add(row.RowNo); AddIssue(result, SalaryMasterBulkUploadIssue.Error, row, employeeNoColumn, employeeNo, "Employee " + employeeNo + " is not live — the status is " + employee.Status + "."); } string name = ValueOf(row, nameColumn.Key); if (string.IsNullOrWhiteSpace(name)) { continue; // reported by layer 4 } if (MasterKey(name) != MasterKey(employee.Name)) { wrongNameRows.Add(row.RowNo); AddIssue(result, SalaryMasterBulkUploadIssue.Error, row, nameColumn, name, "\"" + name + "\" is not the name of employee " + employeeNo + ". In the system the name is \"" + employee.Name + "\"."); } else { row.Values[nameColumn.Key] = employee.Name; } } PassOrFail(exists, missingRows, "have an " + employeeNoColumn.Label + " that is not an employee of the current payroll type", "Enter the Employee ID exactly as in the system, and upload only employees of the current payroll type."); PassOrFail(live, notLiveRows, "have an employee who is not live", "Remove the rows of employees who are not live, or correct their status in the system first."); PassOrFail(nameMatches, wrongNameRows, "have a " + nameColumn.Label + " that does not match the employee", "Enter each employee's name as it is in the system, or correct the Employee ID."); } #endregion #region Layer 6 — master data /// /// Department, Designation, Location, Cost-Center and Grade, when filled, are the name of an /// active record of the current payroll type (case and extra spaces ignored), and Is Confirm /// is Yes or No. A matching value is kept as the system spells it. Each master table is read /// once, and only when the file uses that column. /// private void CheckMasterData(EmployeeBulkUploadLayer layer, SalaryMasterBulkUploadReadResult result, int payrollTypeID, bool run) { List masterKeys = new List { SalaryMasterBulkUploadTemplate.DepartmentKey, SalaryMasterBulkUploadTemplate.DesignationKey, SalaryMasterBulkUploadTemplate.LocationKey, SalaryMasterBulkUploadTemplate.CostCenterKey, SalaryMasterBulkUploadTemplate.GradeKey }; Dictionary checks = new Dictionary(); foreach (string key in masterKeys) { SalaryMasterBulkUploadColumn column = ColumnOf(key); checks[key] = layer.AddCheck(column.Label.Replace("-", string.Empty) + "Exists", "Every " + column.Label + " exists in the system for the current payroll type"); } SalaryMasterBulkUploadColumn confirmColumn = ColumnOf(SalaryMasterBulkUploadTemplate.IsConfirmKey); EmployeeBulkUploadCheck confirm = layer.AddCheck("IsConfirmValue", confirmColumn.Label + " is " + SalaryMasterBulkUploadTemplate.ConfirmYes + " or " + SalaryMasterBulkUploadTemplate.ConfirmNo); if (!run) { return; } foreach (string key in masterKeys) { SalaryMasterBulkUploadColumn column = ColumnOf(key); List rows = result.Rows .Where(x => !string.IsNullOrWhiteSpace(ValueOf(x, key))).ToList(); if (rows.Count == 0) { checks[key].Pass(); continue; } // Name as the system spells it, by matching key; the first one wins when a name repeats. Dictionary names = new Dictionary(); foreach (string name in LoadMasterNames(key, payrollTypeID)) { string masterKey = MasterKey(name); if (masterKey.Length > 0 && !names.ContainsKey(masterKey)) { names[masterKey] = name.Trim(); } } Dictionary> unknown = new Dictionary>(StringComparer.OrdinalIgnoreCase); foreach (SalaryMasterBulkUploadRow row in rows) { string value = ValueOf(row, key).Trim(); string systemName; if (names.TryGetValue(MasterKey(value), out systemName)) { row.Values[key] = systemName; continue; } if (!unknown.ContainsKey(value)) { unknown[value] = new List(); } unknown[value].Add(row.RowNo); AddIssue(result, SalaryMasterBulkUploadIssue.Error, row, column, value, column.Label + " \"" + value + "\" does not exist in the system for the current payroll type."); } if (unknown.Count == 0) { checks[key].Pass(); } else { checks[key].Fail(unknown.Count + " " + column.Label + " name(s) in the file do not exist in the system.", unknown.Select(x => "\"" + x.Key + "\" — row " + string.Join(", ", x.Value)), "Use the " + column.Label + " name exactly as it is in the system, " + "or add the " + column.Label + " in the system first."); } } List badConfirmRows = new List(); foreach (SalaryMasterBulkUploadRow row in result.Rows) { string value = ValueOf(row, confirmColumn.Key).Trim(); if (value.Length == 0) { continue; // reported by layer 4 } if (string.Equals(value, SalaryMasterBulkUploadTemplate.ConfirmYes, StringComparison.OrdinalIgnoreCase)) { row.Values[confirmColumn.Key] = SalaryMasterBulkUploadTemplate.ConfirmYes; } else if (string.Equals(value, SalaryMasterBulkUploadTemplate.ConfirmNo, StringComparison.OrdinalIgnoreCase)) { row.Values[confirmColumn.Key] = SalaryMasterBulkUploadTemplate.ConfirmNo; } else { badConfirmRows.Add(row.RowNo); AddIssue(result, SalaryMasterBulkUploadIssue.Error, row, confirmColumn, value, "\"" + value + "\" is not allowed. " + confirmColumn.Label + " takes " + SalaryMasterBulkUploadTemplate.ConfirmYes + " or " + SalaryMasterBulkUploadTemplate.ConfirmNo + " only."); } } PassOrFail(confirm, badConfirmRows, "have an " + confirmColumn.Label + " that is not " + SalaryMasterBulkUploadTemplate.ConfirmYes + " or " + SalaryMasterBulkUploadTemplate.ConfirmNo, "Pick Yes or No from the cell's dropdown."); } /// The names of every active record of one master table for the payroll type, through its own service. private List LoadMasterNames(string key, int payrollTypeID) { switch (key) { case SalaryMasterBulkUploadTemplate.DepartmentKey: return new DepartmentService().Get(EnumStatus.Active, payrollTypeID).Select(x => x.Name).ToList(); case SalaryMasterBulkUploadTemplate.DesignationKey: return new DesignationService().Get(EnumStatus.Active, payrollTypeID).Select(x => x.Name).ToList(); case SalaryMasterBulkUploadTemplate.LocationKey: return new LocationService().Get(EnumStatus.Active, payrollTypeID).Select(x => x.Name).ToList(); case SalaryMasterBulkUploadTemplate.CostCenterKey: return new CostcenterService().Get(EnumStatus.Active, payrollTypeID).Select(x => x.Name).ToList(); case SalaryMasterBulkUploadTemplate.GradeKey: return new GradeService().Get(EnumStatus.Active, payrollTypeID).Select(x => x.Name).ToList(); default: throw new ArgumentException("Column " + key + " has no master data."); } } /// Trimmed, inner spaces collapsed, lower case — how a cell is matched to the system. private static string MasterKey(string text) { return string.Join(" ", (text ?? string.Empty).Split((char[])null, StringSplitOptions.RemoveEmptyEntries)) .ToLowerInvariant(); } #endregion #region Layer 7 — amounts /// /// Every filled amount cell holds a number that is not negative; a blank cell is saved /// empty (NULL), not as 0. Once they do, each filled total is compared with its items — /// Total Gross with the Gross items, Total Deduction with the Deduction items, Net Pay with /// Total Gross less Total Deduction (a blank item counts as 0 in the sum). A total that does /// not add up is a warning only: the row is saved exactly as the file has it. /// private void CheckAmounts(EmployeeBulkUploadLayer layer, SalaryMasterBulkUploadReadResult result, bool run) { EmployeeBulkUploadCheck isNumber = layer.AddCheck("AmountIsNumber", "Every amount is a number (a blank amount is saved empty)"); EmployeeBulkUploadCheck notNegative = layer.AddCheck("AmountNotNegative", "No amount is negative"); EmployeeBulkUploadCheck totals = layer.AddCheck("TotalsAddUp", "Total Gross, Total Deduction and Net Pay add up (a warning only — rows are saved as in the file)"); if (!run) { return; } List amountColumns = SalaryMasterBulkUploadTemplate.GetColumns() .Where(x => x.IsAmount).ToList(); List notNumberRows = new List(); List negativeRows = new List(); foreach (SalaryMasterBulkUploadRow row in result.Rows) { foreach (SalaryMasterBulkUploadColumn column in amountColumns) { string text = ValueOf(row, column.Key); double? amount; string problem = ReadAmount(text, column, out amount); if (problem != null) { AddMiss(notNumberRows, row.RowNo); AddIssue(result, SalaryMasterBulkUploadIssue.Error, row, column, text, problem); continue; } if (amount == null) { continue; } if (amount < 0) { AddMiss(negativeRows, row.RowNo); AddIssue(result, SalaryMasterBulkUploadIssue.Error, row, column, text, column.Label + " is negative. Enter the amount without a minus sign."); continue; } row.Amounts[column.Key] = amount.Value; } } PassOrFail(isNumber, notNumberRows, "has an amount that is not a number", "Enter digits only, with a decimal point when needed — e.g. 15000 or 1250.50 — without currency signs or words. " + "Replace formulas that show an error with their value."); PassOrFail(notNegative, negativeRows, "has a negative amount", "Enter every amount without a minus sign."); if (!isNumber.IsPassed || !notNegative.IsPassed) { totals.Skip("Not checked — some amounts above are not valid numbers."); return; } int warnedRows = 0; foreach (SalaryMasterBulkUploadRow row in result.Rows) { if (CheckTotals(result, row, amountColumns)) { warnedRows++; } } totals.Pass(); if (warnedRows > 0) { totals.Message = warnedRows + " row(s) have a total that does not add up. They are listed as warnings " + "below and are saved exactly as the file has them."; } } /// The amount a cell holds; null for a blank cell. Returns why it is not an amount, or null. private string ReadAmount(string text, SalaryMasterBulkUploadColumn column, out double? amount) { amount = null; if (string.IsNullOrWhiteSpace(text)) { return null; } if (text.StartsWith("#")) { return "The cell shows the Excel error " + text + " — usually a formula that could not work out " + "its value. Type the amount itself instead."; } decimal number; if (decimal.TryParse(text.Trim(), NumberStyles.Number, CultureInfo.InvariantCulture, out number)) { amount = (double)number; return null; } return "\"" + text + "\" is not a number. " + column.Label + " takes an amount — digits only, without words or currency signs."; } /// Adds a warning per total of the row that does not add up; true when there was one. private bool CheckTotals(SalaryMasterBulkUploadReadResult result, SalaryMasterBulkUploadRow row, List amountColumns) { List grossItems = amountColumns .Where(x => x.ItemSide == HistorySalaryItemSide.Gross).ToList(); List deductionItems = amountColumns .Where(x => x.ItemSide == HistorySalaryItemSide.Deduction).ToList(); double gross = grossItems.Sum(x => AmountOf(row, x.Key)); double deduction = deductionItems.Sum(x => AmountOf(row, x.Key)); double totalGross = AmountOf(row, SalaryMasterBulkUploadTemplate.TotalGrossKey); double totalDeduction = AmountOf(row, SalaryMasterBulkUploadTemplate.TotalDeductionKey); double netPay = AmountOf(row, SalaryMasterBulkUploadTemplate.NetPayKey); // A blank total is not compared: there is nothing in the file to be wrong. bool warned = false; warned |= WarnTotal(result, row, SalaryMasterBulkUploadTemplate.TotalGrossKey, totalGross, gross, string.Join(" + ", grossItems.Select(x => x.Label))); warned |= WarnTotal(result, row, SalaryMasterBulkUploadTemplate.TotalDeductionKey, totalDeduction, deduction, string.Join(" + ", deductionItems.Select(x => x.Label))); warned |= WarnTotal(result, row, SalaryMasterBulkUploadTemplate.NetPayKey, netPay, totalGross - totalDeduction, ColumnOf(SalaryMasterBulkUploadTemplate.TotalGrossKey).Label + " - " + ColumnOf(SalaryMasterBulkUploadTemplate.TotalDeductionKey).Label); return warned; } private bool WarnTotal(SalaryMasterBulkUploadReadResult result, SalaryMasterBulkUploadRow row, string totalKey, double total, double expected, string formula) { if (!row.Amounts.ContainsKey(totalKey) || Math.Abs(total - expected) <= SalaryMasterBulkUploadTemplate.TotalTolerance) { return false; } SalaryMasterBulkUploadColumn column = ColumnOf(totalKey); AddIssue(result, SalaryMasterBulkUploadIssue.Warning, row, column, ValueOf(row, totalKey), column.Label + " is " + total.ToString("#,##0.00", CultureInfo.InvariantCulture) + ", but " + formula + " is " + expected.ToString("#,##0.00", CultureInfo.InvariantCulture) + ". The row is saved as the file has it."); return true; } #endregion /// Marks the rows whose employee already has a row for the month — saving replaces it. private void MarkReplacements(SalaryMasterBulkUploadReadResult result, int payrollTypeID) { HashSet existing = new HashSet( new HistorySalaryMonthlyService().GetEmployeeNos(result.SalaryMonth, payrollTypeID), StringComparer.OrdinalIgnoreCase); foreach (SalaryMasterBulkUploadRow row in result.Rows) { string employeeNo = ValueOf(row, SalaryMasterBulkUploadTemplate.EmployeeNoKey).Trim(); row.IsReplace = employeeNo.Length > 0 && existing.Contains(employeeNo); } result.ReplaceCount = result.Rows.Count(x => x.IsReplace); result.NewCount = result.Rows.Count - result.ReplaceCount; } /// /// One problem-list entry per detail of every failed file-level check (layers 1 to 3) — /// row 0, column "File" — so the problem list alone tells the user everything that is wrong. /// private void AddFileIssues(SalaryMasterBulkUploadReadResult result) { foreach (EmployeeBulkUploadLayer layer in result.Layers.Where(x => x.LayerNo <= 3)) { foreach (EmployeeBulkUploadCheck check in layer.Checks.Where(x => x.Status == EnumBulkUploadCheckStatus.Failed)) { string message = check.Message + (string.IsNullOrEmpty(check.Fix) ? string.Empty : " How to fix: " + check.Fix); List details = check.Details.Count > 0 ? check.Details : new List { string.Empty }; foreach (string detail in details) { result.Issues.Add(new SalaryMasterBulkUploadIssue { Severity = SalaryMasterBulkUploadIssue.Error, RowNo = 0, Cell = string.Empty, EmployeeNo = string.Empty, EmployeeName = string.Empty, Column = check.Title, Value = detail, Message = message }); } } } } #endregion #region Save upload /// /// Validates the file again — the page's earlier read is never trusted — and, only when /// every layer passes, saves every row in one transaction: all rows or none. /// public SalaryMasterBulkUploadSaveResult SaveUpload(Stream stream, string fileName, long fileLength, DateTime salaryMonth, int payrollTypeID, int userID) { SalaryMasterBulkUploadSaveResult saveResult = new SalaryMasterBulkUploadSaveResult(); SalaryMasterBulkUploadReadResult validation = ReadUpload(stream, fileName, fileLength, salaryMonth, payrollTypeID); if (!validation.CanSave) { saveResult.Saved = false; saveResult.Message = "Nothing was saved: the file does not pass every check. Correct the problems and upload it again."; saveResult.Validation = validation; return saveResult; } try { List items = validation.Rows.Select(ToHistory).ToList(); int replaced = new HistorySalaryMonthlyService().Save(items, validation.SalaryMonth, payrollTypeID, userID); saveResult.Saved = true; saveResult.ReplacedCount = replaced; saveResult.InsertedCount = items.Count - replaced; saveResult.Message = items.Count + " employee(s) saved for " + SalaryMasterBulkUploadTemplate.MonthText(validation.SalaryMonth) + (replaced > 0 ? " — " + replaced + " of them replaced their earlier data for the month." : "."); } catch (Exception e) { #region Handle Exception ExceptionLog.Write(e); throw new ServiceException(e.Message, e); #endregion } return saveResult; } /// One validated row as a header with one detail per amount column, in sheet order. private HistorySalaryMonthly ToHistory(SalaryMasterBulkUploadRow row) { HistorySalaryMonthly item = new HistorySalaryMonthly { EmployeeNo = ValueOf(row, SalaryMasterBulkUploadTemplate.EmployeeNoKey).Trim(), Name = ValueOf(row, SalaryMasterBulkUploadTemplate.NameKey).Trim(), Department = ValueOf(row, SalaryMasterBulkUploadTemplate.DepartmentKey).Trim(), Designation = ValueOf(row, SalaryMasterBulkUploadTemplate.DesignationKey).Trim(), Location = ValueOf(row, SalaryMasterBulkUploadTemplate.LocationKey).Trim(), CostCenter = ValueOf(row, SalaryMasterBulkUploadTemplate.CostCenterKey).Trim(), Grade = ValueOf(row, SalaryMasterBulkUploadTemplate.GradeKey).Trim(), IsConfirm = ValueOf(row, SalaryMasterBulkUploadTemplate.IsConfirmKey).Trim() }; int sequence = 1; foreach (SalaryMasterBulkUploadColumn column in SalaryMasterBulkUploadTemplate.GetColumns().Where(x => x.IsAmount)) { item.Details.Add(new HistorySalaryMonthlyDetail { ItemName = column.Label, ItemSide = column.ItemSide, Amount = row.Amounts.ContainsKey(column.Key) ? row.Amounts[column.Key] : (double?)null, Sequence = sequence++ }); } return item; } #endregion #region Helpers private static SalaryMasterBulkUploadColumn ColumnOf(string key) { return SalaryMasterBulkUploadTemplate.GetColumns().First(x => x.Key == key); } private static string ValueOf(SalaryMasterBulkUploadRow row, string key) { string value; return row.Values.TryGetValue(key, out value) && value != null ? value : string.Empty; } private static double AmountOf(SalaryMasterBulkUploadRow row, string key) { double amount; return row.Amounts.TryGetValue(key, out amount) ? amount : 0; } private void AddIssue(SalaryMasterBulkUploadReadResult result, string severity, SalaryMasterBulkUploadRow row, SalaryMasterBulkUploadColumn column, string value, string message) { int columnNo = SalaryMasterBulkUploadTemplate.GetColumns().FindIndex(x => x.Key == column.Key) + 1; result.Issues.Add(new SalaryMasterBulkUploadIssue { Severity = severity, RowNo = row.RowNo, Cell = ColumnLetter(columnNo) + row.RowNo, EmployeeNo = ValueOf(row, SalaryMasterBulkUploadTemplate.EmployeeNoKey), EmployeeName = ValueOf(row, SalaryMasterBulkUploadTemplate.NameKey), Column = column.Label, Value = value, Message = message }); } private static void AddMiss(List rowNos, int rowNo) { if (!rowNos.Contains(rowNo)) { rowNos.Add(rowNo); } } /// Passes the check, or fails it naming the rows (the first 20, then a count). private static void PassOrFail(EmployeeBulkUploadCheck check, List rowNos, string what, string fix) { if (rowNos.Count == 0) { check.Pass(); return; } string shown = string.Join(", ", rowNos.Take(20)) + (rowNos.Count > 20 ? " and " + (rowNos.Count - 20) + " more" : string.Empty); check.Fail(rowNos.Count + " row(s) " + what + " — row " + shown + ". Each cell is listed in the Problems table below.", null, fix); } /// A cell as text: dates as dd-MMM-yyyy, everything else as Excel shows it. private string CellText(IXLCell cell) { if (cell.IsEmpty()) { return string.Empty; } if (cell.DataType == XLDataType.DateTime) { return cell.GetDateTime().ToString("dd-MMM-yyyy"); } return cell.GetFormattedString().Trim(); } private string ColumnLetter(int column) { return XLHelper.GetColumnLetterFromNumber(column); } #endregion } }