HRM-10/HRM.DA/Service/EmployeeBulkUpload/EmployeeBulkUploadService.cs
Mohaimen Hasan 838b8d4480 first commit
2026-11-03 13:06:00 +06:00

4241 lines
217 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

using System;
using System.Collections.Generic;
using System.Data;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Reflection;
using ClosedXML.Excel;
using Ease.Core.DataAccess;
using Ease.Core.Model;
using Ease.Core.Utility;
using HRM.BO;
namespace HRM.DA
{
/// <summary>
/// Builds the downloadable Employee Bulk Upload workbook from
/// <see cref="EmployeeBulkUploadTemplate"/>, validates an uploaded one layer by layer, and
/// adds missing master data. It opens no transaction of its own: master tables are read and
/// written only through their own services, the way the rest of the app does.
/// </summary>
public class EmployeeBulkUploadService : IEmployeeBulkUploadService
{
#region Template
public byte[] GetTemplate()
{
return GetTemplate(null);
}
public byte[] GetTemplate(List<string> subGroupNames)
{
try
{
List<EmployeeBulkUploadGroup> groups = EmployeeBulkUploadTemplate.GetGroups(subGroupNames);
using (XLWorkbook workbook = new XLWorkbook())
{
IXLWorksheet sheet = workbook.Worksheets.Add(EmployeeBulkUploadTemplate.SheetName);
BuildHeader(sheet, groups);
ApplyDataArea(sheet, EmployeeBulkUploadTemplate.GetColumns(groups));
ApplySheetView(sheet);
using (MemoryStream stream = new MemoryStream())
{
workbook.SaveAs(stream);
return stream.ToArray();
}
}
}
catch (Exception e)
{
#region Handle Exception
ExceptionLog.Write(e);
throw new ServiceException(e.Message, e);
#endregion
}
}
#endregion
#region Read upload
/// <summary>
/// Validates an upload phase by phase. All checks of a phase run and are reported together,
/// but the next phase runs only when this one passed — a phase that fails stops the ones
/// after it, which are left not checked. The phases, in order:
/// 1 the file is a readable Excel workbook made from the template;
/// 2 it holds exactly the ticked sections, each with exactly the template's columns and headings;
/// 3 it holds employee rows;
/// 4 every row has its Employee No and Employee Name (and no Employee No is in two rows);
/// 5 every mandatory (*) column has a value;
/// 6 every value is of its column's type;
/// 7 each Employee No is an active employee of this payroll type, or new with General;
/// 8 every master-data value is in the system.
/// The file can be saved only when every phase passed. Problems go to result.Issues.
/// </summary>
public EmployeeBulkUploadReadResult ReadUpload(Stream stream, string fileName, long fileLength,
List<string> requestedSubGroups, int payrollTypeID, List<EmployeeBulkUploadReplacement> replacements = null)
{
try
{
EmployeeBulkUploadReadResult result = new EmployeeBulkUploadReadResult();
result.FileName = fileName;
// Template sections — the declaration of what this upload holds; echoed back even when
// the file fails, so the page can tell when the ticks change after an upload.
List<string> unknownSections;
result.SelectedSubGroups = EmployeeBulkUploadTemplate.ResolveSelection(requestedSubGroups, out unknownSections);
EmployeeBulkUploadLayer fileLayer = result.AddLayer(1, "Excel file", "The upload is a readable Excel workbook (.xlsx) made from the template.");
EmployeeBulkUploadLayer columnLayer = result.AddLayer(2, "Sections and columns", "The file holds exactly the sections ticked in Template sections, and every section holds "
+ "exactly the template's columns, under exactly the template's headings.");
EmployeeBulkUploadLayer rowLayer = result.AddLayer(3, "Rows", "The file holds employee rows, and no more than one upload takes.");
EmployeeBulkUploadLayer identityLayer = result.AddLayer(4, "Employee No and Name", "Every row has its Employee No and Employee Name, and each Employee No is in one row only.");
EmployeeBulkUploadLayer requiredLayer = result.AddLayer(5, "Mandatory values",
"Every ticked section is filled in every row (a repeat group: at least one of its blocks), and "
+ "every mandatory column (marked *) of a filled section has a value.");
EmployeeBulkUploadLayer formatLayer = result.AddLayer(6, "Data types", "Every filled cell holds the kind of value its column takes — a real date in DD-MMM-YYYY, "
+ "a number, an e-mail address, or one of the listed values.");
EmployeeBulkUploadLayer employeeLayer = result.AddLayer(7, "Employees", "Each Employee No either is an active employee of the current payroll type (the row updates "
+ "that employee) or is new and has the General section (the row creates one).");
EmployeeBulkUploadLayer masterLayer = result.AddLayer(8, "Master data", "Every master-data value (District, Religion, Nationality, Education Level, Grade, Line Manager, ...) is in the "
+ "system and active. Tables kept per payroll type are looked up in the current payroll type.");
using (UploadFile upload = new UploadFile())
{
CheckFile(fileLayer, stream, fileName, fileLength, upload);
bool run = fileLayer.Status == EnumBulkUploadCheckStatus.Passed;
CheckSections(columnLayer, result, unknownSections, run ? upload : null);
CheckColumns(columnLayer, run ? upload : null);
Gate(columnLayer, run, fileLayer);
run = columnLayer.Status == EnumBulkUploadCheckStatus.Passed;
CheckRows(rowLayer, result, run ? upload : null);
if (run && result.Rows.Count > 0)
{
// Values the user swapped for existing records in the master-data dialog:
// every later phase, and the save, see the chosen record.
ApplyReplacements(result, replacements, payrollTypeID);
}
Gate(rowLayer, run, columnLayer);
if (!run && fileLayer.Status == EnumBulkUploadCheckStatus.Passed)
{
// The page always shows what the file holds, as far as its columns could be matched.
ReadRowsForDisplay(result, upload);
}
run = rowLayer.Status == EnumBulkUploadCheckStatus.Passed;
CheckRequiredValues(identityLayer, result, run, true);
Gate(identityLayer, run, rowLayer);
run = identityLayer.Status == EnumBulkUploadCheckStatus.Passed;
CheckRequiredValues(requiredLayer, result, run, false);
Gate(requiredLayer, run, identityLayer);
run = requiredLayer.Status == EnumBulkUploadCheckStatus.Passed;
CheckDataTypes(formatLayer, result, run ? upload : null);
Gate(formatLayer, run, requiredLayer);
}
bool runEmployees = formatLayer.Status == EnumBulkUploadCheckStatus.Passed;
Dictionary<int, EmployeeRef> employees = CheckEmployees(employeeLayer, result, payrollTypeID, runEmployees);
Gate(employeeLayer, runEmployees, formatLayer);
if (!runEmployees && payrollTypeID > 0 && result.Rows.Count > 0)
{
// Read only: marks new employees in the data grid even when the phase did not run.
ClassifyEmployees(result, payrollTypeID);
}
bool runMaster = employeeLayer.Status == EnumBulkUploadCheckStatus.Passed;
CheckMasterData(masterLayer, result, payrollTypeID, runMaster, employees);
Gate(masterLayer, runMaster, employeeLayer);
// A failed check about the file itself has no cell of its own; it goes into the
// problem list too, so the list holds every problem.
AddFileIssues(result);
// Every problem points at its Excel cell (e.g. AB12) and the issues read top to bottom.
foreach (EmployeeBulkUploadIssue issue in result.Issues)
{
issue.Cell = issue.RowNo > 0 ? CellOf(result, issue) : string.Empty;
}
result.Issues = result.Issues
.OrderBy(x => x.RowNo)
.ThenBy(x => x.Severity == EmployeeBulkUploadIssue.Error ? 0 : 1)
.ThenBy(x => SheetColumnOf(result, x))
.ToList();
// The rows always go back, so the page shows the uploaded data whatever the problems;
// the save validates the file again and refuses one that does not pass.
return result;
}
catch (Exception e)
{
#region Handle Exception
ExceptionLog.Write(e);
throw new ServiceException(e.Message, e);
#endregion
}
}
/// <summary>
/// A phase that did not run because <paramref name="previous"/> did not pass: its checks
/// say so, naming that phase.
/// </summary>
private static void Gate(EmployeeBulkUploadLayer layer, bool ran, EmployeeBulkUploadLayer previous)
{
if (!ran)
{
layer.SkipRemaining("Not checked — runs after phase " + previous.LayerNo + " ("
+ previous.Name + ") passes.");
}
}
/// <summary>The checks whose problems are about the file as a whole, not about a cell of a row.</summary>
private static readonly HashSet<string> FileLevelChecks = new HashSet<string>
{
"HasRows", "RowLimit", "PayrollType"
};
/// <summary>
/// One problem-list entry per detail of every failed file-level check (phases 1 and 2, no
/// rows, too many rows, no payroll type) — row 0, section "File" — so that the problem list
/// alone tells the user everything that is wrong. Row-level checks already add their own.
/// </summary>
private void AddFileIssues(EmployeeBulkUploadReadResult result)
{
foreach (EmployeeBulkUploadLayer layer in result.Layers)
{
foreach (EmployeeBulkUploadCheck check in layer.Checks.Where(x => x.Status == EnumBulkUploadCheckStatus.Failed
&& (layer.LayerNo <= 2 || FileLevelChecks.Contains(x.Code))))
{
string message = check.Message + (string.IsNullOrEmpty(check.Fix) ? string.Empty : " How to fix: " + check.Fix);
List<string> details = check.Details.Count > 0 ? check.Details : new List<string> { string.Empty };
foreach (string detail in details)
{
result.Issues.Add(new EmployeeBulkUploadIssue
{
Severity = EmployeeBulkUploadIssue.Error,
RowNo = 0,
EmployeeNo = string.Empty,
EmployeeName = string.Empty,
Group = "File",
SubGroup = "File",
Column = check.Title,
Value = detail,
Message = message
});
}
}
}
}
/// <summary>The opened workbook and what phases 1 and 2 found in its header rows.</summary>
private class UploadFile : IDisposable
{
public UploadFile()
{
Bands = new List<UploadBand>();
ColumnMap = new Dictionary<int, EmployeeBulkUploadColumn>();
}
public MemoryStream Buffer { get; set; }
public XLWorkbook Workbook { get; set; }
public IXLWorksheet Sheet { get; set; }
public List<UploadBand> Bands { get; set; }
/// <summary>Sheet column number → template column; filled by phase 2.</summary>
public Dictionary<int, EmployeeBulkUploadColumn> ColumnMap { get; set; }
/// <summary>The template sections found in the file, once each, in file order.</summary>
public List<UploadBand> Sections
{
get { return Bands.Where(x => x.Section != null && !x.IsRepeat).ToList(); }
}
public void Dispose()
{
if (Workbook != null)
{
Workbook.Dispose();
}
if (Buffer != null)
{
Buffer.Dispose();
}
}
}
/// <summary>A run of row-2 cells with the same section name.</summary>
private class UploadBand
{
public string Name { get; set; }
public int Start { get; set; }
public int End { get; set; }
/// <summary>The template section the name matches; null when it is not a template section.</summary>
public EmployeeBulkUploadSubGroup Section { get; set; }
/// <summary>A second copy of a section already found further left.</summary>
public bool IsRepeat { get; set; }
}
#region Phase 1 — Excel file
/// <summary>
/// The file itself: uploaded, .xlsx, not empty and within the size limit, opens as a
/// workbook, has the template's sheet and its header rows. Any other kind of file is refused here.
/// </summary>
private void CheckFile(EmployeeBulkUploadLayer layer, Stream stream, string fileName, long fileLength,
UploadFile upload)
{
string templateHint = "Download the template from this page for the ticked sections and copy your data into it.";
EmployeeBulkUploadCheck fileReceived = layer.AddCheck("FileReceived", "A file is uploaded");
EmployeeBulkUploadCheck fileType = layer.AddCheck("FileType", "The file is an Excel workbook (.xlsx)");
EmployeeBulkUploadCheck fileSize = layer.AddCheck("FileSize",
"The file is not empty and is at most " + EmployeeBulkUploadTemplate.MaxUploadMegabytes + " MB");
EmployeeBulkUploadCheck fileOpens = layer.AddCheck("FileOpens", "The file opens as an Excel workbook");
EmployeeBulkUploadCheck sheetFound = layer.AddCheck("SheetFound",
"The workbook has the \"" + EmployeeBulkUploadTemplate.SheetName + "\" sheet");
EmployeeBulkUploadCheck headerRows = layer.AddCheck("HeaderRows",
"Row " + EmployeeBulkUploadTemplate.SubBandRow + " holds the section names and row "
+ EmployeeBulkUploadTemplate.HeadingRow + " the column headings");
if (stream == null || string.IsNullOrWhiteSpace(fileName))
{
fileReceived.Fail("No file reached the server.", null, "Choose the filled template and upload it again.");
layer.SkipRemaining("Not checked — no file was uploaded.");
return;
}
fileReceived.Pass();
string extension = Path.GetExtension(fileName);
if (!string.Equals(extension, ".xlsx", StringComparison.OrdinalIgnoreCase))
{
fileType.Fail("\"" + fileName + "\" is "
+ (string.IsNullOrEmpty(extension) ? "a file without an extension" : "a " + extension + " file")
+ ", not an Excel workbook (.xlsx).",
null, "Use the template downloaded from this page. If the file was saved in another format, "
+ "open it in Excel and save it as \"Excel Workbook (*.xlsx)\".");
}
else
{
fileType.Pass();
}
if (fileLength <= 0)
{
fileSize.Fail("The file is empty (0 KB).", null, "Fill in the template, save it and upload it again.");
}
else if (fileLength > EmployeeBulkUploadTemplate.MaxUploadMegabytes * 1024L * 1024L)
{
fileSize.Fail("The file is " + (fileLength / (1024.0 * 1024.0)).ToString("0.00") + " MB; the limit is "
+ EmployeeBulkUploadTemplate.MaxUploadMegabytes + " MB.",
null, "Split the employees into several files and upload them one by one.");
}
else
{
fileSize.Pass();
}
if (!fileType.IsPassed || !fileSize.IsPassed)
{
layer.SkipRemaining("Not checked — the file type or size check above failed.");
return;
}
// ClosedXML needs a seekable stream; an upload stream may not be one.
upload.Buffer = new MemoryStream();
stream.CopyTo(upload.Buffer);
upload.Buffer.Position = 0;
try
{
upload.Workbook = new XLWorkbook(upload.Buffer);
}
catch (Exception)
{
fileOpens.Fail("The file is named .xlsx, but its content is not a readable Excel workbook — "
+ "it may be another kind of file renamed to .xlsx, damaged, or password-protected.",
null, "Open it in Excel, save it as \"Excel Workbook (*.xlsx)\" without a password and upload it again.");
layer.SkipRemaining("Not checked — the file could not be opened.");
return;
}
fileOpens.Pass();
IXLWorksheet sheet;
if (!upload.Workbook.TryGetWorksheet(EmployeeBulkUploadTemplate.SheetName, out sheet))
{
sheetFound.Fail("The workbook has no sheet named \"" + EmployeeBulkUploadTemplate.SheetName + "\".",
upload.Workbook.Worksheets.Select(x => "Sheet in the file: \"" + x.Name + "\""),
"Start from the template downloaded from this page and do not rename its sheet.");
layer.SkipRemaining("Not checked — the \"" + EmployeeBulkUploadTemplate.SheetName + "\" sheet was not found.");
return;
}
sheetFound.Pass();
upload.Sheet = sheet;
IXLCell lastBand = sheet.Row(EmployeeBulkUploadTemplate.SubBandRow).LastCellUsed();
IXLCell lastHeading = sheet.Row(EmployeeBulkUploadTemplate.HeadingRow).LastCellUsed();
if (lastBand == null || lastHeading == null)
{
List<string> emptyRows = new List<string>();
if (lastBand == null)
{
emptyRows.Add("Row " + EmployeeBulkUploadTemplate.SubBandRow + " (section names) is empty");
}
if (lastHeading == null)
{
emptyRows.Add("Row " + EmployeeBulkUploadTemplate.HeadingRow + " (column headings) is empty");
}
headerRows.Fail("The header rows of the template are missing.", emptyRows,
"Keep the template's three header rows as they are and enter the data from row "
+ EmployeeBulkUploadTemplate.FirstDataRow + ". " + templateHint);
layer.SkipRemaining("Not checked — the header rows were not found.");
return;
}
headerRows.Pass();
}
#endregion
#region Phase 2 — sections and columns
/// <summary>
/// The sections: the ticked selection is valid, every row-2 section name is a template
/// section used once, the Employee section is there, and the file holds exactly the ticked
/// sections — none missing, none extra. Runs only on a file that passed phase 1
/// (<paramref name="upload"/> is null otherwise). Fills upload.Bands.
/// </summary>
private void CheckSections(EmployeeBulkUploadLayer layer, EmployeeBulkUploadReadResult result,
List<string> unknownSections, UploadFile upload)
{
string templateHint = "Download the template from this page for the ticked sections and copy your data into it.";
EmployeeBulkUploadCheck sectionsTicked = layer.AddCheck("SectionsTicked",
"At least one section is ticked in Template sections");
EmployeeBulkUploadCheck knownSections = layer.AddCheck("KnownSections",
"Every section name in row " + EmployeeBulkUploadTemplate.SubBandRow
+ " is a template section, used once");
EmployeeBulkUploadCheck employeeSection = layer.AddCheck("EmployeeSection",
"The \"" + EmployeeBulkUploadTemplate.AlwaysIncludedSubGroup
+ "\" section (Employee No, Employee Name) is in the file");
EmployeeBulkUploadCheck selectedInFile = layer.AddCheck("SelectedInFile",
"Every section ticked in Template sections is in the file");
EmployeeBulkUploadCheck fileInSelected = layer.AddCheck("FileInSelected",
"Every section in the file is ticked in Template sections");
if (upload == null)
{
layer.SkipRemaining("Not checked — the file is not a readable template.");
return;
}
if (unknownSections.Count > 0)
{
sectionsTicked.Fail("Template sections sent names that are not sections of the template.",
unknownSections, "Reload the page, tick the sections again and upload.");
}
else if (!result.SelectedSubGroups.Any(x => x != EmployeeBulkUploadTemplate.AlwaysIncludedSubGroup))
{
sectionsTicked.Fail("No section is ticked in Template sections, so it is not known what this file holds.",
null, "Tick the sections this file holds, then upload it again.");
}
else
{
sectionsTicked.Pass();
}
IXLWorksheet sheet = upload.Sheet;
IXLCell lastBand = sheet.Row(EmployeeBulkUploadTemplate.SubBandRow).LastCellUsed();
IXLCell lastHeading = sheet.Row(EmployeeBulkUploadTemplate.HeadingRow).LastCellUsed();
// Walk the row-2 bands left to right.
Dictionary<string, EmployeeBulkUploadSubGroup> known = EmployeeBulkUploadTemplate.GetGroups()
.SelectMany(g => g.SubGroups)
.ToDictionary(s => s.Name, StringComparer.OrdinalIgnoreCase);
List<string> sectionProblems = new List<string>();
int lastColumn = Math.Max(lastBand.Address.ColumnNumber, lastHeading.Address.ColumnNumber);
int column = 1;
while (column <= lastColumn)
{
string bandName = BandText(sheet.Cell(EmployeeBulkUploadTemplate.SubBandRow, column));
int start = column;
while (column <= lastColumn && string.Equals(bandName,
BandText(sheet.Cell(EmployeeBulkUploadTemplate.SubBandRow, column)), StringComparison.OrdinalIgnoreCase))
{
column++;
}
int end = column - 1;
if (string.IsNullOrEmpty(bandName))
{
List<string> orphans = Enumerable.Range(start, end - start + 1)
.Select(c => sheet.Cell(EmployeeBulkUploadTemplate.HeadingRow, c).GetString().Trim())
.Where(x => x.Length > 0)
.ToList();
if (orphans.Count > 0)
{
sectionProblems.Add("The heading" + (orphans.Count == 1 ? " " : "s ")
+ string.Join(", ", orphans.Select(x => "\"" + x + "\""))
+ (orphans.Count == 1 ? " has" : " have") + " no section name above "
+ (orphans.Count == 1 ? "it" : "them") + " in row " + EmployeeBulkUploadTemplate.SubBandRow);
}
continue;
}
UploadBand band = new UploadBand { Name = bandName, Start = start, End = end };
EmployeeBulkUploadSubGroup section;
if (!known.TryGetValue(bandName, out section))
{
sectionProblems.Add("\"" + bandName + "\" is not a section of the template");
}
else
{
band.Section = section;
if (upload.Bands.Any(x => x.Section == section))
{
band.IsRepeat = true;
sectionProblems.Add("\"" + section.Name + "\" appears more than once in the file");
}
}
upload.Bands.Add(band);
}
if (sectionProblems.Count > 0)
{
knownSections.Fail("Row " + EmployeeBulkUploadTemplate.SubBandRow
+ " has section names that do not match the template.",
sectionProblems, "Do not rename, copy or add sections. " + templateHint);
}
else
{
knownSections.Pass();
}
List<UploadBand> found = upload.Sections;
if (!found.Any(x => x.Section.Name == EmployeeBulkUploadTemplate.AlwaysIncludedSubGroup))
{
employeeSection.Fail("The \"" + EmployeeBulkUploadTemplate.AlwaysIncludedSubGroup
+ "\" section is not in the file. It ties each row to an employee, so every upload needs it.",
null, "Keep the \"" + EmployeeBulkUploadTemplate.AlwaysIncludedSubGroup
+ "\" section (Employee No, Employee Name) as the first columns of the sheet.");
}
else
{
employeeSection.Pass();
}
// The selection says what this upload is; the file has to match it exactly, both ways.
if (!sectionsTicked.IsPassed)
{
selectedInFile.Skip("Not checked — the Template sections selection above is not valid.");
fileInSelected.Skip("Not checked — the Template sections selection above is not valid.");
return;
}
List<string> notInFile = result.SelectedSubGroups
.Where(x => x != EmployeeBulkUploadTemplate.AlwaysIncludedSubGroup
&& !found.Any(f => string.Equals(f.Section.Name, x, StringComparison.OrdinalIgnoreCase)))
.ToList();
if (notInFile.Count > 0)
{
selectedInFile.Fail(Count(notInFile.Count, "section is", "sections are")
+ " ticked in Template sections, but the file does not have "
+ (notInFile.Count == 1 ? "it" : "them") + ".",
notInFile.Select(SectionLabel),
"Untick " + (notInFile.Count == 1 ? "it" : "them")
+ " in Template sections, or download the template again for this selection and fill it in.");
}
else
{
selectedInFile.Pass();
}
List<UploadBand> notSelected = found
.Where(x => x.Section.Name != EmployeeBulkUploadTemplate.AlwaysIncludedSubGroup
&& !result.SelectedSubGroups.Contains(x.Section.Name, StringComparer.OrdinalIgnoreCase))
.ToList();
if (notSelected.Count > 0)
{
fileInSelected.Fail("The file has " + Count(notSelected.Count, "section", "sections")
+ " that " + (notSelected.Count == 1 ? "is" : "are") + " not ticked in Template sections.",
notSelected.Select(x => SectionLabel(x.Section.Name)),
"Tick " + (notSelected.Count == 1 ? "it" : "them")
+ " in Template sections, or remove " + (notSelected.Count == 1 ? "it" : "them")
+ " from the file, then upload again.");
}
else
{
fileInSelected.Pass();
}
}
/// <summary>
/// The columns, part of phase 2. Runs only on a file that passed phase 1
/// (<paramref name="upload"/> is null otherwise, and every check is left not run). A
/// heading must be the template's column name, letter for letter and space for space;
/// capital/small letters and the " *" do not matter (<see cref="HeadingKey"/>) — which
/// columns are mandatory is known here, not read from the file. Columns are matched by that heading
/// within their section, never by position; position is used only to suggest the heading a
/// wrong one should have been. Each section must hold exactly its template columns — none
/// missing, none extra, none twice. Fills upload.ColumnMap.
/// </summary>
private void CheckColumns(EmployeeBulkUploadLayer layer, UploadFile upload)
{
EmployeeBulkUploadCheck allColumns = layer.AddCheck("AllColumns",
"Every section in the file has all of its columns");
EmployeeBulkUploadCheck columnNames = layer.AddCheck("ColumnNames",
"Every column heading in row " + EmployeeBulkUploadTemplate.HeadingRow
+ " is a column name of the template, used once, with no extra column");
if (upload == null)
{
layer.SkipRemaining("Not checked — the file is not a readable template.");
return;
}
string templateHint = "Download the template from this page for the ticked sections and copy your data into it.";
IXLWorksheet sheet = upload.Sheet;
List<string> missingColumns = new List<string>();
List<string> headingProblems = new List<string>();
foreach (UploadBand band in upload.Sections)
{
EmployeeBulkUploadSubGroup section = band.Section;
Dictionary<string, EmployeeBulkUploadColumn> byHeading = section.Columns
.ToDictionary(c => HeadingKey(c.Label), StringComparer.Ordinal);
// Only to explain a heading that differs from the name just in its spacing.
Dictionary<string, EmployeeBulkUploadColumn> byLooseHeading = section.Columns
.ToDictionary(c => EmployeeBulkUploadTemplate.NormalizeHeading(c.Label));
Dictionary<string, int> matchedAt = new Dictionary<string, int>(StringComparer.Ordinal);
List<int> wrongHeadings = new List<int>();
List<int> repeatedHeadings = new List<int>();
// First pass: match every heading of the section to a template column by its name.
for (int c = band.Start; c <= band.End; c++)
{
string key = HeadingKey(sheet.Cell(EmployeeBulkUploadTemplate.HeadingRow, c).GetString());
EmployeeBulkUploadColumn definition;
if (key.Length == 0 || !byHeading.TryGetValue(key, out definition))
{
wrongHeadings.Add(c);
}
else if (matchedAt.ContainsKey(key))
{
repeatedHeadings.Add(c);
}
else
{
matchedAt[key] = c;
upload.ColumnMap[c] = definition;
}
}
List<EmployeeBulkUploadColumn> missing = section.Columns
.Where(x => !matchedAt.ContainsKey(HeadingKey(x.Label)))
.ToList();
if (missing.Count > 0)
{
missingColumns.Add(section.Name + " — missing " + Count(missing.Count, "column", "columns") + ": "
+ string.Join(", ", missing.Select(x => "\"" + x.Label + "\"")));
}
// Second pass: explain each wrong heading; when the template column that belongs at
// the same place is missing, that is almost always the one that was renamed.
foreach (int c in wrongHeadings.Concat(repeatedHeadings).OrderBy(x => x))
{
string heading = sheet.Cell(EmployeeBulkUploadTemplate.HeadingRow, c).GetString();
int offset = c - band.Start;
EmployeeBulkUploadColumn expected = offset < section.Columns.Count
&& missing.Contains(section.Columns[offset]) ? section.Columns[offset] : null;
EmployeeBulkUploadColumn loose;
byLooseHeading.TryGetValue(EmployeeBulkUploadTemplate.NormalizeHeading(heading), out loose);
string text = section.Name + ", column " + ColumnLetter(c) + ": ";
if (heading.Trim().Length == 0)
{
text += "the heading is empty";
}
else if (repeatedHeadings.Contains(c))
{
text += "\"" + heading + "\" is already in column " + ColumnLetter(matchedAt[HeadingKey(heading)]);
}
else if (loose != null && missing.Contains(loose))
{
// The right column, but its name is spaced differently from the template's.
text += "\"" + heading + "\" is not written as in the template — it must be \""
+ loose.Label + "\" (same letters and spaces; capital/small letters and the \"*\" do not matter)";
expected = null;
}
else
{
text += "\"" + heading + "\" is not a column of this section (an extra or renamed column)";
}
if (expected != null)
{
text += " — it should be \"" + expected.Label + "\"";
}
headingProblems.Add(text);
}
}
if (missingColumns.Count > 0)
{
allColumns.Fail(Count(missingColumns.Count, "section does", "sections do") + " not have all of "
+ (missingColumns.Count == 1 ? "its" : "their") + " columns.",
missingColumns, "Put the missing columns back under their section — a column may be left empty, "
+ "but it must not be removed. " + templateHint);
}
else
{
allColumns.Pass();
}
if (headingProblems.Count > 0)
{
columnNames.Fail(Count(headingProblems.Count, "column heading does", "column headings do")
+ " not match the template.",
headingProblems, "Type the column name as in the template (capital/small letters and the * do not matter), or copy it from a freshly "
+ "downloaded template. Do not rename, add or copy columns. " + templateHint);
}
else
{
columnNames.Pass();
}
}
#endregion
/// <summary>
/// A column heading as phase 2 compares it: its name letter for letter and space for space,
/// ignoring capital/small letters, the spaces around it and a mandatory "*" — "Employee No *",
/// "employee no" and "EMPLOYEE NO*" are the same column; "EmployeeNo" is not.
/// </summary>
private static string HeadingKey(string heading)
{
return (heading ?? string.Empty).Trim().TrimEnd('*').Trim().ToLowerInvariant();
}
#region Phase 3 — rows
/// <summary>
/// Runs only on a file that passed phase 2 (<paramref name="upload"/> is null otherwise,
/// and every check is left not run). Fills result.Groups, result.Rows and result.RowLabels.
/// </summary>
private void CheckRows(EmployeeBulkUploadLayer layer, EmployeeBulkUploadReadResult result, UploadFile upload)
{
EmployeeBulkUploadCheck hasRows = layer.AddCheck("HasRows", "The file holds at least one employee row");
EmployeeBulkUploadCheck rowLimit = layer.AddCheck("RowLimit",
"The file holds at most " + EmployeeBulkUploadTemplate.MaxUploadRows + " employees");
if (upload == null)
{
layer.SkipRemaining("Not checked — no column of the file could be matched to the template.");
return;
}
List<EmployeeBulkUploadRow> rows = ReadUploadRows(upload.Sheet, upload.ColumnMap);
result.RowCount = rows.Count;
if (rows.Count == 0)
{
hasRows.Fail("No employee data was found from row " + EmployeeBulkUploadTemplate.FirstDataRow + " onwards.",
null, "Enter one row per employee under the headings, starting at row "
+ EmployeeBulkUploadTemplate.FirstDataRow + ".");
}
else
{
hasRows.Pass();
}
if (rows.Count > EmployeeBulkUploadTemplate.MaxUploadRows)
{
rowLimit.Fail("The file holds " + rows.Count + " employees; one upload takes at most "
+ EmployeeBulkUploadTemplate.MaxUploadRows + ".",
null, "Split the employees into files of up to " + EmployeeBulkUploadTemplate.MaxUploadRows
+ " and upload them one by one.");
}
else
{
rowLimit.Pass();
}
result.Groups = EmployeeBulkUploadTemplate.GetGroups(upload.Sections.Select(x => x.Section.Name).ToList());
result.Rows = rows;
foreach (KeyValuePair<int, EmployeeBulkUploadColumn> entry in upload.ColumnMap)
{
result.SheetColumns[entry.Value.Key] = entry.Key;
}
// Phase 2 passed, so the Employee section is there: name each row after its employee.
EmployeeBulkUploadSubGroup employeeSection = result.Groups.SelectMany(g => g.SubGroups)
.First(s => s.Name == EmployeeBulkUploadTemplate.AlwaysIncludedSubGroup);
string employeeNoKey = employeeSection.Columns[0].Key;
string employeeNameKey = employeeSection.Columns[1].Key;
foreach (EmployeeBulkUploadRow row in rows)
{
result.RowLabels[row.RowNo] = (ValueOf(row, employeeNoKey).Trim() + " " + ValueOf(row, employeeNameKey).Trim()).Trim();
}
}
/// <summary>
/// The rows as far as the file's columns could be matched — only so the page can show
/// what the file holds when phase 2 failed. Nothing is checked on them.
/// </summary>
private void ReadRowsForDisplay(EmployeeBulkUploadReadResult result, UploadFile upload)
{
if (upload.Sheet == null || upload.ColumnMap.Count == 0)
{
return;
}
result.Rows = ReadUploadRows(upload.Sheet, upload.ColumnMap);
result.RowCount = result.Rows.Count;
result.Groups = EmployeeBulkUploadTemplate.GetGroups(upload.Sections.Select(x => x.Section.Name).ToList());
foreach (KeyValuePair<int, EmployeeBulkUploadColumn> entry in upload.ColumnMap)
{
result.SheetColumns[entry.Value.Key] = entry.Key;
}
}
/// <summary>
/// One employee, one row: an employee's children, education and the other repeated data go
/// in the numbered blocks of that row. Employee Nos are compared ignoring case and spacing,
/// so "e001" and "E001 " are the same employee. Every row of a repeated number gets its own
/// error naming the other rows.
/// </summary>
private void CheckOneRowPerEmployee(EmployeeBulkUploadCheck check, EmployeeBulkUploadReadResult result,
EmployeeBulkUploadGroup employeeGroup, EmployeeBulkUploadSubGroup employeeSection, string employeeNoKey,
string employeeNameKey)
{
List<List<EmployeeBulkUploadRow>> repeated = result.Rows
.Where(r => ValueOf(r, employeeNoKey).Trim().Length > 0)
.GroupBy(r => MasterKey(ValueOf(r, employeeNoKey)))
.Where(g => g.Count() > 1)
.Select(g => g.ToList())
.ToList();
if (repeated.Count == 0)
{
check.Pass();
return;
}
foreach (List<EmployeeBulkUploadRow> rows in repeated)
{
foreach (EmployeeBulkUploadRow row in rows)
{
AddRowIssue(result, row, employeeGroup, employeeSection, employeeSection.Columns[0].Label,
ValueOf(row, employeeNoKey),
"Employee No \"" + ValueOf(row, employeeNoKey).Trim() + "\" is in " + rows.Count + " rows — this one and "
+ string.Join(", ", rows.Where(x => x != row).Select(x => RowLabel(result, x.RowNo)))
+ ". One employee may have one row only: put all of this employee's data in one row and "
+ "delete the others.",
employeeNoKey, employeeNameKey);
}
}
check.Fail(Count(repeated.Count, "employee is", "employees are") + " in more than one row ("
+ repeated.Sum(x => x.Count) + " rows in all). Each row is listed in the Problems table below.",
repeated.Select(rows => "Employee No \"" + ValueOf(rows[0], employeeNoKey).Trim() + "\": "
+ string.Join(", ", rows.Select(x => RowLabel(result, x.RowNo)))),
"Keep exactly one row per employee. Children, spouse, education, nominee, experience and training "
+ "go in the numbered blocks (Children 01, Children 02, ...) of that one row — not in extra rows.");
}
/// <summary>
/// The same person under two Employee Nos: two rows with one National ID No. Checked only
/// when Extended Personal Information is in the file; the ID is compared on its letters and
/// digits, so spaces and dashes do not hide a repeat.
/// </summary>
private void CheckOneRowPerPerson(EmployeeBulkUploadCheck check, EmployeeBulkUploadReadResult result,
List<EmployeeBulkUploadSubGroup> sections, string employeeNoKey, string employeeNameKey)
{
EmployeeBulkUploadSubGroup extended = sections.FirstOrDefault(s => s.Name == "Extended Personal Information");
if (extended == null)
{
check.Pass();
return;
}
EmployeeBulkUploadColumn nationalID = extended.Columns.First(c => c.Label == "National ID No");
EmployeeBulkUploadGroup group = result.Groups.First(g => g.SubGroups.Contains(extended));
List<List<EmployeeBulkUploadRow>> repeated = result.Rows
.Select(r => new { Row = r, Key = new string(ValueOf(r, nationalID.Key).Where(char.IsLetterOrDigit).ToArray()).ToUpperInvariant() })
.Where(x => x.Key.Length > 0)
.GroupBy(x => x.Key, x => x.Row)
// Rows that also share the Employee No are already reported as one employee in two rows.
.Where(g => g.Select(r => MasterKey(ValueOf(r, employeeNoKey))).Distinct().Count() > 1)
.Select(g => g.ToList())
.ToList();
if (repeated.Count == 0)
{
check.Pass();
return;
}
foreach (List<EmployeeBulkUploadRow> rows in repeated)
{
foreach (EmployeeBulkUploadRow row in rows)
{
AddRowIssue(result, row, group, extended, nationalID.Label, ValueOf(row, nationalID.Key),
"National ID No \"" + ValueOf(row, nationalID.Key).Trim() + "\" is also in "
+ string.Join(", ", rows.Where(x => x != row).Select(x => RowLabel(result, x.RowNo)))
+ " under another Employee No. One person can be one employee only — keep one row, or "
+ "correct the National ID No.",
employeeNoKey, employeeNameKey);
}
}
check.Fail(Count(repeated.Count, "National ID No is", "National ID Nos are")
+ " in rows of different Employee Nos. Each row is listed in the Problems table below.",
repeated.Select(rows => "National ID No \"" + ValueOf(rows[0], nationalID.Key).Trim() + "\": "
+ string.Join(", ", rows.Select(x => RowLabel(result, x.RowNo)))),
"Keep one row per person. If two different people really were given the same National ID No, "
+ "correct the wrong one.");
}
/// <summary>Every data row that has at least one non-empty mapped cell.</summary>
private List<EmployeeBulkUploadRow> ReadUploadRows(IXLWorksheet sheet,
Dictionary<int, EmployeeBulkUploadColumn> columnMap)
{
List<EmployeeBulkUploadRow> rows = new List<EmployeeBulkUploadRow>();
IXLRow lastUsed = sheet.LastRowUsed();
int lastRow = lastUsed == null ? 0 : lastUsed.RowNumber();
for (int row = EmployeeBulkUploadTemplate.FirstDataRow; row <= lastRow; row++)
{
EmployeeBulkUploadRow item = new EmployeeBulkUploadRow();
item.RowNo = row;
foreach (KeyValuePair<int, EmployeeBulkUploadColumn> entry in columnMap)
{
string text = CellText(sheet.Cell(row, entry.Key));
if (text.Length > 0)
{
item.Values[entry.Value.Key] = text;
}
}
if (item.Values.Count > 0)
{
rows.Add(item);
}
}
return rows;
}
#endregion
/// <summary>
/// Puts the chosen existing record into every cell that holds a replaced value: the same
/// table, the same value (case and spacing aside) and — for Thana / Education Level — the same
/// parent value in the block. The file is not touched; only the rows read from it change.
/// A replacement whose record is gone or inactive is reported and left out. Each replaced
/// cell is listed in result.ReplacedCells and as a warning in the Problems grid, so the
/// user sees what the file had and what is used instead.
/// </summary>
private void ApplyReplacements(EmployeeBulkUploadReadResult result, List<EmployeeBulkUploadReplacement> replacements,
int payrollTypeID)
{
if (replacements == null || replacements.Count == 0)
{
return;
}
List<EmployeeBulkUploadSubGroup> sections = result.Groups.SelectMany(g => g.SubGroups).ToList();
EmployeeBulkUploadSubGroup employeeSection = sections.FirstOrDefault(s => s.Name == EmployeeBulkUploadTemplate.AlwaysIncludedSubGroup);
if (employeeSection == null)
{
return;
}
string employeeNoKey = employeeSection.Columns[0].Key;
string employeeNameKey = employeeSection.Columns[1].Key;
Dictionary<EnumBulkUploadMasterData, List<MasterRecord>> tables = new Dictionary<EnumBulkUploadMasterData, List<MasterRecord>>();
foreach (EmployeeBulkUploadReplacement replacement in replacements.Where(x => x != null
&& x.MasterData != EnumBulkUploadMasterData.None && x.MasterData != EnumBulkUploadMasterData.Employee
&& x.MasterData != EnumBulkUploadMasterData.Division && !string.IsNullOrWhiteSpace(x.Value) && x.ReplacementID > 0)
// A Thana / Education Level is found by the parent value the file has, so it is replaced
// before that parent (District / Education Platform) is: a pair chosen together still meets.
.OrderBy(x => MatchesWithinParent(x.MasterData) ? 0 : 1))
{
List<MasterRecord> records;
if (!tables.TryGetValue(replacement.MasterData, out records))
{
records = LoadMaster(replacement.MasterData, payrollTypeID);
tables[replacement.MasterData] = records;
}
string masterName = EmployeeBulkUploadMasterInfo.DisplayName(replacement.MasterData);
MasterRecord record = records.FirstOrDefault(x => x.ID == replacement.ReplacementID);
if (record == null || record.IsInactive || string.IsNullOrWhiteSpace(record.Name))
{
result.Issues.Add(new EmployeeBulkUploadIssue
{
Severity = EmployeeBulkUploadIssue.Warning,
RowNo = 0,
EmployeeNo = string.Empty,
EmployeeName = string.Empty,
Group = "File",
SubGroup = "File",
Column = masterName,
Value = replacement.Value,
Message = "The " + masterName + " chosen to use instead of \"" + replacement.Value.Trim()
+ "\" is no longer in the system (or is inactive), so the file's value is kept. Choose again."
});
continue;
}
string to = record.Name.Trim();
string valueKey = MasterKey(replacement.Value);
bool withinParent = MatchesWithinParent(replacement.MasterData);
string parentKey = withinParent && !string.IsNullOrWhiteSpace(replacement.ParentValue)
? MasterKey(replacement.ParentValue) : null;
foreach (EmployeeBulkUploadGroup group in result.Groups)
{
foreach (EmployeeBulkUploadSubGroup section in group.SubGroups)
{
foreach (EmployeeBulkUploadColumn column in section.Columns.Where(c => c.MasterData == replacement.MasterData))
{
EmployeeBulkUploadColumn parentColumn = withinParent ? ParentColumn(section, column) : null;
foreach (EmployeeBulkUploadRow row in result.Rows)
{
string value;
if (!row.Values.TryGetValue(column.Key, out value) || MasterKey(value) != valueKey)
{
continue;
}
if (parentKey != null && (parentColumn == null || MasterKey(ValueOf(row, parentColumn.Key)) != parentKey))
{
continue;
}
row.Values[column.Key] = to;
result.ReplacedCells.Add(new EmployeeBulkUploadReplacedCell
{
RowNo = row.RowNo,
ColumnKey = column.Key,
SubGroup = section.Name,
Column = column.Label,
MasterData = replacement.MasterData,
From = value,
To = to
});
result.Issues.Add(new EmployeeBulkUploadIssue
{
Severity = EmployeeBulkUploadIssue.Warning,
RowNo = row.RowNo,
EmployeeNo = ValueOf(row, employeeNoKey),
EmployeeName = ValueOf(row, employeeNameKey),
Group = group.Name,
SubGroup = section.Name,
Column = column.Label,
Value = to,
Message = "The file has \"" + value.Trim() + "\"; the existing " + masterName + " \"" + to
+ "\" chosen in the Master data dialog is used instead. The file itself is not changed."
});
}
}
}
}
}
}
#region Phases 4 and 5 — Employee No and Name, mandatory values
/// <summary>
/// Phase 4 (<paramref name="identityOnly"/>): Employee No and Employee Name in every row,
/// and each Employee No in one row only. Phase 5: every ticked section is filled in every
/// row — a section without mandatory columns needs at least one value, a repeat group
/// (Spouse 01, Spouse 02, ...) at least one filled block — and every mandatory column has a
/// value in every row of its section, except in a repeat block left wholly empty while
/// another block of its group is filled. Each runs only when the phase before it passed.
/// Every problem becomes one row of result.Issues; each check's details sum them up per column.
/// </summary>
private void CheckRequiredValues(EmployeeBulkUploadLayer layer, EmployeeBulkUploadReadResult result, bool canRun,
bool identityOnly)
{
EmployeeBulkUploadCheck identity = identityOnly
? layer.AddCheck("EmployeeNoAndName", "Every row has an Employee No and an Employee Name")
: null;
EmployeeBulkUploadCheck onePerEmployee = identityOnly
? layer.AddCheck("OneRowPerEmployee", "Each employee has one row only — no Employee No is in two or more rows")
: null;
EmployeeBulkUploadCheck filled = identityOnly
? null
: layer.AddCheck("SectionsFilled",
"Every ticked section is filled in every row — for Children, Spouse, Education, Nominee, "
+ "Experience and Training at least one of its blocks");
EmployeeBulkUploadCheck mandatory = identityOnly
? null
: layer.AddCheck("MandatoryValues",
"Every mandatory column (marked *) of the ticked sections has a value in every row");
if (!canRun)
{
layer.SkipRemaining("Not checked — the file holds no employee rows that could be read.");
return;
}
EmployeeBulkUploadSubGroup employeeSection = result.Groups
.SelectMany(g => g.SubGroups)
.First(s => s.Name == EmployeeBulkUploadTemplate.AlwaysIncludedSubGroup);
string employeeNoKey = employeeSection.Columns[0].Key;
string employeeNameKey = employeeSection.Columns[1].Key;
// (section, column) → Excel rows where the value is missing, in first-seen order.
List<KeyValuePair<string, List<int>>> identityMisses = new List<KeyValuePair<string, List<int>>>();
List<KeyValuePair<string, List<int>>> mandatoryMisses = new List<KeyValuePair<string, List<int>>>();
List<KeyValuePair<string, List<int>>> emptyMisses = new List<KeyValuePair<string, List<int>>>();
foreach (EmployeeBulkUploadRow row in result.Rows)
{
foreach (EmployeeBulkUploadGroup group in result.Groups)
{
if (!identityOnly)
{
CheckSectionsFilled(result, row, group, employeeSection, employeeNoKey, employeeNameKey, emptyMisses);
}
foreach (EmployeeBulkUploadSubGroup section in group.SubGroups)
{
bool isIdentity = section == employeeSection;
if (isIdentity != identityOnly)
{
continue;
}
if (section.IsRepeatBlock && !section.Columns.Any(c => row.Values.ContainsKey(c.Key)))
{
continue;
}
foreach (EmployeeBulkUploadColumn column in section.Columns.Where(c => c.IsMandatory))
{
if (row.Values.ContainsKey(column.Key))
{
continue;
}
string message;
if (isIdentity)
{
message = column.Label + " is empty. Every row needs its Employee No and Employee Name "
+ "— they tie the row to an employee.";
}
else if (section.IsRepeatBlock)
{
message = "\"" + column.Label + "\" is mandatory in " + section.Name + " and is empty. "
+ "This row has other " + section.Name + " data, so fill it in — or leave the whole "
+ section.Name + " block empty when another " + group.Name + " block of this row is filled.";
}
else
{
message = "\"" + column.Label + "\" is mandatory in " + section.Name + " and is empty.";
}
result.Issues.Add(new EmployeeBulkUploadIssue
{
Severity = EmployeeBulkUploadIssue.Error,
RowNo = row.RowNo,
EmployeeNo = ValueOf(row, employeeNoKey),
EmployeeName = ValueOf(row, employeeNameKey),
Group = group.Name,
SubGroup = section.Name,
Column = column.Label,
Value = string.Empty,
Message = message
});
AddMiss(isIdentity ? identityMisses : mandatoryMisses,
section.Name + " — \"" + column.Label + "\"", row.RowNo);
}
}
}
}
string tableHint = " Each empty cell is listed in the Problems table below.";
if (identityOnly)
{
if (identityMisses.Count > 0)
{
identity.Fail(MissSummary(identityMisses) + tableHint, MissDetails(result, identityMisses),
"Fill in Employee No and Employee Name in every row that holds data, or delete rows that "
+ "do not belong to an employee.");
}
else
{
identity.Pass();
}
EmployeeBulkUploadGroup employeeGroup = result.Groups.First(g => g.SubGroups.Contains(employeeSection));
CheckOneRowPerEmployee(onePerEmployee, result, employeeGroup, employeeSection, employeeNoKey, employeeNameKey);
return;
}
if (emptyMisses.Count > 0)
{
filled.Fail(Count(emptyMisses.Sum(x => x.Value.Count), "ticked section is", "ticked sections are")
+ " empty in " + Count(emptyMisses.SelectMany(x => x.Value).Distinct().Count(), "row", "rows")
+ ". Each one is listed in the Problems table below.",
MissDetails(result, emptyMisses),
"Every employee in the file needs data in every ticked section — for Children, Spouse, Education, "
+ "Nominee, Experience and Training at least one block (e.g. Children 01). Fill it in, take the "
+ "employee out of this file, or untick the section and download the template again.");
}
else
{
filled.Pass();
}
if (mandatoryMisses.Count > 0)
{
mandatory.Fail(MissSummary(mandatoryMisses) + tableHint, MissDetails(result, mandatoryMisses),
"Fill in every column marked * in the sections you ticked. For Children, Spouse, Education, "
+ "Nominee, Experience and Training, every block that is filled needs all its * columns; the other "
+ "blocks of the group may be left wholly empty.");
}
else
{
mandatory.Pass();
}
}
/// <summary>
/// One row, one header group: a repeat group (Spouse 01, Spouse 02, ...) with none of its
/// blocks filled is one problem; so is a section without mandatory columns with no value at
/// all (a section with mandatory columns reports each empty * cell instead). The problem is
/// placed on the section's first mandatory column, or its first column.
/// </summary>
private void CheckSectionsFilled(EmployeeBulkUploadReadResult result, EmployeeBulkUploadRow row,
EmployeeBulkUploadGroup group, EmployeeBulkUploadSubGroup employeeSection, string employeeNoKey,
string employeeNameKey, List<KeyValuePair<string, List<int>>> emptyMisses)
{
Func<EmployeeBulkUploadSubGroup, bool> hasData = s => s.Columns.Any(c => row.Values.ContainsKey(c.Key));
Action<EmployeeBulkUploadSubGroup, string, string> add = (section, where, message) =>
{
EmployeeBulkUploadColumn column = section.Columns.FirstOrDefault(c => c.IsMandatory) ?? section.Columns[0];
result.Issues.Add(new EmployeeBulkUploadIssue
{
Severity = EmployeeBulkUploadIssue.Error,
RowNo = row.RowNo,
EmployeeNo = ValueOf(row, employeeNoKey),
EmployeeName = ValueOf(row, employeeNameKey),
Group = group.Name,
SubGroup = section.Name,
Column = column.Label,
Value = string.Empty,
Message = message
});
AddMiss(emptyMisses, where, row.RowNo);
};
List<EmployeeBulkUploadSubGroup> blocks = group.SubGroups.Where(s => s.IsRepeatBlock).ToList();
if (blocks.Count > 0 && !blocks.Any(hasData))
{
string names = blocks.Count == 2
? "neither " + blocks[0].Name + " nor " + blocks[1].Name
: "none of " + string.Join(", ", blocks.Take(blocks.Count - 1).Select(s => s.Name)) + " or " + blocks.Last().Name;
add(blocks[0], group.Name + " — no block filled",
group.Name + " is ticked, but this row fills " + names + ". Every employee in the file needs at least "
+ "one " + group.Name + " block filled (e.g. " + blocks[0].Name + ") — fill one in, or take this employee "
+ "out of the file.");
}
foreach (EmployeeBulkUploadSubGroup section in group.SubGroups.Where(s => !s.IsRepeatBlock && s != employeeSection
&& !s.Columns.Any(c => c.IsMandatory) && !hasData(s)))
{
add(section, section.Name + " — empty",
section.Name + " is ticked, but this row has nothing in it. Every employee in the file needs at least "
+ "one " + section.Name + " value — fill it in, or take this employee out of the file.");
}
}
private string ValueOf(EmployeeBulkUploadRow row, string key)
{
string value;
return row.Values.TryGetValue(key, out value) ? value : string.Empty;
}
private void AddMiss(List<KeyValuePair<string, List<int>>> misses, string where, int rowNo)
{
int index = misses.FindIndex(x => x.Key == where);
if (index < 0)
{
misses.Add(new KeyValuePair<string, List<int>>(where, new List<int> { rowNo }));
}
else
{
misses[index].Value.Add(rowNo);
}
}
/// <summary>"5 empty cells in 3 rows."</summary>
private string MissSummary(List<KeyValuePair<string, List<int>>> misses, string what = "empty")
{
int cells = misses.Sum(x => x.Value.Count);
int rows = misses.SelectMany(x => x.Value).Distinct().Count();
return Count(cells, what + " cell", what + " cells") + " in " + Count(rows, "row", "rows") + ".";
}
/// <summary>
/// One line per column, naming each row with its employee:
/// General — "Date of Birth": empty in 2 rows (row 5 · E001 Rahim, row 9 · E004 Karim).
/// </summary>
private IEnumerable<string> MissDetails(EmployeeBulkUploadReadResult result,
List<KeyValuePair<string, List<int>>> misses, string what = "empty")
{
const int shownRows = 10;
return misses.Select(x => x.Key + ": " + what + " in " + Count(x.Value.Count, "row", "rows") + " ("
+ string.Join(", ", x.Value.Take(shownRows).Select(rowNo => RowLabel(result, rowNo)))
+ (x.Value.Count > shownRows ? " and " + (x.Value.Count - shownRows) + " more" : string.Empty)
+ ")");
}
/// <summary>The sheet column of an issue's cell; 0 when it is not a template column.</summary>
private static int SheetColumnOf(EmployeeBulkUploadReadResult result, EmployeeBulkUploadIssue issue)
{
EmployeeBulkUploadColumn column = result.Groups
.SelectMany(g => g.SubGroups)
.Where(s => s.Name == issue.SubGroup)
.SelectMany(s => s.Columns)
.FirstOrDefault(c => c.Label == issue.Column);
int sheetColumn;
return column != null && result.SheetColumns.TryGetValue(column.Key, out sheetColumn) ? sheetColumn : 0;
}
/// <summary>The issue's Excel cell, e.g. "AB12"; just the row ("Row 12") when the column is not known.</summary>
private static string CellOf(EmployeeBulkUploadReadResult result, EmployeeBulkUploadIssue issue)
{
int sheetColumn = SheetColumnOf(result, issue);
return sheetColumn > 0
? XLHelper.GetColumnLetterFromNumber(sheetColumn) + issue.RowNo
: "Row " + issue.RowNo;
}
/// <summary>"row 5 · E001 Rahim" — the Excel row with its employee, as far as the file names one.</summary>
private static string RowLabel(EmployeeBulkUploadReadResult result, int rowNo)
{
string who;
return result.RowLabels.TryGetValue(rowNo, out who) && who.Length > 0
? "row " + rowNo + " · " + who
: "row " + rowNo;
}
#endregion
#region Phase 6 — data types
private const int MinYear = 1900;
private const int MaxYear = 2100;
/// <summary>The template's date format (DD-MMM-YYYY) as .NET reads it.</summary>
private static readonly string[] TemplateDateFormats = { "d-MMM-yyyy", "dd-MMM-yyyy" };
/// <summary>
/// Other ways people write dates. Used only to suggest the intended date in the message —
/// a value in one of these is still wrong, since 05/06/1990 reads differently on
/// different computers.
/// </summary>
private static readonly string[][] OtherDateFormats =
{
new[] { "d/M/yyyy", "d-M-yyyy", "d.M.yyyy" },
new[] { "M/d/yyyy", "M-d-yyyy", "M.d.yyyy" },
new[] { "yyyy-M-d", "yyyy/M/d", "yyyy.M.d" },
new[] { "d MMM yyyy", "d MMMM yyyy", "d-MMMM-yyyy", "MMM d, yyyy", "MMMM d, yyyy", "MMM d yyyy", "MMMM d yyyy" }
};
/// <summary>The kinds of value phase 6 checks; one check each, in this order.</summary>
private enum TypeRule
{
ErrorCell,
Date,
WholeNumber,
Decimal,
Percentage,
Email,
Mobile,
List
}
/// <summary>
/// Runs only when phase 5 passed (<paramref name="upload"/> is null otherwise). Reads each
/// filled cell's real Excel value — not its text — so a date typed as text, a number in a
/// date column or a formula error is told apart. Every wrong cell becomes one row of
/// result.Issues; each check's details sum them up per column.
/// </summary>
private void CheckDataTypes(EmployeeBulkUploadLayer layer, EmployeeBulkUploadReadResult result, UploadFile upload)
{
Dictionary<TypeRule, EmployeeBulkUploadCheck> checks = new Dictionary<TypeRule, EmployeeBulkUploadCheck>
{
{ TypeRule.ErrorCell, layer.AddCheck("NoErrorCells",
"No cell shows an Excel error such as #N/A, #VALUE! or #REF!") },
{ TypeRule.Date, layer.AddCheck("DateValues",
"Every date column holds a real date in the DD-MMM-YYYY format (e.g. 15-Jan-1990), between "
+ MinYear + " and " + MaxYear) },
{ TypeRule.WholeNumber, layer.AddCheck("WholeNumberValues",
"Every whole-number column (Passing Year, Months, Days, ...) holds a whole number, 0 or more") },
{ TypeRule.Decimal, layer.AddCheck("DecimalValues",
"Every number column (GPA / Marks, Training Fees, ...) holds a number, 0 or more") },
{ TypeRule.Percentage, layer.AddCheck("PercentageValues",
"Every percentage column holds a number from 0 to 100") },
{ TypeRule.Email, layer.AddCheck("EmailValues",
"Every e-mail column holds a valid e-mail address (e.g. name@company.com)") },
{ TypeRule.Mobile, layer.AddCheck("MobileValues",
"Every mobile-number column holds a Bangladesh mobile number (e.g. 01712345678)") },
{ TypeRule.List, layer.AddCheck("ListValues",
"Every dropdown column (Gender, Blood Group, Marital Status, Tax Circle, Sponsored Type) holds one of its listed values") }
};
if (upload == null)
{
layer.SkipRemaining("Not checked — the file holds no employee rows that could be read.");
return;
}
Dictionary<string, EmployeeBulkUploadGroup> groupOf = new Dictionary<string, EmployeeBulkUploadGroup>();
Dictionary<string, EmployeeBulkUploadSubGroup> sectionOf = new Dictionary<string, EmployeeBulkUploadSubGroup>();
foreach (EmployeeBulkUploadGroup group in result.Groups)
{
foreach (EmployeeBulkUploadSubGroup section in group.SubGroups)
{
foreach (EmployeeBulkUploadColumn column in section.Columns)
{
groupOf[column.Key] = group;
sectionOf[column.Key] = section;
}
}
}
EmployeeBulkUploadSubGroup employeeSection = sectionOf.Values
.First(s => s.Name == EmployeeBulkUploadTemplate.AlwaysIncludedSubGroup);
string employeeNoKey = employeeSection.Columns[0].Key;
string employeeNameKey = employeeSection.Columns[1].Key;
Dictionary<TypeRule, List<KeyValuePair<string, List<int>>>> misses = checks.Keys
.ToDictionary(x => x, x => new List<KeyValuePair<string, List<int>>>());
foreach (EmployeeBulkUploadRow row in result.Rows)
{
foreach (KeyValuePair<int, EmployeeBulkUploadColumn> entry in upload.ColumnMap.OrderBy(x => x.Key))
{
EmployeeBulkUploadColumn column = entry.Value;
string shown;
if (!row.Values.TryGetValue(column.Key, out shown))
{
continue; // empty cells are phases 4 and 5's business
}
string text = shown;
// A number cell shows what its format makes of it — 2010.5 under "0" shows 2010 —
// so messages quote the number itself.
IXLCell cell = upload.Sheet.Cell(row.RowNo, entry.Key);
if (cell.Value.IsNumber)
{
shown = cell.Value.GetNumber().ToString(CultureInfo.InvariantCulture);
}
TypeRule rule;
string problem = CheckCellType(cell, column, shown, out rule);
if (problem == null)
{
row.Parsed[column.Key] = TypedValue(cell, column, text);
continue;
}
EmployeeBulkUploadSubGroup section = sectionOf[column.Key];
result.Issues.Add(new EmployeeBulkUploadIssue
{
Severity = EmployeeBulkUploadIssue.Error,
RowNo = row.RowNo,
EmployeeNo = ValueOf(row, employeeNoKey),
EmployeeName = ValueOf(row, employeeNameKey),
Group = groupOf[column.Key].Name,
SubGroup = section.Name,
Column = column.Label,
Value = shown,
Message = problem
});
AddMiss(misses[rule], section.Name + " — \"" + column.Label + "\"", row.RowNo);
}
}
string tableHint = " Each wrong cell is listed, with what is wrong with it, in the Problems table below.";
foreach (KeyValuePair<TypeRule, EmployeeBulkUploadCheck> pair in checks)
{
List<KeyValuePair<string, List<int>>> found = misses[pair.Key];
if (found.Count == 0)
{
pair.Value.Pass();
}
else
{
pair.Value.Fail(MissSummary(found, "wrong") + tableHint, MissDetails(result, found, "wrong"), TypeFix(pair.Key));
}
}
}
/// <summary>
/// What is wrong with one filled cell, or null when it holds the kind of value its column
/// takes. <paramref name="shown"/> is the cell as the user sees it, for the message.
/// </summary>
private string CheckCellType(IXLCell cell, EmployeeBulkUploadColumn column, string shown, out TypeRule rule)
{
XLCellValue value = cell.Value;
string label = "\"" + column.Label + "\"";
string quoted = "\"" + shown + "\"";
if (value.IsError)
{
rule = TypeRule.ErrorCell;
return "The cell shows the Excel error " + shown + " — usually a formula that could not work out "
+ "its value. Type the value itself instead.";
}
switch (column.Kind)
{
case EnumBulkUploadColumnKind.Date:
rule = TypeRule.Date;
return CheckDate(value, label, quoted);
case EnumBulkUploadColumnKind.Integer:
{
rule = TypeRule.WholeNumber;
string example = column.Label.IndexOf("Year", StringComparison.OrdinalIgnoreCase) >= 0 ? "2015" : "12";
decimal number;
string notNumber = ReadNumber(value, label, quoted, "a whole number, e.g. " + example, false, out number);
if (notNumber != null)
{
return notNumber;
}
if (number != decimal.Truncate(number))
{
return quoted + " has a fraction. " + label + " takes a whole number, e.g. " + example + ".";
}
if (number < 0)
{
return quoted + " is negative. " + label + " takes a whole number of 0 or more.";
}
return null;
}
case EnumBulkUploadColumnKind.Decimal:
{
rule = TypeRule.Decimal;
decimal number;
string notNumber = ReadNumber(value, label, quoted, "a number, e.g. 3.75 or 15000", false, out number);
if (notNumber != null)
{
return notNumber;
}
if (number < 0)
{
return quoted + " is negative. " + label + " takes a number of 0 or more.";
}
return null;
}
case EnumBulkUploadColumnKind.Percentage:
{
rule = TypeRule.Percentage;
decimal number;
string notNumber = ReadNumber(value, label, quoted, "a number from 0 to 100, e.g. 50", true, out number);
if (notNumber != null)
{
return notNumber;
}
// A cell formatted as % holds 0.5 for 50%.
if (value.IsNumber && (cell.Style.NumberFormat.Format ?? string.Empty).Contains("%"))
{
number *= 100;
}
if (number < 0 || number > 100)
{
return quoted + " is outside 0 to 100. " + label + " takes the share as a number from 0 to 100, "
+ "e.g. 50 for half.";
}
return null;
}
case EnumBulkUploadColumnKind.Email:
rule = TypeRule.Email;
if (!value.IsText)
{
return quoted + " is not an e-mail address. " + label + " takes an address such as name@company.com.";
}
string email = value.GetText().Trim();
if (email.Any(char.IsWhiteSpace))
{
return quoted + " has a space in it. " + label + " takes a single address such as name@company.com, "
+ "without spaces.";
}
// The pattern the Employee Profile form checks e-mail addresses with.
if (!System.Text.RegularExpressions.Regex.IsMatch(email, @"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$"))
{
return quoted + " is not a valid e-mail address. " + label + " takes an address such as "
+ "name@company.com — one @, then a domain with a dot.";
}
return null;
case EnumBulkUploadColumnKind.Mobile:
{
rule = TypeRule.Mobile;
// A number cell loses the leading 0 (1712345678); the shown text is what is checked.
string mobile = (value.IsText ? value.GetText() : shown).Trim();
if (!MobilePattern.IsMatch(mobile))
{
return quoted + " is not a Bangladesh mobile number. " + label + " takes 11 digits starting "
+ "with 01 (e.g. 01712345678), or the same number as 8801712345678 or +8801712345678"
+ (value.IsNumber ? " — type it in a text cell so the leading 0 is kept." : ".");
}
return null;
}
default:
rule = TypeRule.List;
string list = column.DropdownList;
if (list == null)
{
return null; // free text: any value is fine
}
string[] allowed = list.Split(',');
string text = value.IsText ? value.GetText().Trim() : shown;
if (!allowed.Contains(text, StringComparer.OrdinalIgnoreCase))
{
return quoted + " is not one of the values " + label + " takes: " + string.Join(", ", allowed)
+ ". Pick one from the cell's dropdown.";
}
return null;
}
}
/// <summary>The mobile-number pattern the Employee Profile form checks with.</summary>
private static readonly System.Text.RegularExpressions.Regex MobilePattern =
new System.Text.RegularExpressions.Regex(@"^(?:\+8801[3-9]\d{8}|01[3-9]\d{8}|8801[3-9]\d{8})$");
/// <summary>
/// The value of a cell that passed its type check, as the save needs it: a DateTime for a
/// date, a decimal for a number (a percentage as 0–100), the listed spelling for a
/// dropdown value, and the trimmed text otherwise. <paramref name="text"/> is the cell as
/// the user sees it — a phone number keeps its leading zero.
/// </summary>
private object TypedValue(IXLCell cell, EmployeeBulkUploadColumn column, string text)
{
XLCellValue value = cell.Value;
switch (column.Kind)
{
case EnumBulkUploadColumnKind.Date:
{
if (value.IsDateTime)
{
return value.GetDateTime().Date;
}
return DateTime.ParseExact(value.GetText().Trim(), TemplateDateFormats, CultureInfo.InvariantCulture,
DateTimeStyles.None).Date;
}
case EnumBulkUploadColumnKind.Integer:
case EnumBulkUploadColumnKind.Decimal:
case EnumBulkUploadColumnKind.Percentage:
{
decimal number;
if (value.IsNumber)
{
number = (decimal)value.GetNumber();
if (column.Kind == EnumBulkUploadColumnKind.Percentage
&& (cell.Style.NumberFormat.Format ?? string.Empty).Contains("%"))
{
number *= 100;
}
}
else
{
number = decimal.Parse(value.GetText().Trim().TrimEnd('%').Trim(), NumberStyles.Number,
CultureInfo.InvariantCulture);
}
return number;
}
default:
{
string list = column.DropdownList;
string trimmed = (text ?? string.Empty).Trim();
if (list != null)
{
return list.Split(',').First(x => string.Equals(x, trimmed, StringComparison.OrdinalIgnoreCase));
}
return trimmed;
}
}
}
/// <summary>Null when the cell holds a real date in the accepted years; otherwise why not.</summary>
private string CheckDate(XLCellValue value, string label, string quoted)
{
string takes = label + " takes a date in DD-MMM-YYYY, e.g. 15-Jan-1990.";
if (value.IsDateTime)
{
return YearProblem(value.GetDateTime(), quoted);
}
if (value.IsNumber)
{
return quoted + " is a number, not a date. " + takes
+ " If it is a date that Excel shows as a number, format the cell as a date.";
}
if (!value.IsText)
{
return quoted + " is not a date. " + takes;
}
string text = value.GetText().Trim();
DateTime date;
if (DateTime.TryParseExact(text, TemplateDateFormats, CultureInfo.InvariantCulture,
DateTimeStyles.None, out date))
{
return YearProblem(date, quoted);
}
// Written the template's way, yet not a date: a wrong month name or a day that does not exist.
System.Text.RegularExpressions.Match parts =
System.Text.RegularExpressions.Regex.Match(text, @"^(\d{1,2})-([A-Za-z]+)-(\d{4})$");
if (parts.Success)
{
string month = parts.Groups[2].Value;
if (!CultureInfo.InvariantCulture.DateTimeFormat.AbbreviatedMonthNames
.Any(x => x.Length > 0 && string.Equals(x, month, StringComparison.OrdinalIgnoreCase)))
{
return quoted + " has \"" + month + "\" as the month, which is not a month. Use the first three "
+ "letters: Jan, Feb, Mar, Apr, May, Jun, Jul, Aug, Sep, Oct, Nov, Dec.";
}
return quoted + " is not a real date — that month has no day " + parts.Groups[1].Value
+ ". Check the day and the month.";
}
// Another way of writing a date: say which date it most likely is, but still ask for DD-MMM-YYYY.
List<DateTime> readings = OtherDateFormats
.Select(formats => DateTime.TryParseExact(text, formats, CultureInfo.InvariantCulture,
DateTimeStyles.None, out date) ? (DateTime?)date : null)
.Where(x => x.HasValue)
.Select(x => x.Value)
.Distinct()
.ToList();
if (readings.Count == 1)
{
return quoted + " is not in the template's date format. Enter it as "
+ readings[0].ToString("dd-MMM-yyyy", CultureInfo.InvariantCulture) + " (DD-MMM-YYYY).";
}
if (readings.Count > 1)
{
return quoted + " is not in the template's date format, and it can be read as "
+ string.Join(" or ", readings.Select(x => x.ToString("dd-MMM-yyyy", CultureInfo.InvariantCulture)))
+ ". Enter it as DD-MMM-YYYY so there is no doubt.";
}
return quoted + " is not a date. " + takes;
}
private string YearProblem(DateTime date, string quoted)
{
if (date.Year < MinYear || date.Year > MaxYear)
{
return quoted + " is in the year " + date.Year + ", outside " + MinYear + " to " + MaxYear
+ ". Check the year.";
}
return null;
}
/// <summary>
/// Reads the cell as a number: a number cell, or text holding one ("1,250.50"; with
/// <paramref name="allowPercentSign"/>, also "50%"). Returns why not, or null.
/// </summary>
private string ReadNumber(XLCellValue value, string label, string quoted, string takes,
bool allowPercentSign, out decimal number)
{
number = 0;
if (value.IsNumber)
{
number = (decimal)value.GetNumber();
return null;
}
if (value.IsDateTime)
{
return quoted + " is a date, but " + label + " takes " + takes + ".";
}
if (value.IsText)
{
string text = value.GetText().Trim();
if (allowPercentSign && text.EndsWith("%"))
{
text = text.Substring(0, text.Length - 1).Trim();
}
if (decimal.TryParse(text, NumberStyles.Number, CultureInfo.InvariantCulture, out number))
{
return null;
}
}
return quoted + " is not a number. " + label + " takes " + takes
+ " — digits only, without words, units or currency signs.";
}
private string TypeFix(TypeRule rule)
{
switch (rule)
{
case TypeRule.ErrorCell:
return "Correct the formula, or replace it with its value (Copy, then Paste Special → Values).";
case TypeRule.Date:
return "Type dates as DD-MMM-YYYY, e.g. 15-Jan-1990. Do not write 15/01/1990 or 1990-01-15 — "
+ "such dates are read differently on different computers.";
case TypeRule.WholeNumber:
return "Enter digits only, without decimals, words or units — e.g. 2015, not 2015.5 or \"2015 year\".";
case TypeRule.Decimal:
return "Enter digits only, with a decimal point when needed — e.g. 3.75 or 15000 — without "
+ "currency signs or words.";
case TypeRule.Percentage:
return "Enter a number from 0 to 100, e.g. 50 for half.";
case TypeRule.Email:
return "Enter one address per cell in the form name@company.com, without spaces.";
case TypeRule.Mobile:
return "Enter the mobile number as 01XXXXXXXXX (11 digits), 8801XXXXXXXXX or +8801XXXXXXXXX.";
default:
return "Pick the value from the cell's dropdown list instead of typing or pasting it.";
}
}
#endregion
#region Phase 8 — master data
/// <summary>One record of a master table, reduced to what matching needs.</summary>
private class MasterRecord
{
public int ID { get; set; }
public string Name { get; set; }
/// <summary>Only an explicit Inactive status counts — some tables do not map Status at all.</summary>
public bool IsInactive { get; set; }
/// <summary>District of a Thana, Education Platform of an Education Level, Education Level of a Discipline.</summary>
public int ParentID { get; set; }
}
private static MasterRecord Record(int id, string name, EnumStatus status, int parentID = 0)
{
return new MasterRecord
{
ID = id,
Name = name,
IsInactive = status == EnumStatus.Inactive,
ParentID = parentID
};
}
/// <summary>Trimmed, inner spaces collapsed, lower case — how a cell is matched to a record.</summary>
private static string MasterKey(string text)
{
return string.Join(" ", (text ?? string.Empty).Split((char[])null, StringSplitOptions.RemoveEmptyEntries))
.ToLowerInvariant();
}
/// <summary>
/// Every record of one master table, through that table's own service — the same lists the
/// app's screens read. The tables that have a PayrollTypeID column (<see cref="PayrollTypeTables"/>:
/// Category, Religion, Grades) are read for <paramref name="payrollTypeID"/> only; the other
/// tables have no such column and are shared by all payroll types. Employee is not loaded here.
/// </summary>
private List<MasterRecord> LoadMaster(EnumBulkUploadMasterData masterData, int payrollTypeID)
{
switch (masterData)
{
case EnumBulkUploadMasterData.Category:
return new CategoryService().Get(EnumStatus.Regardless, payrollTypeID)
.Select(x => Record(x.ID, x.Name, x.Status)).ToList();
case EnumBulkUploadMasterData.Grades:
return new GradeService().Get(EnumStatus.Regardless, payrollTypeID)
.Select(x => Record(x.ID, x.Name, x.Status)).ToList();
case EnumBulkUploadMasterData.Religion:
return new ReligionService().Get(EnumStatus.Regardless, payrollTypeID)
.Select(x => Record(x.ID, x.Name, x.Status)).ToList();
case EnumBulkUploadMasterData.Occupation:
return new OccupationService().Get().Select(x => Record(x.ID, x.Description, x.Status)).ToList();
case EnumBulkUploadMasterData.Nationality:
return new NationalityService().Get().Select(x => Record(x.ID, x.Description, x.Status)).ToList();
case EnumBulkUploadMasterData.Division:
return new DivisionService().Get().Select(x => Record(x.ID, x.Name, x.Status)).ToList();
case EnumBulkUploadMasterData.District:
return new DistrictService().Get().Select(x => Record(x.ID, x.Name, x.Status, x.DivisionID)).ToList();
case EnumBulkUploadMasterData.Thana:
return new ThanaService().Get().Select(x => Record(x.ID, x.Name, x.Status, x.DistrictID)).ToList();
case EnumBulkUploadMasterData.PersonRelation:
return new RelationService().Get().Select(x => Record(x.ID, x.Description, x.Status)).ToList();
case EnumBulkUploadMasterData.EducationType:
return new EducationTypeService().Get().Select(x => Record(x.ID, x.Description, x.Status)).ToList();
case EnumBulkUploadMasterData.EducationLevel:
return new EducationLevelService().Get()
.Select(x => Record(x.ID, x.Description, x.Status, x.EducationTypeID)).ToList();
case EnumBulkUploadMasterData.Institution:
return new InstitutionService().Get().Select(x => Record(x.ID, x.Name, x.Status)).ToList();
case EnumBulkUploadMasterData.Discipline:
return new DisciplineService().Get()
.Select(x => Record(x.ID, x.Description, x.Status, x.EducationLevelID)).ToList();
case EnumBulkUploadMasterData.ResultType:
return new ResultTypeService().Get().Select(x => Record(x.ID, x.Description, x.Status)).ToList();
case EnumBulkUploadMasterData.NominationPurpose:
return new NominationPurposeService().Get().Select(x => Record(x.ID, x.Description, x.Status)).ToList();
case EnumBulkUploadMasterData.TrainingType:
return new TrainingTypeService().Get().Select(x => Record(x.ID, x.Name, x.Status)).ToList();
case EnumBulkUploadMasterData.NatureOfTraining:
return new NatureOfTrainingService().Get().Select(x => Record(x.ID, x.Name, x.Status)).ToList();
case EnumBulkUploadMasterData.Country:
// CountryService does not read Status, so every country counts as active.
return new CountryService().Get().Select(x => Record(x.ID, x.Name, EnumStatus.Active)).ToList();
default:
throw new ArgumentException("Master data " + masterData + " cannot be loaded here.");
}
}
/// <summary>
/// Thana and Education Level are matched within their parent (a Thana of that District);
/// a Discipline is matched by name alone, because its link to an Education Level is not
/// kept reliably, and every other table by name alone.
/// </summary>
private static bool MatchesWithinParent(EnumBulkUploadMasterData masterData)
{
return masterData == EnumBulkUploadMasterData.Thana || masterData == EnumBulkUploadMasterData.EducationLevel;
}
/// <summary>
/// The records named <paramref name="key"/>; within <paramref name="parentID"/> when the
/// table is matched within its parent and the parent is known.
/// </summary>
private static List<MasterRecord> FindRecords(List<MasterRecord> records, EnumBulkUploadMasterData masterData,
string key, int parentID)
{
return records
.Where(x => MasterKey(x.Name) == key
&& (!MatchesWithinParent(masterData) || parentID <= 0 || x.ParentID == parentID))
.ToList();
}
/// <summary>How a cell resolved against a master table.</summary>
private class MasterMatch
{
public MasterMatch()
{
Ambiguous = new List<KeyValuePair<MasterRecord, int>>();
}
/// <summary>The record taken; null when none (missing, or more than one close match).</summary>
public MasterRecord Record { get; set; }
/// <summary>100 for an exact match; otherwise how alike the value and the record are.</summary>
public int Percent { get; set; }
/// <summary>Two or more different records the value is equally close to — then nothing is taken.</summary>
public List<KeyValuePair<MasterRecord, int>> Ambiguous { get; set; }
}
/// <summary>
/// Resolves a value against a master table: an exact match (case and spacing aside) first —
/// an active record before an inactive one — then, among active records, a close match:
/// the one record at least <see cref="EmployeeBulkUploadTemplate.CloseMatchPercent"/>% alike.
/// When two different names are that close, nothing is taken: guessing between them could
/// put an employee under the wrong Thana. Thana and Education Level are looked for within
/// their parent when it is known.
/// </summary>
private static MasterMatch ResolveMaster(List<MasterRecord> records, EnumBulkUploadMasterData masterData,
string value, int parentID)
{
MasterMatch match = new MasterMatch();
List<MasterRecord> exact = FindRecords(records, masterData, MasterKey(value), parentID);
if (exact.Count > 0)
{
match.Record = exact.FirstOrDefault(x => !x.IsInactive) ?? exact[0];
match.Percent = 100;
return match;
}
// Compared on the same text the exact match uses (lower case, single spaces), so every
// character that differs — punctuation included — counts: "Munshigonj." is 90%, not 100%.
string key = MasterKey(value);
if (key.Length == 0)
{
return match;
}
// One candidate per distinct name: two rows of the same name are not a choice between two.
List<KeyValuePair<MasterRecord, int>> close = records
.Where(x => !x.IsInactive && !string.IsNullOrWhiteSpace(x.Name)
&& (!MatchesWithinParent(masterData) || parentID <= 0 || x.ParentID == parentID))
.Select(x => new KeyValuePair<MasterRecord, int>(x, Similarity(key, MasterKey(x.Name))))
.Where(x => x.Value >= EmployeeBulkUploadTemplate.CloseMatchPercent)
.GroupBy(x => MasterKey(x.Key.Name))
.Select(g => g.OrderByDescending(x => x.Value).First())
.OrderByDescending(x => x.Value)
.ToList();
if (close.Count == 1)
{
match.Record = close[0].Key;
match.Percent = close[0].Value;
}
else if (close.Count > 1)
{
match.Ambiguous = close;
}
return match;
}
/// <summary>
/// How alike two strings are, 0–100: 100 × (1 − edits ÷ longer length), where edits is
/// the Levenshtein distance — the fewest single-character insertions, deletions or
/// substitutions that turn one into the other. Rounded down, so 89.9 never counts as 90.
/// </summary>
private static int Similarity(string a, string b)
{
if (a == b)
{
return 100;
}
int longer = Math.Max(a.Length, b.Length);
if (longer == 0)
{
return 100;
}
// Two rows of the distance table are enough.
int[] previous = new int[b.Length + 1];
int[] current = new int[b.Length + 1];
for (int j = 0; j <= b.Length; j++)
{
previous[j] = j;
}
for (int i = 1; i <= a.Length; i++)
{
current[0] = i;
for (int j = 1; j <= b.Length; j++)
{
int cost = a[i - 1] == b[j - 1] ? 0 : 1;
current[j] = Math.Min(Math.Min(current[j - 1] + 1, previous[j] + 1), previous[j - 1] + cost);
}
int[] swap = previous;
previous = current;
current = swap;
}
int edits = previous[b.Length];
return (int)Math.Floor(100.0 * (longer - edits) / longer);
}
/// <summary>
/// Records a value that resolved to a record — closely (<paramref name="close"/> = the close
/// matches) or exactly (the exact matches) — once per value and record, with where it is used,
/// the parent it was looked for under, and what adding the file's value instead would need.
/// </summary>
private void AddCloseMatch(Dictionary<string, EmployeeBulkUploadCloseMatch> close,
EnumBulkUploadMasterData masterData, string value, MasterMatch match, string parentValue, int parentID,
string where, int rowNo, EmployeeBulkUploadSubGroup section)
{
string key = (int)masterData + "|" + MasterKey(value) + "|" + match.Record.ID;
EmployeeBulkUploadCloseMatch item;
if (!close.TryGetValue(key, out item))
{
EnumBulkUploadMasterData parentType = EmployeeBulkUploadMasterInfo.ParentOf(masterData);
item = new EmployeeBulkUploadCloseMatch
{
MasterData = masterData,
MasterName = EmployeeBulkUploadMasterInfo.DisplayName(masterData),
Value = value.Trim(),
MatchedName = match.Record.Name.Trim(),
MatchedID = match.Record.ID,
Percent = match.Percent,
ParentValue = parentValue == null ? null : parentValue.Trim(),
ParentID = parentID,
ParentMasterData = parentType,
ParentMasterName = parentType == EnumBulkUploadMasterData.None
? null : EmployeeBulkUploadMasterInfo.DisplayName(parentType)
};
close[key] = item;
}
if (!item.UsedIn.Contains(where))
{
item.UsedIn.Add(where);
}
if (!item.RowNos.Contains(rowNo))
{
item.RowNos.Add(rowNo);
}
if (masterData == EnumBulkUploadMasterData.Institution)
{
int type = section.Name.StartsWith("Training", StringComparison.OrdinalIgnoreCase) ? 2 : 1;
item.SuggestedInstituteType = item.SuggestedInstituteType == 0 || item.SuggestedInstituteType == type
? type : 3;
}
}
/// <summary>The section column a master column belongs under (District for Thana, ...), or null.</summary>
private static EmployeeBulkUploadColumn ParentColumn(EmployeeBulkUploadSubGroup section, EmployeeBulkUploadColumn column)
{
EnumBulkUploadMasterData parent = EmployeeBulkUploadMasterInfo.ParentOf(column.MasterData);
if (parent == EnumBulkUploadMasterData.None)
{
return null;
}
return section.Columns.FirstOrDefault(x => column.DependsOn != null && x.Label == column.DependsOn)
?? section.Columns.FirstOrDefault(x => x.MasterData == parent);
}
/// <summary>
/// Phase 8 — runs only when phase 7 (Employees) passed. Every filled master-data cell is
/// looked up in its table; a value that is not there becomes one entry of
/// result.MissingMasterData (per distinct value and parent) plus one Problems row per cell,
/// and a value that matches only an inactive record is reported as such. Line Manager ID
/// must be a live employee of the current payroll type. Then the employee checks that need
/// the looked-up values (nominee shares, Grade / Tax Circle kept) run.
/// </summary>
private void CheckMasterData(EmployeeBulkUploadLayer layer, EmployeeBulkUploadReadResult result,
int payrollTypeID, bool canRun, Dictionary<int, EmployeeRef> currentByRow)
{
EmployeeBulkUploadCheck found = layer.AddCheck("MasterValuesFound",
"Every master-data value is in the system — exactly, or at least "
+ EmployeeBulkUploadTemplate.CloseMatchPercent + "% alike to one record");
EmployeeBulkUploadCheck active = layer.AddCheck("MasterValuesActive",
"No master-data value matches only an inactive record");
EmployeeBulkUploadCheck unambiguous = layer.AddCheck("MasterValuesClear",
"No value that is not an exact match is " + EmployeeBulkUploadTemplate.CloseMatchPercent
+ "% or more alike to two different records");
EmployeeBulkUploadCheck nomineeShares = layer.AddCheck("NomineeShares",
"In each row, the nominee shares of one nomination purpose add up to 100% or less");
// Phase 7 made sure of the payroll type.
if (!canRun || payrollTypeID <= 0)
{
layer.SkipRemaining("Not checked — the file holds no employee rows that could be read.");
return;
}
EmployeeBulkUploadSubGroup employeeSection = result.Groups.SelectMany(g => g.SubGroups)
.First(s => s.Name == EmployeeBulkUploadTemplate.AlwaysIncludedSubGroup);
string employeeNoKey = employeeSection.Columns[0].Key;
string employeeNameKey = employeeSection.Columns[1].Key;
// The tables this file uses, loaded once each (with the parents needed to place children).
HashSet<EnumBulkUploadMasterData> used = new HashSet<EnumBulkUploadMasterData>(result.Groups
.SelectMany(g => g.SubGroups).SelectMany(s => s.Columns)
.Where(c => c.IsMasterData && result.Rows.Any(r => r.Values.ContainsKey(c.Key)))
.Select(c => c.MasterData));
foreach (EnumBulkUploadMasterData type in used.ToList())
{
EnumBulkUploadMasterData parent = EmployeeBulkUploadMasterInfo.ParentOf(type);
if (parent != EnumBulkUploadMasterData.None)
{
used.Add(parent);
}
}
Dictionary<EnumBulkUploadMasterData, List<MasterRecord>> tables = used
.Where(x => x != EnumBulkUploadMasterData.Employee)
.ToDictionary(x => x, x => LoadMaster(x, payrollTypeID));
EmployeeService employeeService = new EmployeeService();
// Employee No → (EmployeeID, problem or "") — each Line Manager looked up once.
Dictionary<string, KeyValuePair<int, string>> employeeLookups = new Dictionary<string, KeyValuePair<int, string>>();
Dictionary<string, EmployeeBulkUploadMissingMaster> missing = new Dictionary<string, EmployeeBulkUploadMissingMaster>();
Dictionary<string, EmployeeBulkUploadCloseMatch> close = new Dictionary<string, EmployeeBulkUploadCloseMatch>();
Dictionary<string, EmployeeBulkUploadCloseMatch> exact = new Dictionary<string, EmployeeBulkUploadCloseMatch>();
List<KeyValuePair<string, List<int>>> missingCells = new List<KeyValuePair<string, List<int>>>();
List<KeyValuePair<string, List<int>>> inactiveCells = new List<KeyValuePair<string, List<int>>>();
List<KeyValuePair<string, List<int>>> ambiguousCells = new List<KeyValuePair<string, List<int>>>();
foreach (EmployeeBulkUploadRow row in result.Rows)
{
foreach (EmployeeBulkUploadGroup group in result.Groups)
{
foreach (EmployeeBulkUploadSubGroup section in group.SubGroups)
{
foreach (EmployeeBulkUploadColumn column in section.Columns.Where(c => c.IsMasterData))
{
string value;
if (!row.Values.TryGetValue(column.Key, out value))
{
continue;
}
string key = MasterKey(value);
string where = section.Name + " — \"" + column.Label + "\"";
string masterName = EmployeeBulkUploadMasterInfo.DisplayName(column.MasterData);
string problem = null;
bool isInactive = false;
bool isAmbiguous = false;
if (column.MasterData == EnumBulkUploadMasterData.Employee)
{
// An Employee No is an identifier: matched exactly, never "close enough".
KeyValuePair<int, string> lookup;
if (!employeeLookups.TryGetValue(key, out lookup))
{
Employee manager = employeeService.Get(value.Trim(), payrollTypeID);
string notFound = manager != null ? string.Empty
: employeeService.GetFromAll(value.Trim(), payrollTypeID) == null
? "No employee of the current payroll type has Employee No \"" + value + "\"."
: "Employee No \"" + value + "\" is not a live employee of the current payroll type.";
lookup = new KeyValuePair<int, string>(manager != null ? manager.ID : 0, notFound);
employeeLookups[key] = lookup;
}
if (lookup.Value.Length == 0)
{
row.MasterIDs[column.Key] = lookup.Key;
continue;
}
problem = lookup.Value + " " + EmployeeBulkUploadMasterInfo.CannotAddReason(column.MasterData);
AddMissing(missing, column.MasterData, value, null, EnumBulkUploadMasterData.None, 0, false,
where, row.RowNo, section);
}
else
{
// The parent from the same block: the District of this Thana, ... — itself
// matched exactly or closely, so a Thana is looked for under the right District.
EmployeeBulkUploadColumn parentColumn = ParentColumn(section, column);
EnumBulkUploadMasterData parentType = EmployeeBulkUploadMasterInfo.ParentOf(column.MasterData);
string parentValue = null;
int parentID = 0;
bool parentIsMissing = false;
if (parentColumn != null && row.Values.TryGetValue(parentColumn.Key, out parentValue))
{
MasterMatch parent = ResolveMaster(tables[parentType], parentType, parentValue, 0);
if (parent.Record != null)
{
parentID = parent.Record.ID;
}
else
{
parentIsMissing = true;
}
}
// A missing parent hides the value only where it is matched within its parent
// (Thana, Education Level); a Discipline is matched by name alone, so it is
// still found when its Education Level is new.
MasterMatch match = parentIsMissing && MatchesWithinParent(column.MasterData)
? new MasterMatch()
: ResolveMaster(tables[column.MasterData], column.MasterData, value, parentID);
if (match.Record != null && !match.Record.IsInactive)
{
row.MasterIDs[column.Key] = match.Record.ID;
if (match.Percent >= 100 && MasterKey(match.Record.Name) == key)
{
// An exact match: listed under "Fully Match" in the master-data dialog.
AddCloseMatch(exact, column.MasterData, value, match, parentValue, parentID, where,
row.RowNo, section);
continue;
}
// A close match: taken, shown as a warning, confirmed by the user before saving.
AddCloseMatch(close, column.MasterData, value, match, parentValue, parentID, where,
row.RowNo, section);
result.Issues.Add(new EmployeeBulkUploadIssue
{
Severity = EmployeeBulkUploadIssue.Warning,
RowNo = row.RowNo,
EmployeeNo = ValueOf(row, employeeNoKey),
EmployeeName = ValueOf(row, employeeNameKey),
Group = group.Name,
SubGroup = section.Name,
Column = column.Label,
Value = value,
Message = "\"" + value + "\" is not an exact " + masterName + " name, but it is "
+ match.Percent + "% alike to \"" + match.Record.Name.Trim() + "\", so it is taken as \""
+ match.Record.Name.Trim() + "\". If that is wrong, correct the value in the file."
});
continue;
}
if (match.Ambiguous.Count > 1)
{
isAmbiguous = true;
problem = "\"" + value + "\" is not an exact " + masterName + " name and is close to more than one: "
+ string.Join(", ", match.Ambiguous.Select(x => "\"" + x.Key.Name.Trim() + "\" (" + x.Value + "%)"))
+ ". Write the exact name in the file.";
}
else if (match.Record != null)
{
isInactive = true;
problem = "\"" + value + "\" is in " + masterName + " but is inactive. Activate it in the "
+ masterName + " screen, or use an active value.";
}
else
{
string ofParent = parentValue != null && MatchesWithinParent(column.MasterData)
? " under " + EmployeeBulkUploadMasterInfo.DisplayName(parentType) + " \"" + parentValue + "\""
: string.Empty;
string cannot = EmployeeBulkUploadMasterInfo.CannotAddReason(column.MasterData);
problem = "\"" + value + "\" is not in " + masterName + ofParent + ". "
+ (cannot ?? "If it is new, add it from the \"Add missing master data\" dialog; "
+ "if it is a typing mistake, correct it in the file.");
AddMissing(missing, column.MasterData, value, parentValue, parentType, parentID,
parentIsMissing, where, row.RowNo, section);
}
}
result.Issues.Add(new EmployeeBulkUploadIssue
{
Severity = EmployeeBulkUploadIssue.Error,
RowNo = row.RowNo,
EmployeeNo = ValueOf(row, employeeNoKey),
EmployeeName = ValueOf(row, employeeNameKey),
Group = group.Name,
SubGroup = section.Name,
Column = column.Label,
Value = value,
Message = problem
});
AddMiss(isAmbiguous ? ambiguousCells : isInactive ? inactiveCells : missingCells, where, row.RowNo);
}
}
}
}
result.CloseMatches = close.Values
.OrderBy(x => Array.IndexOf(EmployeeBulkUploadMasterInfo.AddOrder, x.MasterData))
.ThenBy(x => x.Value, StringComparer.OrdinalIgnoreCase)
.ToList();
result.ExactMatches = exact.Values
.OrderBy(x => Array.IndexOf(EmployeeBulkUploadMasterInfo.AddOrder, x.MasterData))
.ThenBy(x => x.Value, StringComparer.OrdinalIgnoreCase)
.ToList();
if (ambiguousCells.Count > 0)
{
unambiguous.Fail(MissSummary(ambiguousCells, "unclear") + " Each cell is listed in the Problems table below.",
MissDetails(result, ambiguousCells, "unclear"),
"A value close to two or more records cannot be decided for you. Write the exact name in the file.");
}
else
{
unambiguous.Pass();
}
result.MissingMasterData = missing.Values
.OrderBy(x => Array.IndexOf(EmployeeBulkUploadMasterInfo.AddOrder, x.MasterData))
.ThenBy(x => x.ParentValue ?? string.Empty, StringComparer.OrdinalIgnoreCase)
.ThenBy(x => x.Value, StringComparer.OrdinalIgnoreCase)
.ToList();
string tableHint = " Each cell is listed in the Problems table below.";
if (missingCells.Count > 0)
{
// One line per missing value: the table, the value, where it is used and by whom.
List<string> details = result.MissingMasterData
.Select(x => x.MasterName + " \"" + x.Value + "\""
+ (x.ParentValue != null && MatchesWithinParent(x.MasterData)
? " under " + x.ParentMasterName + " \"" + x.ParentValue + "\"" : string.Empty)
+ (x.CanAdd ? string.Empty : " (cannot be added from here)")
+ " — " + string.Join(", ", x.UsedIn) + " — "
+ string.Join(", ", x.RowNos.Take(10).Select(rowNo => RowLabel(result, rowNo)))
+ (x.RowNos.Count > 10 ? " and " + (x.RowNos.Count - 10) + " more" : string.Empty))
.ToList();
found.Fail(Count(result.MissingMasterData.Count, "value is", "values are") + " not in the system, used in "
+ MissSummary(missingCells, "master-data") + tableHint, details,
"Add the new values from the \"Add missing master data\" dialog — it opens by itself — then the file "
+ "is checked again. If a value is a typing mistake, correct it in the file and upload again. "
+ "Grade and Line Manager ID values cannot be added from here.");
}
else
{
found.Pass();
}
if (inactiveCells.Count > 0)
{
active.Fail(MissSummary(inactiveCells, "inactive") + tableHint, MissDetails(result, inactiveCells, "inactive"),
"Activate the record in its master screen, or change the value in the file to an active one.");
}
else
{
active.Pass();
}
CheckEmployeeData(nomineeShares, result, currentByRow);
}
private void AddMissing(Dictionary<string, EmployeeBulkUploadMissingMaster> missing,
EnumBulkUploadMasterData masterData, string value, string parentValue, EnumBulkUploadMasterData parentType,
int parentID, bool parentIsMissing, string where, int rowNo, EmployeeBulkUploadSubGroup section)
{
// A Thana is new per District; any other value is new once, whatever its parent.
string scope = MatchesWithinParent(masterData) ? MasterKey(parentValue) : string.Empty;
string key = (int)masterData + "|" + MasterKey(value) + "|" + scope;
EmployeeBulkUploadMissingMaster item;
if (!missing.TryGetValue(key, out item))
{
string cannot = EmployeeBulkUploadMasterInfo.CannotAddReason(masterData);
item = new EmployeeBulkUploadMissingMaster
{
MasterData = masterData,
MasterName = EmployeeBulkUploadMasterInfo.DisplayName(masterData),
Value = value.Trim(),
ParentValue = parentValue == null ? null : parentValue.Trim(),
ParentMasterData = parentType,
ParentMasterName = parentType == EnumBulkUploadMasterData.None
? null : EmployeeBulkUploadMasterInfo.DisplayName(parentType),
ParentID = parentID,
ParentIsMissing = parentIsMissing,
CanAdd = cannot == null,
CannotAddReason = cannot
};
missing[key] = item;
}
else if (item.ParentValue == null && parentValue != null)
{
// The same Discipline met again with its Education Level: keep the parent for the add.
item.ParentValue = parentValue.Trim();
item.ParentID = parentID;
item.ParentIsMissing = parentIsMissing;
}
if (!item.UsedIn.Contains(where))
{
item.UsedIn.Add(where);
}
if (!item.RowNos.Contains(rowNo))
{
item.RowNos.Add(rowNo);
}
if (masterData == EnumBulkUploadMasterData.Institution)
{
// Education blocks need an academic institution, Training blocks an institutional one.
int type = section.Name.StartsWith("Training", StringComparison.OrdinalIgnoreCase) ? 2 : 1;
item.SuggestedInstituteType = item.SuggestedInstituteType == 0 || item.SuggestedInstituteType == type
? type : 3;
}
}
#endregion
#region Phase 7 — employees
/// <summary>An existing employee an Employee No belongs to.</summary>
private class EmployeeRef
{
public int ID { get; set; }
public int PayrollTypeID { get; set; }
public int GradeID { get; set; }
public int TaxCircle { get; set; }
/// <summary>EnumEmployeeStatus: only Live (1) is an active employee.</summary>
public int Status { get; set; }
}
/// <summary>
/// Sets each row's EmployeeState (and EmployeeID): "existing" — an active (Live) employee of
/// the current payroll type; "inactive" — of this payroll type but not Live; "otherPayroll" —
/// the Employee No belongs to another payroll type only; "new" — the Employee No is nowhere.
/// Returns the current-payroll-type employee per row, for the checks that need it. Read only.
/// </summary>
private Dictionary<int, EmployeeRef> ClassifyEmployees(EmployeeBulkUploadReadResult result, int payrollTypeID)
{
Dictionary<int, EmployeeRef> currentByRow = new Dictionary<int, EmployeeRef>();
EmployeeBulkUploadSubGroup employeeSection = result.Groups.SelectMany(g => g.SubGroups)
.FirstOrDefault(s => s.Name == EmployeeBulkUploadTemplate.AlwaysIncludedSubGroup);
if (employeeSection == null || payrollTypeID <= 0)
{
return currentByRow;
}
string employeeNoKey = employeeSection.Columns[0].Key;
Dictionary<string, List<EmployeeRef>> known = LoadEmployeeRefs(
result.Rows.Select(r => ValueOf(r, employeeNoKey).Trim()).Where(x => x.Length > 0).Distinct());
foreach (EmployeeBulkUploadRow row in result.Rows)
{
string employeeNo = ValueOf(row, employeeNoKey).Trim();
if (employeeNo.Length == 0)
{
row.EmployeeState = string.Empty;
continue;
}
List<EmployeeRef> refs;
known.TryGetValue(employeeNo.ToLowerInvariant(), out refs);
EmployeeRef current = refs == null ? null : refs.FirstOrDefault(x => x.PayrollTypeID == payrollTypeID);
row.EmployeeID = 0;
if (current != null)
{
currentByRow[row.RowNo] = current;
if (current.Status == (int)EnumEmployeeStatus.Live)
{
row.EmployeeState = EmployeeStates.Existing;
row.EmployeeID = current.ID;
}
else
{
row.EmployeeState = EmployeeStates.Inactive;
}
}
else
{
row.EmployeeState = refs != null && refs.Count > 0 ? EmployeeStates.OtherPayroll : EmployeeStates.New;
}
}
return currentByRow;
}
/// <summary>
/// Phase 7 — runs only when phase 6 (Data types) passed. Decides, per row, whether it
/// updates an active employee of the current payroll type or creates a new one
/// (row.EmployeeID); an Employee No of another payroll type, an inactive employee, or a new
/// employee without General is an error — no employee is added in any other case. Also
/// checks that one person is not entered under two Employee Nos. Returns the
/// current-payroll-type employee per row (empty when the phase did not run), for the
/// employee checks that run with the master data phase.
/// </summary>
private Dictionary<int, EmployeeRef> CheckEmployees(EmployeeBulkUploadLayer layer, EmployeeBulkUploadReadResult result,
int payrollTypeID, bool canRun)
{
EmployeeBulkUploadCheck payrollType = layer.AddCheck("PayrollType",
"The current payroll type is known (Category, Grade and employees are kept per payroll type)");
EmployeeBulkUploadCheck otherPayroll = layer.AddCheck("EmployeeNoPayrollType",
"No Employee No belongs to an employee of another payroll type");
EmployeeBulkUploadCheck activeEmployee = layer.AddCheck("EmployeeActive",
"Every existing employee in the file is active (Live)");
EmployeeBulkUploadCheck newHaveGeneral = layer.AddCheck("NewEmployeesHaveGeneral",
"Each new employee's row has the General section (date of birth, joining, category, grade, ...)");
EmployeeBulkUploadCheck onePerPerson = layer.AddCheck("OneRowPerPerson",
"No National ID No is in two rows under different Employee Nos (one person entered twice)");
if (!canRun)
{
layer.SkipRemaining("Not checked — the file holds no employee rows that could be read.");
return new Dictionary<int, EmployeeRef>();
}
if (payrollTypeID <= 0)
{
payrollType.Fail("The current payroll type could not be found for this login.", null,
"Log out, log in again, choose the payroll type, and upload the file again.");
layer.SkipRemaining("Not checked — the current payroll type is not known.");
return new Dictionary<int, EmployeeRef>();
}
payrollType.Pass();
List<EmployeeBulkUploadSubGroup> sections = result.Groups.SelectMany(g => g.SubGroups).ToList();
EmployeeBulkUploadSubGroup employeeSection = sections.First(s => s.Name == EmployeeBulkUploadTemplate.AlwaysIncludedSubGroup);
string employeeNoKey = employeeSection.Columns[0].Key;
string employeeNameKey = employeeSection.Columns[1].Key;
EmployeeBulkUploadSubGroup general = sections.FirstOrDefault(s => s.Name == "General");
EmployeeBulkUploadGroup employeeGroup = result.Groups.First(g => g.SubGroups.Contains(employeeSection));
Dictionary<int, EmployeeRef> currentByRow = ClassifyEmployees(result, payrollTypeID);
List<KeyValuePair<string, List<int>>> otherCells = new List<KeyValuePair<string, List<int>>>();
List<KeyValuePair<string, List<int>>> inactiveCells = new List<KeyValuePair<string, List<int>>>();
List<KeyValuePair<string, List<int>>> noGeneralCells = new List<KeyValuePair<string, List<int>>>();
int newCount = 0;
int existingCount = 0;
foreach (EmployeeBulkUploadRow row in result.Rows)
{
string employeeNo = ValueOf(row, employeeNoKey).Trim();
EmployeeRef current;
currentByRow.TryGetValue(row.RowNo, out current);
if (row.EmployeeState == EmployeeStates.Existing)
{
// The Grade / Tax Circle "not changed" warnings need the looked-up grade, so they
// come with the master data phase (CheckEmployeeData).
existingCount++;
}
else if (row.EmployeeState == EmployeeStates.Inactive)
{
// Of this payroll type but not Live: nothing may be added or changed for this employee.
AddRowIssue(result, row, employeeGroup, employeeSection, employeeSection.Columns[0].Label, employeeNo,
"Employee No \"" + employeeNo + "\" is an employee of this payroll type but is not active (status: "
+ (Enum.IsDefined(typeof(EnumEmployeeStatus), current.Status)
? ((EnumEmployeeStatus)current.Status).ToString() : current.Status.ToString())
+ "). Only an active (Live) employee can be updated from here — remove this row, or correct the Employee No.",
employeeNoKey, employeeNameKey);
AddMiss(inactiveCells, employeeSection.Name + " — \"" + employeeSection.Columns[0].Label + "\"", row.RowNo);
}
else if (row.EmployeeState == EmployeeStates.OtherPayroll)
{
AddRowIssue(result, row, employeeGroup, employeeSection, employeeSection.Columns[0].Label, employeeNo,
"Employee No \"" + employeeNo + "\" belongs to an employee of another payroll type. Nothing can be "
+ "added or changed for another payroll type's employee here — switch to that payroll type, or "
+ "correct the Employee No.",
employeeNoKey, employeeNameKey);
AddMiss(otherCells, employeeSection.Name + " — \"" + employeeSection.Columns[0].Label + "\"", row.RowNo);
}
else if (row.EmployeeState == EmployeeStates.New)
{
newCount++;
if (general == null)
{
AddRowIssue(result, row, employeeGroup, employeeSection, employeeSection.Columns[0].Label, employeeNo,
"Employee No \"" + employeeNo + "\" is not an employee of the current payroll type, so this row "
+ "would create a new employee — and a new employee needs the General section. Tick General "
+ "and fill it in, or correct the Employee No.",
employeeNoKey, employeeNameKey);
AddMiss(noGeneralCells, employeeSection.Name + " — \"" + employeeSection.Columns[0].Label + "\"", row.RowNo);
}
}
}
result.NewEmployeeCount = newCount;
result.ExistingEmployeeCount = existingCount;
string tableHint = " Each row is listed in the Problems table below.";
if (otherCells.Count > 0)
{
otherPayroll.Fail(MissSummary(otherCells, "Employee No") + tableHint, MissDetails(result, otherCells, "of another payroll type"),
"Upload these employees under their own payroll type, or correct the Employee No.");
}
else
{
otherPayroll.Pass();
}
if (inactiveCells.Count > 0)
{
activeEmployee.Fail(MissSummary(inactiveCells, "inactive employee") + tableHint,
MissDetails(result, inactiveCells, "not active"),
"Remove the rows of inactive employees from the file, or correct their Employee No. An inactive "
+ "employee is changed only through their own screens (e.g. re-joining), not by a bulk upload.");
}
else
{
activeEmployee.Pass();
}
if (noGeneralCells.Count > 0)
{
newHaveGeneral.Fail(MissSummary(noGeneralCells, "new employee") + tableHint,
MissDetails(result, noGeneralCells, "new, without General"),
"Tick General in Template sections, download the template again and fill in General for the new "
+ "employees — or correct the Employee No if the employee already exists.");
}
else
{
newHaveGeneral.Pass();
}
CheckOneRowPerPerson(onePerPerson, result, sections, employeeNoKey, employeeNameKey);
return currentByRow;
}
/// <summary>
/// The employee checks that need the looked-up master data, run with the master data phase
/// once its own checks are done: the nominee shares of one nomination purpose add up to 100%
/// or less (a failed check of <paramref name="nomineeShares"/>), and an existing employee's
/// Grade / Tax Circle in the file that differs from the system is kept as it is (a warning).
/// </summary>
private void CheckEmployeeData(EmployeeBulkUploadCheck nomineeShares, EmployeeBulkUploadReadResult result,
Dictionary<int, EmployeeRef> currentByRow)
{
List<EmployeeBulkUploadSubGroup> sections = result.Groups.SelectMany(g => g.SubGroups).ToList();
EmployeeBulkUploadSubGroup employeeSection = sections.First(s => s.Name == EmployeeBulkUploadTemplate.AlwaysIncludedSubGroup);
string employeeNoKey = employeeSection.Columns[0].Key;
string employeeNameKey = employeeSection.Columns[1].Key;
EmployeeBulkUploadSubGroup general = sections.FirstOrDefault(s => s.Name == "General");
EmployeeBulkUploadGroup generalGroup = general == null ? null : result.Groups.First(g => g.SubGroups.Contains(general));
List<KeyValuePair<string, List<int>>> shareCells = new List<KeyValuePair<string, List<int>>>();
foreach (EmployeeBulkUploadRow row in result.Rows)
{
EmployeeRef current;
if (general != null && row.EmployeeState == EmployeeStates.Existing
&& currentByRow.TryGetValue(row.RowNo, out current))
{
// A grade or tax-circle change is not a profile edit; the upload does not make it.
WarnNotChanged(result, row, generalGroup, general, "Grade Code", current.GradeID,
"A grade change goes through Employee Life Cycle (promotion / demotion), so it is not changed here.",
employeeNoKey, employeeNameKey);
WarnNotChanged(result, row, generalGroup, general, "Tax Circle", current.TaxCircle,
"The employee profile does not change the tax circle, so the upload does not either.",
employeeNoKey, employeeNameKey);
}
// Nominee shares in this row, per nomination purpose.
foreach (IGrouping<int, KeyValuePair<EmployeeBulkUploadSubGroup, double>> purpose in sections
.Where(s => s.Name.StartsWith("Nominee", StringComparison.OrdinalIgnoreCase))
.Select(s => new
{
Section = s,
Purpose = s.Columns.First(c => c.Label == "Nomination Purpose"),
Share = s.Columns.First(c => c.Label == "Percentage")
})
.Where(x => row.MasterIDs.ContainsKey(x.Purpose.Key) && row.Parsed.ContainsKey(x.Share.Key))
.GroupBy(x => row.MasterIDs[x.Purpose.Key],
x => new KeyValuePair<EmployeeBulkUploadSubGroup, double>(x.Section, (double)(decimal)row.Parsed[x.Share.Key])))
{
double total = purpose.Sum(x => x.Value);
if (total > 100)
{
EmployeeBulkUploadSubGroup first = purpose.First().Key;
AddRowIssue(result, row, result.Groups.First(g => g.SubGroups.Contains(first)), first, "Percentage",
total.ToString("0.##", CultureInfo.InvariantCulture) + "%",
"The nominees of one nomination purpose (" + string.Join(", ", purpose.Select(x => x.Key.Name))
+ ") have " + total.ToString("0.##", CultureInfo.InvariantCulture)
+ "% together — more than 100%. Correct the shares.",
employeeNoKey, employeeNameKey);
AddMiss(shareCells, "Nominee — \"Percentage\"", row.RowNo);
}
}
}
string tableHint = " Each row is listed in the Problems table below.";
if (shareCells.Count > 0)
{
nomineeShares.Fail(MissSummary(shareCells, "nominee share") + tableHint, MissDetails(result, shareCells, "over 100%"),
"Make the shares of each nomination purpose add up to 100% or less.");
}
else
{
nomineeShares.Pass();
}
}
/// <summary>Every employee — any payroll type, any status — with one of these Employee Nos, keyed by lower-case number.</summary>
private Dictionary<string, List<EmployeeRef>> LoadEmployeeRefs(IEnumerable<string> employeeNos)
{
Dictionary<string, List<EmployeeRef>> known = new Dictionary<string, List<EmployeeRef>>();
List<string> all = employeeNos.ToList();
TransactionContext tc = null;
try
{
tc = TransactionContext.Begin();
for (int start = 0; start < all.Count; start += 500)
{
using (IDataReader reader = EmployeeBulkUploadDA.GetEmployeesByNo(tc, all.Skip(start).Take(500)))
{
while (reader.Read())
{
string no = Convert.ToString(reader["EmployeeNo"]).Trim().ToLowerInvariant();
List<EmployeeRef> refs;
if (!known.TryGetValue(no, out refs))
{
refs = new List<EmployeeRef>();
known[no] = refs;
}
refs.Add(new EmployeeRef
{
ID = Convert.ToInt32(reader["EmployeeID"]),
PayrollTypeID = reader["PayrollTypeID"] == DBNull.Value ? 0 : Convert.ToInt32(reader["PayrollTypeID"]),
GradeID = reader["GradeID"] == DBNull.Value ? 0 : Convert.ToInt32(reader["GradeID"]),
TaxCircle = reader["TaxCircle"] == DBNull.Value ? 0 : Convert.ToInt32(reader["TaxCircle"]),
Status = reader["Status"] == DBNull.Value ? 0 : Convert.ToInt32(reader["Status"])
});
}
}
}
tc.End();
}
catch (Exception)
{
if (tc != null)
{
tc.HandleError();
}
throw;
}
return known;
}
/// <summary>A warning when the file gives an existing employee another grade / tax circle than they have.</summary>
private void WarnNotChanged(EmployeeBulkUploadReadResult result, EmployeeBulkUploadRow row, EmployeeBulkUploadGroup group,
EmployeeBulkUploadSubGroup general, string label, int currentValue, string why, string employeeNoKey, string employeeNameKey)
{
EmployeeBulkUploadColumn column = general.Columns.First(c => c.Label == label);
object value;
int fileValue;
if (!TryGetSaveValue(row, column, out value) || (fileValue = Convert.ToInt32(value)) == currentValue)
{
return;
}
AddRowIssue(result, row, group, general, label, ValueOf(row, column.Key),
"This employee already exists; \"" + label + "\" is kept as it is in the system. " + why,
employeeNoKey, employeeNameKey, EmployeeBulkUploadIssue.Warning);
}
private void AddRowIssue(EmployeeBulkUploadReadResult result, EmployeeBulkUploadRow row, EmployeeBulkUploadGroup group,
EmployeeBulkUploadSubGroup section, string column, string value, string message, string employeeNoKey,
string employeeNameKey, string severity = EmployeeBulkUploadIssue.Error)
{
result.Issues.Add(new EmployeeBulkUploadIssue
{
Severity = severity,
RowNo = row.RowNo,
EmployeeNo = ValueOf(row, employeeNoKey),
EmployeeName = ValueOf(row, employeeNameKey),
Group = group.Name,
SubGroup = section.Name,
Column = column,
Value = value,
Message = message
});
}
#endregion
#region Save
/// <summary>One column the save writes: its template label, its DB column, how it is written, its BO property.</summary>
private class SaveField
{
public SaveField(string label, string column, EmployeeBulkUploadDA.SqlKind kind, string property)
{
Label = label;
Column = column;
Kind = kind;
Property = property;
}
public string Label { get; private set; }
public string Column { get; private set; }
public EmployeeBulkUploadDA.SqlKind Kind { get; private set; }
public string Property { get; private set; }
/// <summary>Used only when the employee is new (Grade, Tax Circle).</summary>
public bool NewOnly { get; set; }
/// <summary>Not written by HREmployeeDA.Insert, so set right after it (Training's Sponsor Type).</summary>
public bool AfterInsert { get; set; }
}
/// <summary>A repeated profile table (Children, Education, ...): where it lives and how a record is recognised.</summary>
private class RepeatTable
{
public string Table { get; set; }
public string IdColumn { get; set; }
/// <summary>The labels whose values identify a record; the second one counts only when the file fills it.</summary>
public string[] KeyLabels { get; set; }
public List<SaveField> Fields { get; set; }
}
private static readonly EmployeeBulkUploadDA.SqlKind Text = EmployeeBulkUploadDA.SqlKind.Text;
private static readonly EmployeeBulkUploadDA.SqlKind Unicode = EmployeeBulkUploadDA.SqlKind.Unicode;
private static readonly EmployeeBulkUploadDA.SqlKind Number = EmployeeBulkUploadDA.SqlKind.Number;
private static readonly EmployeeBulkUploadDA.SqlKind Day = EmployeeBulkUploadDA.SqlKind.Date;
/// <summary>
/// Employee-table columns per section; the DB columns are the ones HREmployeeDA.Update and
/// Insert write, the properties those of HREmployee.
/// </summary>
private static readonly Dictionary<string, List<SaveField>> EmployeeFields = new Dictionary<string, List<SaveField>>
{
{
EmployeeBulkUploadTemplate.AlwaysIncludedSubGroup, new List<SaveField>
{
new SaveField("Employee Name", "Name", Text, "Name")
}
},
{
"General", new List<SaveField>
{
new SaveField("First Name", "FirstName", Text, "FirstName"),
new SaveField("Middle Name", "MiddleName", Text, "MiddleName"),
new SaveField("Last Name", "LastName", Text, "LastName"),
new SaveField("Name (Ban)", "Banglaname", Unicode, "BanglaName"),
new SaveField("Date of Birth", "BirthDate", Day, "BirthDate"),
new SaveField("Gender", "Gender", Number, "Gender"),
new SaveField("Mobile No", "MobileNo", Text, "MobileNo"),
new SaveField("Email", "EmailAddress", Text, "EmailAddress"),
new SaveField("Date of Joining", "JoiningDate", Day, "JoiningDate"),
new SaveField("Blood Group", "BloodGroup", Number, "BloodGroup"),
new SaveField("Marital Status", "MaritalStatusID", Number, "MaritalStatus"),
new SaveField("Religion Name", "ReligionID", Number, "ReligionID"),
new SaveField("Nationality", "NationalityID", Number, "NationalityID"),
new SaveField("Category Name", "CategoryID", Number, "CategoryID"),
new SaveField("Tax Circle", "TaxCircle", Number, "TaxCircle") { NewOnly = true },
new SaveField("Grade Code", "GradeID", Number, "GradeID") { NewOnly = true },
new SaveField("Confirmation Date", "DateOfConfirmation", Day, "ConfirDate"),
new SaveField("Line Manager ID", "LineManagerID", Number, "LineManagerID")
}
},
{
"Extended Personal Information", new List<SaveField>
{
new SaveField("Global Emp ID", "GlobalID", Text, "GlobalID"),
new SaveField("National ID No", "NationalID", Text, "NationalID"),
new SaveField("TIN No", "TinNo", Text, "TinNo"),
new SaveField("Passport No", "PassportNo", Text, "PassportNo"),
new SaveField("Passport Issued Date", "PassportIssueDate", Day, "PassportIssueDate"),
new SaveField("Passport Expiry Date", "PassportExpDate", Day, "PassportExpDate"),
new SaveField("Driving License No", "DrivingLicenceNo", Text, "DrivingLicenceNo"),
new SaveField("Father's Name", "FatherName", Text, "FatherName"),
new SaveField("Father's Name (Ban)", "FatherNameBangla", Unicode, "FatherNameBangla"),
new SaveField("Father's Occupation", "FatherOccupationID", Number, "FatherOccupationID"),
new SaveField("Mother's Name", "MotherName", Text, "MotherName"),
new SaveField("Mother's Name (Ban)", "MotherNameBangla", Unicode, "MotherNameBangla"),
new SaveField("Mother's Occupation", "MotherOccupationID", Number, "MotherOccupationID"),
new SaveField("Birth Place", "BirthPlace", Text, "BirthPlace"),
new SaveField("Height (Cm)", "Height", Text, "Height")
}
}
};
/// <summary>EmpContact columns per section — the columns HREmployeeDA.Insert/update(EmpContact) write.</summary>
private static readonly Dictionary<string, List<SaveField>> ContactFields = new Dictionary<string, List<SaveField>>
{
{
"Present Address", new List<SaveField>
{
new SaveField("Address", "PresentAddress", Text, "PresentAddress"),
new SaveField("Address (Bn)", "PresentAddressInBangla", Unicode, "PresentAddressInBangla"),
new SaveField("District", "PresentDistrictID", Number, "PresentDistrictID"),
new SaveField("Thana", "PresentThanaID", Number, "PresentThanaID"),
new SaveField("Land Phone No", "PresentTelephone", Text, "PresentTelephone"),
new SaveField("Post Office Name (Bn)", "PresentPOInBangla", Unicode, "PresentPOInBangla"),
new SaveField("Mobile No", "PresentMobile", Text, "PresentMobile"),
new SaveField("Postal Code", "PresentPostCodeNo", Text, "PresentPostCodeNo")
}
},
{
"Permanent Address", new List<SaveField>
{
new SaveField("Address", "ParmanentAddress", Text, "PermanentAddress"),
new SaveField("Address (Bn)", "PermanentAddressInBangla", Unicode, "PermanentAddressInBangla"),
new SaveField("District", "ParmanentDistrictID", Number, "PermanentDistrictID"),
new SaveField("Thana", "ParmanentThanaID", Number, "PermanentThanaID"),
new SaveField("Land Phone No", "ParmanentTelephone", Text, "PermanentTelephone"),
new SaveField("Post Office Name (Bn)", "ParmanentPOInBangla", Unicode, "ParmanentPOInBangla"),
new SaveField("Mobile No", "ParmanentMobile", Text, "PermanentMobile"),
new SaveField("Postal Code", "PermanentPostCodeNo", Text, "PermanentPostCodeNo")
}
},
{
"Contact Details", new List<SaveField>
{
new SaveField("Land Phone No", "PersonalTelephone", Text, "PersonalTelephone"),
new SaveField("Mobile No", "Mobile", Text, "Mobile"),
new SaveField("Personal Email", "PersonalEmail", Text, "PersonalEmail"),
new SaveField("Fax", "Fax", Text, "Fax")
}
},
{
"Emergency Contact Details", new List<SaveField>
{
new SaveField("Contact Person", "EmergencyContactPerson", Text, "EmergencyContactPerson"),
new SaveField("Address", "EmergencyContactAddress", Text, "EmergencyContactAddress"),
new SaveField("Relation", "CPRelationID", Number, "ContactPersonRelationId"),
new SaveField("Mobile No", "EmergencyMobile", Text, "EmergencyMobile")
}
}
};
/// <summary>The repeated profile tables, by the start of their section names (Children 01 → "Children").</summary>
private static readonly Dictionary<string, RepeatTable> RepeatTables = new Dictionary<string, RepeatTable>
{
{
"Children", new RepeatTable
{
Table = "EmpChildren", IdColumn = "ChildID", KeyLabels = new[] { "Name" },
Fields = new List<SaveField>
{
new SaveField("Name", "Name", Text, "Name"),
new SaveField("Gender", "Gender", Number, "Gender"),
new SaveField("Birth Date", "BirthDate", Day, "BirthDate"),
new SaveField("Occupation", "OccupationID", Number, "OccupationID"),
new SaveField("Marital Status", "MaritalStatus", Number, "MaritalStatus")
}
}
},
{
"Spouse", new RepeatTable
{
Table = "EmpSpouse", IdColumn = "SpouseID", KeyLabels = new[] { "Name" },
Fields = new List<SaveField>
{
new SaveField("Name", "Name", Text, "Name"),
new SaveField("Level of Education", "EducationLevelID", Number, "EducationLevelID"),
new SaveField("Occupation", "OccupationID", Number, "OccupationID"),
new SaveField("Birth Date", "DateOfBirth", Day, "DateOfBirth"),
new SaveField("Marriage Date", "MarriageDate", Day, "MarriageDate")
}
}
},
{
"Education", new RepeatTable
{
Table = "EmpAcademic", IdColumn = "AcademicID", KeyLabels = new[] { "Education Level" },
Fields = new List<SaveField>
{
new SaveField("Education Platform", "EducationTypeID", Number, "EducationTypeID"),
new SaveField("Education Level", "EducationLevelID", Number, "EducationLevelID"),
new SaveField("University / Board", "InstitutionID", Number, "InstitutionID"),
new SaveField("Discipline", "DisciplineID", Number, "DisciplineID"),
new SaveField("Institute Name", "InstituteName", Text, "InstituteName"),
new SaveField("Exam Year", "ExamHeldOn", Number, "ExamDate"),
new SaveField("Passing Year", "PassingYear", Number, "PassingYear"),
new SaveField("Result Standard", "ResultTypeID", Number, "ResultTypeID"),
new SaveField("GPA / Marks", "GPAOrMarks", Number, "GPAOrMarks"),
new SaveField("Out of", "OutOf", Number, "OutOf")
}
}
},
{
"Nominee", new RepeatTable
{
Table = "EmpNominee", IdColumn = "NomineeID", KeyLabels = new[] { "Nomination Purpose", "Nominee Name" },
Fields = new List<SaveField>
{
new SaveField("Nomination Purpose", "NominationPurposeID", Number, "NominationPurposeID"),
new SaveField("Nominee Name", "Name", Text, "Name"),
new SaveField("Relation", "RelationID", Number, "RelationID"),
new SaveField("Percentage", "Percentage", Number, "Percentage"),
new SaveField("Birth Date", "BirthDate", Day, "BirthDate"),
new SaveField("Occupation", "OccupationID", Number, "OccupationID"),
new SaveField("Address", "Address", Text, "Address"),
new SaveField("Telephone", "TelePhone", Text, "TelePhone"),
new SaveField("Email", "EmailAddress", Text, "EmailAddress")
}
}
},
{
"Experience", new RepeatTable
{
Table = "EmpExperience", IdColumn = "ExperienceID", KeyLabels = new[] { "Employer", "From Date" },
Fields = new List<SaveField>
{
new SaveField("Employer", "Employer", Text, "Employer"),
new SaveField("Business Type", "EmployerActivity", Text, "EmployerActivity"),
new SaveField("Employer Contact Person", "ContactPerson", Text, "ContactPerson"),
new SaveField("Designation", "Designation", Text, "Designation"),
new SaveField("Address", "Address", Text, "Address"),
new SaveField("Role Definition", "JobDescription", Text, "JobDescription"),
new SaveField("Telephone No", "Telephone", Text, "Telephone"),
new SaveField("From Date", "FromDate", Day, "FromDate"),
new SaveField("To Date", "ToDate", Day, "ToDate")
}
}
},
{
"Training", new RepeatTable
{
Table = "EmpTraining", IdColumn = "EmpTrainingID", KeyLabels = new[] { "Training Title", "From Date" },
Fields = new List<SaveField>
{
new SaveField("Training Type", "TrainingTypeID", Number, "TrainingTypeID"),
new SaveField("Nature of Training", "NatureOfTrainingID", Number, "NatureOfTrainingID"),
new SaveField("Training Title", "Name", Text, "Name"),
new SaveField("Description", "Description", Text, "Description"),
new SaveField("Institution", "InstitutionID", Number, "InstitutionID"),
new SaveField("Institute Place", "TrainingPlace", Text, "Place"),
new SaveField("Country", "CountryID", Number, "CountryID"),
new SaveField("Achievement", "Achievement", Text, "Achievement"),
new SaveField("From Date", "FromDate", Day, "FromDate"),
new SaveField("To Date", "ToDate", Day, "ToDate"),
new SaveField("Sponsor Type", "SponsoredType", Number, "SponsoredType") { AfterInsert = true },
new SaveField("Months", "TrainingMonth", Number, "TrainingMonth"),
new SaveField("Days", "TrainingDay", Number, "TrainingDay"),
new SaveField("Hours", "TrainingHour", Number, "TrainingHour"),
new SaveField("Training Fees", "Fees", Number, "Fees"),
new SaveField("Other Costs", "OtherCost", Number, "OtherCost")
}
}
}
};
/// <summary>Thrown inside the save transaction; its message already names the row.</summary>
private class BulkSaveException : Exception
{
public BulkSaveException(string message, Exception inner) : base(message, inner)
{
}
}
public EmployeeBulkUploadSaveResult SaveUpload(Stream stream, string fileName, long fileLength, List<string> requestedSubGroups, int payrollTypeID, int userID, bool closeMatchesConfirmed,
List<EmployeeBulkUploadReplacement> replacements = null)
{
try
{
EmployeeBulkUploadSaveResult save = new EmployeeBulkUploadSaveResult();
// The file is validated again here — the server never saves on the page's word alone.
EmployeeBulkUploadReadResult result = ReadUpload(stream, fileName, fileLength, requestedSubGroups, payrollTypeID,
replacements);
if (result.FailedLayerNo != 0)
{
save.Message = "The file does not pass validation phase " + result.FailedLayerNo
+ " any more, so nothing was saved. The result below shows why.";
save.Validation = result;
return save;
}
if (result.CloseMatches.Count > 0 && !closeMatchesConfirmed)
{
save.Message = "Some values were taken as close matches. Confirm them before saving — nothing was saved.";
save.Validation = result;
return save;
}
if (userID <= 0)
{
throw new ArgumentException("The current user is not known. Log in again.");
}
List<EmployeeBulkUploadSubGroup> sections = result.Groups.SelectMany(g => g.SubGroups).ToList();
EmployeeBulkUploadSubGroup employeeSection = sections.First(s => s.Name == EmployeeBulkUploadTemplate.AlwaysIncludedSubGroup);
string employeeNoKey = employeeSection.Columns[0].Key;
DateTime today = DateTime.Today;
List<KeyValuePair<int, EmployeeBulkUploadRow>> withNominees = new List<KeyValuePair<int, EmployeeBulkUploadRow>>();
// All rows in ONE transaction: every row is saved, or none is.
TransactionContext tc = null;
try
{
tc = TransactionContext.Begin(true);
foreach (EmployeeBulkUploadRow row in result.Rows)
{
string employeeNo = ValueOf(row, employeeNoKey).Trim();
try
{
int employeeID = SaveRow(tc, sections, row, employeeNo, payrollTypeID, userID, today, save);
if (sections.Any(s => s.Name.StartsWith("Nominee", StringComparison.OrdinalIgnoreCase)))
{
withNominees.Add(new KeyValuePair<int, EmployeeBulkUploadRow>(employeeID, row));
}
}
catch (Exception e)
{
throw new BulkSaveException("Row " + row.RowNo + " (Employee No " + employeeNo + "): "
+ Innermost(e).Message, e);
}
}
// The same rule SaveEmpNominee keeps: one purpose's shares add up to 100% at most —
// checked on what the employee has after the upload, existing nominees included.
foreach (KeyValuePair<int, EmployeeBulkUploadRow> item in withNominees)
{
using (IDataReader reader = EmployeeBulkUploadDA.GetNomineeShareOverflow(tc, item.Key))
{
if (reader.Read())
{
throw new BulkSaveException("Row " + item.Value.RowNo + " (Employee No "
+ ValueOf(item.Value, employeeNoKey).Trim() + "): with the nominees already in the system, "
+ "the shares of one nomination purpose would add up to "
+ Convert.ToDouble(reader["TotalPercentage"]).ToString("0.##", CultureInfo.InvariantCulture)
+ "% — more than 100%. Correct the shares in the file or in the employee's profile.", null);
}
}
}
tc.End();
}
catch (Exception e)
{
if (tc != null)
{
tc.HandleError();
}
ExceptionLog.Write(e);
return new EmployeeBulkUploadSaveResult
{
Saved = false,
Message = "Nothing was saved — every change was rolled back. "
+ (e is BulkSaveException ? e.Message : Innermost(e).Message)
};
}
save.Saved = true;
save.Message = save.CreatedCount + " new employee" + (save.CreatedCount == 1 ? "" : "s") + " created and "
+ save.UpdatedCount + " employee" + (save.UpdatedCount == 1 ? "" : "s") + " updated.";
return save;
}
catch (Exception e)
{
#region Handle Exception
ExceptionLog.Write(e);
throw new ServiceException(e.Message, e);
#endregion
}
}
private static Exception Innermost(Exception e)
{
while (e.InnerException != null)
{
e = e.InnerException;
}
return e;
}
/// <summary>
/// Saves one row inside the caller's transaction and returns the employee's ID. A new
/// employee is inserted the way the employee profile creates one (HREmployee through
/// HREmployeeDA.Insert, current payroll type, created by the user today); an existing one
/// gets ONLY the columns the file fills. Contact sections go to the employee's one
/// EmpContact row; each filled repeat block adds a record or updates the matching one.
/// </summary>
private int SaveRow(TransactionContext tc, List<EmployeeBulkUploadSubGroup> sections, EmployeeBulkUploadRow row,
string employeeNo, int payrollTypeID, int userID, DateTime today, EmployeeBulkUploadSaveResult save)
{
// Only an active employee of this payroll type is updated, and only a new Employee No creates
// one. Phase 7 refuses every other row; this guard makes sure no other path saves one.
if (row.EmployeeState != EmployeeStates.Existing && row.EmployeeState != EmployeeStates.New)
{
throw new InvalidOperationException("Employee No \"" + employeeNo + "\" is "
+ (row.EmployeeState == EmployeeStates.Inactive ? "not an active employee"
: row.EmployeeState == EmployeeStates.OtherPayroll ? "an employee of another payroll type"
: "not checked")
+ ", so it cannot be saved here.");
}
bool isNew = row.EmployeeState == EmployeeStates.New;
List<string> saved = new List<string>();
int employeeID;
// Employee table: Employee, General, Extended Personal Information.
List<KeyValuePair<SaveField, object>> employeeValues = FilledFields(row, sections, EmployeeFields);
ComposeName(employeeValues);
if (isNew)
{
// A new employee is created only from a file that holds General (joining date,
// category, grade, ...). Phase 7 already refuses such a file; this guard makes sure
// no other path can create an employee without it.
if (!sections.Any(s => s.Name == "General"))
{
throw new InvalidOperationException("Employee No \"" + employeeNo + "\" is not an employee of the "
+ "current payroll type, and a new employee can be created only when the General section is in the file.");
}
HREmployee employee = new HREmployee
{
EmployeeNo = employeeNo,
PayrollTypeID = payrollTypeID,
CreatedBy = userID,
CreatedDate = today
};
foreach (KeyValuePair<SaveField, object> item in employeeValues)
{
SetProperty(employee, item.Key.Property, item.Value);
}
if (employee.NationalityID.GetValueOrDefault() <= 0)
{
int bangladeshi = DefaultNationalityID(tc);
if (bangladeshi > 0)
{
employee.NationalityID = bangladeshi;
}
}
employee.ID = HREmployeeDA.GetNewID(tc);
HREmployeeDA.Insert(tc, employee);
employeeID = employee.ID;
ClearEmptyDates(tc, "Employee", "EmployeeID", employeeID,
EmployeeFields.Values.SelectMany(x => x), employeeValues.Select(x => x.Key));
}
else
{
employeeID = row.EmployeeID;
List<KeyValuePair<string, KeyValuePair<EmployeeBulkUploadDA.SqlKind, object>>> columns = Columns(
employeeValues.Where(x => !x.Key.NewOnly));
if (columns.Count > 0)
{
columns.Add(Column("ModifiedBy", Number, userID));
columns.Add(Column("ModifiedDate", Day, today));
EmployeeBulkUploadDA.UpdateColumns(tc, "Employee", "EmployeeID", employeeID, columns);
}
}
saved.AddRange(sections.Where(s => EmployeeFields.ContainsKey(s.Name)
&& s.Name != EmployeeBulkUploadTemplate.AlwaysIncludedSubGroup).Select(s => s.Name));
// EmpContact: one row per employee holds the addresses, contact and emergency details.
List<KeyValuePair<SaveField, object>> contactValues = FilledFields(row, sections, ContactFields);
if (contactValues.Count > 0)
{
int contactID = isNew ? 0 : EmployeeBulkUploadDA.GetContactID(tc, employeeID);
if (contactID > 0)
{
EmployeeBulkUploadDA.UpdateColumns(tc, "EmpContact", "ContactID", contactID, Columns(contactValues));
}
else
{
EmpContact contact = new EmpContact { EmployeeID = employeeID };
foreach (KeyValuePair<SaveField, object> item in contactValues)
{
SetProperty(contact, item.Key.Property, item.Value);
}
contact.ID = HREmployeeDA.GetNewID(tc, "EmpContact", "ContactID");
HREmployeeDA.Insert(tc, contact);
ClearEmptyDates(tc, "EmpContact", "ContactID", contact.ID,
ContactFields.Values.SelectMany(x => x), contactValues.Select(x => x.Key));
}
saved.AddRange(sections.Where(s => ContactFields.ContainsKey(s.Name)).Select(s => s.Name));
}
// Repeat blocks: add a record, or update the employee's record with the same key.
Dictionary<string, List<KeyValuePair<int, string>>> existing = new Dictionary<string, List<KeyValuePair<int, string>>>();
foreach (EmployeeBulkUploadSubGroup section in sections.Where(s => s.IsRepeatBlock))
{
RepeatTable table = RepeatTables.First(x => section.Name.StartsWith(x.Key, StringComparison.OrdinalIgnoreCase)).Value;
List<KeyValuePair<SaveField, object>> values = FilledFields(row, section, table.Fields);
if (values.Count == 0)
{
continue; // an empty block: this employee has no such record
}
List<KeyValuePair<int, string>> records;
if (!existing.TryGetValue(table.Table, out records))
{
records = isNew ? new List<KeyValuePair<int, string>>() : LoadRecordKeys(tc, table, employeeID);
existing[table.Table] = records;
}
string fullKey = RecordKey(table.KeyLabels, values, true);
string shortKey = RecordKey(table.KeyLabels, values, false);
bool fileHasSecondKey = table.KeyLabels.Length > 1 && values.Any(x => x.Key.Label == table.KeyLabels[1]);
KeyValuePair<int, string> match = records.FirstOrDefault(x => fileHasSecondKey
? x.Value == fullKey
: x.Value == fullKey || x.Value.Split('|')[0] == shortKey);
if (match.Key > 0)
{
EmployeeBulkUploadDA.UpdateColumns(tc, table.Table, table.IdColumn, match.Key, Columns(values));
}
else
{
int id = InsertRepeatRecord(tc, table, employeeID, values, today);
records.Add(new KeyValuePair<int, string>(id, fullKey));
List<KeyValuePair<string, KeyValuePair<EmployeeBulkUploadDA.SqlKind, object>>> after =
Columns(values.Where(x => x.Key.AfterInsert));
EmployeeBulkUploadDA.UpdateColumns(tc, table.Table, table.IdColumn, id, after);
}
saved.Add(section.Name);
}
// The employee's profile completion %, worked out again from what is now saved — the same
// procedure the profile-update approval runs, in the same transaction.
ProfileUpdateRequestDA.UpdateProfilePercentage(tc, employeeID);
if (isNew)
{
save.CreatedCount++;
}
else
{
save.UpdatedCount++;
}
save.Lines.Add("Row " + row.RowNo + " · " + employeeNo + " — " + (isNew ? "created" : "updated")
+ (saved.Count > 0 ? ": " + string.Join(", ", saved.Distinct()) : string.Empty));
return employeeID;
}
/// <summary>The "Bangladeshi" nationality (active), looked up once per save; 0 when there is none.</summary>
private int? defaultNationalityID;
/// <summary>
/// The nationality a new employee gets when the file leaves Nationality empty — Bangladeshi,
/// the one the Employee Profile form preselects for a new employee.
/// </summary>
private int DefaultNationalityID(TransactionContext tc)
{
if (!defaultNationalityID.HasValue)
{
defaultNationalityID = 0;
DataReader dr = new DataReader(NationalityDA.Get(tc));
try
{
while (dr.Read())
{
if (string.Equals(dr.GetString("Description", string.Empty).Trim(), "Bangladeshi", StringComparison.OrdinalIgnoreCase)
&& dr.GetInt32("Status", 0) != (int)EnumStatus.Inactive)
{
defaultNationalityID = dr.GetInt32("NationalityID", 0);
break;
}
}
}
finally
{
dr.Close();
}
}
return defaultNationalityID.Value;
}
/// <summary>
/// With General in the file the saved Name is First + Middle + Last Name, as the Employee
/// Profile form saves it; the Employee Name column then only tells the rows apart.
/// Without General (no First Name) the Employee Name is saved as before.
/// </summary>
private static void ComposeName(List<KeyValuePair<SaveField, object>> values)
{
Func<string, string> part = label =>
{
KeyValuePair<SaveField, object> item = values.FirstOrDefault(x => x.Key.Label == label && x.Key.Column != "Name");
return item.Key == null ? string.Empty : Convert.ToString(item.Value, CultureInfo.InvariantCulture).Trim();
};
string first = part("First Name");
if (first.Length == 0)
{
return;
}
string name = string.Join(" ", new[] { first, part("Middle Name"), part("Last Name") }.Where(x => x.Length > 0));
int index = values.FindIndex(x => x.Key.Column == "Name");
if (index >= 0)
{
values[index] = new KeyValuePair<SaveField, object>(values[index].Key, name);
}
}
/// <summary>Inserts one repeat record through HREmployeeDA.Insert — the insert the profile screens use.</summary>
private int InsertRepeatRecord(TransactionContext tc, RepeatTable table, int employeeID,
List<KeyValuePair<SaveField, object>> values, DateTime today)
{
int id = HREmployeeDA.GetNewID(tc, table.Table, table.IdColumn);
switch (table.Table)
{
case "EmpChildren":
{
EmpChildren record = new EmpChildren { EmployeeID = employeeID };
Fill(record, values);
record.ID = id;
HREmployeeDA.Insert(tc, record);
break;
}
case "EmpSpouse":
{
EmpSpouse record = new EmpSpouse { EmployeeID = employeeID };
Fill(record, values);
record.ID = id;
HREmployeeDA.Insert(tc, record);
break;
}
case "EmpAcademic":
{
EmpAcademic record = new EmpAcademic { EmployeeID = employeeID };
Fill(record, values);
record.ID = id;
HREmployeeDA.Insert(tc, record);
break;
}
case "EmpNominee":
{
EmpNominee record = new EmpNominee { EmployeeID = employeeID, NominationDate = today };
Fill(record, values);
record.ID = id;
HREmployeeDA.Insert(tc, record);
break;
}
case "EmpExperience":
{
EmpExperience record = new EmpExperience { EmployeeID = employeeID };
Fill(record, values);
record.ID = id;
HREmployeeDA.Insert(tc, record);
break;
}
case "EmpTraining":
{
EmpTraining record = new EmpTraining { EmployeeID = employeeID };
Fill(record, values);
record.ID = id;
HREmployeeDA.Insert(tc, record);
break;
}
default:
throw new InvalidOperationException("No insert for " + table.Table + ".");
}
ClearEmptyDates(tc, table.Table, table.IdColumn, id, table.Fields, values.Select(x => x.Key));
return id;
}
private static void Fill(object record, List<KeyValuePair<SaveField, object>> values)
{
foreach (KeyValuePair<SaveField, object> item in values)
{
SetProperty(record, item.Key.Property, item.Value);
}
}
/// <summary>The employee's records of one repeat table, as (ID, key) — the key built like <see cref="RecordKey"/>.</summary>
private List<KeyValuePair<int, string>> LoadRecordKeys(TransactionContext tc, RepeatTable table, int employeeID)
{
List<string> keyColumns = table.KeyLabels
.Select(label => table.Fields.First(f => f.Label == label).Column)
.ToList();
List<KeyValuePair<int, string>> records = new List<KeyValuePair<int, string>>();
using (IDataReader reader = EmployeeBulkUploadDA.GetRecords(tc, table.Table, table.IdColumn, keyColumns, employeeID))
{
while (reader.Read())
{
records.Add(new KeyValuePair<int, string>(Convert.ToInt32(reader["RecordID"]),
string.Join("|", keyColumns.Select(c => KeyPart(reader[c])))));
}
}
return records;
}
/// <summary>The key of a repeat block from the file: its key values joined by "|" (all of them, or only the first).</summary>
private static string RecordKey(string[] keyLabels, List<KeyValuePair<SaveField, object>> values, bool all)
{
return string.Join("|", keyLabels.Take(all ? keyLabels.Length : 1)
.Select(label => KeyPart(values.Where(x => x.Key.Label == label).Select(x => x.Value).FirstOrDefault())));
}
/// <summary>One key value made comparable: text by <see cref="MasterKey"/>, dates by day, numbers as integers.</summary>
private static string KeyPart(object value)
{
if (value == null || value == DBNull.Value)
{
return string.Empty;
}
if (value is DateTime)
{
return ((DateTime)value).ToString("yyyyMMdd", CultureInfo.InvariantCulture);
}
if (value is string)
{
return MasterKey((string)value);
}
return Convert.ToInt64(value, CultureInfo.InvariantCulture).ToString(CultureInfo.InvariantCulture);
}
/// <summary>The filled fields of every present section that <paramref name="map"/> covers.</summary>
private List<KeyValuePair<SaveField, object>> FilledFields(EmployeeBulkUploadRow row,
List<EmployeeBulkUploadSubGroup> sections, Dictionary<string, List<SaveField>> map)
{
List<KeyValuePair<SaveField, object>> values = new List<KeyValuePair<SaveField, object>>();
foreach (EmployeeBulkUploadSubGroup section in sections.Where(s => map.ContainsKey(s.Name)))
{
values.AddRange(FilledFields(row, section, map[section.Name]));
}
return values;
}
/// <summary>The filled fields of one section, each with the value to save.</summary>
private List<KeyValuePair<SaveField, object>> FilledFields(EmployeeBulkUploadRow row,
EmployeeBulkUploadSubGroup section, List<SaveField> fields)
{
List<KeyValuePair<SaveField, object>> values = new List<KeyValuePair<SaveField, object>>();
foreach (SaveField field in fields)
{
EmployeeBulkUploadColumn column = section.Columns.FirstOrDefault(c => c.Label == field.Label);
if (column == null)
{
throw new InvalidOperationException("The template has no \"" + field.Label + "\" in " + section.Name + ".");
}
object value;
if (TryGetSaveValue(row, column, out value))
{
values.Add(new KeyValuePair<SaveField, object>(field, value));
}
}
return values;
}
/// <summary>
/// A filled cell's value as it is saved: the resolved record ID for master data, the enum
/// number for a dropdown, an int / double for numbers, a DateTime for a date, else the text.
/// False when the cell is empty.
/// </summary>
private static bool TryGetSaveValue(EmployeeBulkUploadRow row, EmployeeBulkUploadColumn column, out object value)
{
value = null;
if (column.IsMasterData)
{
int id;
if (!row.MasterIDs.TryGetValue(column.Key, out id))
{
return false;
}
value = id;
return true;
}
object parsed;
if (!row.Parsed.TryGetValue(column.Key, out parsed) || parsed == null)
{
return false;
}
switch (column.Kind)
{
case EnumBulkUploadColumnKind.Gender:
value = (int)(EnumGender)Enum.Parse(typeof(EnumGender), (string)parsed, true);
break;
case EnumBulkUploadColumnKind.MaritalStatus:
value = (int)(EnumMaritalStatus)Enum.Parse(typeof(EnumMaritalStatus), (string)parsed, true);
break;
case EnumBulkUploadColumnKind.TaxCircle:
value = (int)(EnumTaxCircle)Enum.Parse(typeof(EnumTaxCircle), (string)parsed, true);
break;
case EnumBulkUploadColumnKind.SponsoredType:
value = (int)(EnumSponsoredType)Enum.Parse(typeof(EnumSponsoredType), (string)parsed, true);
break;
case EnumBulkUploadColumnKind.BloodGroup:
// EnumBloodGroup: A+ = 1 … AB- = 8, the order of the dropdown.
value = Array.FindIndex(EmployeeBulkUploadColumn.BloodGroups.Split(','),
x => string.Equals(x, (string)parsed, StringComparison.OrdinalIgnoreCase)) + 1;
break;
case EnumBulkUploadColumnKind.Integer:
value = (int)(decimal)parsed;
break;
case EnumBulkUploadColumnKind.Decimal:
case EnumBulkUploadColumnKind.Percentage:
value = (double)(decimal)parsed;
break;
default:
value = parsed;
break;
}
return true;
}
private static List<KeyValuePair<string, KeyValuePair<EmployeeBulkUploadDA.SqlKind, object>>> Columns(
IEnumerable<KeyValuePair<SaveField, object>> values)
{
return values.Select(x => Column(x.Key.Column, x.Key.Kind, x.Value)).ToList();
}
private static KeyValuePair<string, KeyValuePair<EmployeeBulkUploadDA.SqlKind, object>> Column(string column,
EmployeeBulkUploadDA.SqlKind kind, object value)
{
return new KeyValuePair<string, KeyValuePair<EmployeeBulkUploadDA.SqlKind, object>>(column,
new KeyValuePair<EmployeeBulkUploadDA.SqlKind, object>(kind, value));
}
/// <summary>
/// A date the file left empty has to stay empty. The record is created through the entity's
/// own Insert, which writes every one of its date properties; one the file never filled is
/// still DateTime.MinValue, so the column ends up holding 01-Jan-0001 instead of nothing.
/// Right after an insert this puts those columns back to NULL.
///
/// Only the date columns of <paramref name="tableFields"/> — the ones this upload itself
/// owns — are touched, so a date the code sets deliberately (EmpNominee.NominationDate,
/// CreatedDate, ModifiedDate) is left alone, and a mandatory date is always among the
/// filled ones. Never called for an update: an update writes only the cells the file
/// fills and must leave the rest of an existing record as it is.
/// </summary>
private static void ClearEmptyDates(TransactionContext tc, string table, string idColumn, int id,
IEnumerable<SaveField> tableFields, IEnumerable<SaveField> filled)
{
HashSet<string> written = new HashSet<string>(filled.Select(x => x.Column), StringComparer.OrdinalIgnoreCase);
List<KeyValuePair<string, KeyValuePair<EmployeeBulkUploadDA.SqlKind, object>>> columns = tableFields
.Where(f => f.Kind == Day && !written.Contains(f.Column))
.Select(f => f.Column)
.Distinct(StringComparer.OrdinalIgnoreCase)
.Select(column => Column(column, Day, null))
.ToList();
if (columns.Count > 0)
{
EmployeeBulkUploadDA.UpdateColumns(tc, table, idColumn, id, columns);
}
}
/// <summary>Sets a BO property from a save value, converting to the property's type (enums, nullables).</summary>
private static void SetProperty(object record, string property, object value)
{
PropertyInfo info = record.GetType().GetProperty(property, BindingFlags.Public | BindingFlags.Instance);
if (info == null || !info.CanWrite)
{
throw new InvalidOperationException(record.GetType().Name + " has no writable property " + property + ".");
}
Type type = Nullable.GetUnderlyingType(info.PropertyType) ?? info.PropertyType;
object converted = type.IsEnum
? Enum.ToObject(type, Convert.ToInt32(value, CultureInfo.InvariantCulture))
: Convert.ChangeType(value, type, CultureInfo.InvariantCulture);
info.SetValue(record, converted, null);
}
#endregion
#region Adding master data
public List<EmployeeBulkUploadMasterOption> GetMasterOptions(EnumBulkUploadMasterData masterData, int payrollTypeID)
{
try
{
if (masterData == EnumBulkUploadMasterData.Employee || masterData == EnumBulkUploadMasterData.None)
{
throw new ArgumentException("No option list for " + masterData + ".");
}
return LoadMaster(masterData, payrollTypeID)
.Where(x => !x.IsInactive && !string.IsNullOrWhiteSpace(x.Name))
.OrderBy(x => x.Name, StringComparer.OrdinalIgnoreCase)
.Select(x => new EmployeeBulkUploadMasterOption { ID = x.ID, Name = x.Name.Trim(), ParentID = x.ParentID })
.ToList();
}
catch (Exception e)
{
#region Handle Exception
ExceptionLog.Write(e);
throw new ServiceException(e.Message, e);
#endregion
}
}
/// <summary>
/// Each value is checked again against the table as it is now (someone may have added it
/// meanwhile, or the same value may be in the request twice) and then saved through the
/// table's own service Save — the path the master screens use — each in its own
/// transaction. Values are processed independently: one failure is reported and the rest
/// go on.
/// </summary>
public List<EmployeeBulkUploadMasterAddResult> AddMasterData(EmployeeBulkUploadMasterAddRequest request,
int payrollTypeID, int userID)
{
try
{
if (request == null || request.Items == null || request.Items.Count == 0)
{
throw new ArgumentException("Nothing to add.");
}
EnumBulkUploadMasterData masterData = request.MasterData;
if (masterData == EnumBulkUploadMasterData.None || masterData == EnumBulkUploadMasterData.Division
|| !Enum.IsDefined(typeof(EnumBulkUploadMasterData), masterData))
{
throw new ArgumentException("Unknown master data.");
}
string cannot = EmployeeBulkUploadMasterInfo.CannotAddReason(masterData);
if (cannot != null)
{
throw new ArgumentException(cannot);
}
if (payrollTypeID <= 0)
{
throw new ArgumentException("The current payroll type is not known. Log in again and retry.");
}
string masterName = EmployeeBulkUploadMasterInfo.DisplayName(masterData);
EnumBulkUploadMasterData parentType = masterData == EnumBulkUploadMasterData.District
? EnumBulkUploadMasterData.Division
: EmployeeBulkUploadMasterInfo.ParentOf(masterData);
string parentName = parentType == EnumBulkUploadMasterData.None
? null : EmployeeBulkUploadMasterInfo.DisplayName(parentType);
List<MasterRecord> records = LoadMaster(masterData, payrollTypeID);
List<MasterRecord> parents = parentType == EnumBulkUploadMasterData.None
? null : LoadMaster(parentType, payrollTypeID);
List<EmployeeBulkUploadMasterAddResult> results = new List<EmployeeBulkUploadMasterAddResult>();
foreach (EmployeeBulkUploadMasterAddItem item in request.Items)
{
EmployeeBulkUploadMasterAddResult outcome = new EmployeeBulkUploadMasterAddResult
{
Value = (item.Value ?? string.Empty).Trim(),
ParentValue = item.ParentValue
};
results.Add(outcome);
if (outcome.Value.Length == 0)
{
outcome.Message = "The value is empty.";
continue;
}
// The parent: from the file by name, else the one picked in the dialog.
int parentID = 0;
if (parents != null)
{
MasterRecord parent = null;
if (!string.IsNullOrWhiteSpace(item.ParentValue))
{
List<MasterRecord> byName = FindRecords(parents, parentType, MasterKey(item.ParentValue), 0);
parent = byName.FirstOrDefault(x => !x.IsInactive) ?? byName.FirstOrDefault();
if (parent == null)
{
outcome.Message = parentName + " \"" + item.ParentValue.Trim() + "\" is not in the system yet. "
+ "Add it first, then add this " + masterName + ".";
continue;
}
}
else if (item.ParentID > 0)
{
parent = parents.FirstOrDefault(x => x.ID == item.ParentID);
if (parent == null)
{
outcome.Message = "The chosen " + parentName + " was not found. Pick it again.";
continue;
}
}
else
{
outcome.Message = "Pick the " + parentName + " this " + masterName + " belongs to.";
continue;
}
parentID = parent.ID;
}
List<MasterRecord> existing = FindRecords(records, masterData, MasterKey(outcome.Value), parentID);
if (existing.Count > 0)
{
outcome.AlreadyExisted = true;
outcome.ID = existing[0].ID;
outcome.Message = existing.Any(x => !x.IsInactive)
? "Already in the system — not added again."
: "Already in the system but inactive — activate it in the " + masterName + " screen.";
continue;
}
string invalid = CheckAddExtras(masterData, item);
if (invalid != null)
{
outcome.Message = invalid;
continue;
}
try
{
outcome.ID = SaveMasterRecord(masterData, outcome.Value, parentID, item, payrollTypeID, userID);
outcome.Saved = true;
outcome.Message = "Added.";
string stampProblem = StampPayrollType(masterData, outcome.ID, payrollTypeID);
if (stampProblem != null)
{
outcome.Message = "Added, but its payroll type could not be set: " + stampProblem;
}
records.Add(new MasterRecord { ID = outcome.ID, Name = outcome.Value, ParentID = parentID });
}
catch (Exception e)
{
// The table's service has already logged it; report it on this value only.
Exception inner = e;
while (inner.InnerException != null)
{
inner = inner.InnerException;
}
outcome.Message = "Could not be added: " + inner.Message;
}
}
return results;
}
catch (Exception e)
{
#region Handle Exception
ExceptionLog.Write(e);
throw new ServiceException(e.Message, e);
#endregion
}
}
/// <summary>
/// The master tables that have a PayrollTypeID column (checked against the database schema):
/// table name and ID column. A record added to one of them belongs to the current payroll type.
/// </summary>
private static readonly Dictionary<EnumBulkUploadMasterData, KeyValuePair<string, string>> PayrollTypeTables =
new Dictionary<EnumBulkUploadMasterData, KeyValuePair<string, string>>
{
{ EnumBulkUploadMasterData.Category, new KeyValuePair<string, string>("Category", "CategoryID") },
{ EnumBulkUploadMasterData.Religion, new KeyValuePair<string, string>("Religion", "ReligionID") },
{ EnumBulkUploadMasterData.Grades, new KeyValuePair<string, string>("Grades", "GradeID") }
};
/// <summary>
/// Sets the PayrollTypeID of a record just added to a table that has one — from the login's
/// payroll type, on the server. Needed because not every table's own Save writes it:
/// ReligionDA.Insert leaves it to the column default (1), whatever the user's payroll type.
/// Returns null when done (or when the table has no PayrollTypeID), else what went wrong.
/// </summary>
private static string StampPayrollType(EnumBulkUploadMasterData masterData, int id, int payrollTypeID)
{
KeyValuePair<string, string> table;
if (!PayrollTypeTables.TryGetValue(masterData, out table))
{
return null;
}
TransactionContext tc = null;
try
{
tc = TransactionContext.Begin(true);
EmployeeBulkUploadDA.UpdateColumns(tc, table.Key, table.Value, id,
new[] { Column("PayrollTypeID", Number, payrollTypeID) });
tc.End();
return null;
}
catch (Exception e)
{
if (tc != null)
{
tc.HandleError();
}
ExceptionLog.Write(e);
return Innermost(e).Message;
}
}
/// <summary>What the table needs besides the name — the same fields its master screen asks for.</summary>
private static string CheckAddExtras(EnumBulkUploadMasterData masterData, EmployeeBulkUploadMasterAddItem item)
{
switch (masterData)
{
case EnumBulkUploadMasterData.EducationLevel:
case EnumBulkUploadMasterData.Institution:
if (!item.BoardRequired.HasValue)
{
return "Choose Board Required or Fixed Subject.";
}
if (masterData == EnumBulkUploadMasterData.Institution && (item.InstituteType < 1 || item.InstituteType > 3))
{
return "Choose whether the institution is Academic, Institutional, or both.";
}
return null;
case EnumBulkUploadMasterData.ResultType:
return item.ResultType < 1 || item.ResultType > 3 ? "Choose GPA/CGPA, Marks or Pass/Fail." : null;
default:
return null;
}
}
/// <summary>
/// The next code the table's own GetNextCode gives — what its master screen fills in —
/// or empty when that fails (e.g. existing codes that are not numbers); an empty code
/// makes the services that allow it use the new record's ID.
/// </summary>
private static string NextCode(Func<string> getNextCode)
{
try
{
return getNextCode() ?? string.Empty;
}
catch (Exception)
{
return string.Empty;
}
}
/// <summary>
/// Builds the record the way BasicBuilderController.SaveItem does for the master screens —
/// created by the user today, active, the current payroll type where the table has one —
/// and saves it through the table's own service. Returns the new ID.
/// </summary>
private static int SaveMasterRecord(EnumBulkUploadMasterData masterData, string value, int parentID,
EmployeeBulkUploadMasterAddItem item, int payrollTypeID, int userID)
{
DateTime today = DateTime.Today;
switch (masterData)
{
case EnumBulkUploadMasterData.Category:
{
CategoryService service = new CategoryService();
Category o = new Category { Code = NextCode(service.GetNextCode), Name = value, PayrollTypeID = payrollTypeID };
Stamp(o, userID, today);
return service.Save(o);
}
case EnumBulkUploadMasterData.Religion:
{
ReligionService service = new ReligionService();
Religion o = new Religion { Code = NextCode(service.GetNextCode), Name = value };
Stamp(o, userID, today);
return service.Save(o);
}
case EnumBulkUploadMasterData.Nationality:
{
NationalityService service = new NationalityService();
Nationality o = new Nationality { Code = NextCode(service.GetNextCode), Description = value };
Stamp(o, userID, today);
return service.Save(o);
}
case EnumBulkUploadMasterData.Occupation:
{
OccupationService service = new OccupationService();
Occupation o = new Occupation { Code = NextCode(service.GetNextCode), Description = value };
Stamp(o, userID, today);
return service.Save(o);
}
case EnumBulkUploadMasterData.District:
{
DistrictService service = new DistrictService();
District o = new District { Code = NextCode(service.GetNextCode), Name = value, DivisionID = parentID };
Stamp(o, userID, today);
return service.Save(o);
}
case EnumBulkUploadMasterData.Thana:
{
ThanaService service = new ThanaService();
Thana o = new Thana { Code = NextCode(service.GetNextCode), Name = value, DistrictID = parentID };
Stamp(o, userID, today);
return service.Save(o);
}
case EnumBulkUploadMasterData.PersonRelation:
{
RelationService service = new RelationService();
Relation o = new Relation { Code = NextCode(service.GetNextCode), Description = value };
Stamp(o, userID, today);
return service.Save(o);
}
case EnumBulkUploadMasterData.EducationType:
{
EducationTypeService service = new EducationTypeService();
EducationType o = new EducationType { Code = NextCode(service.GetNextCode), Description = value };
Stamp(o, userID, today);
return service.Save(o);
}
case EnumBulkUploadMasterData.EducationLevel:
{
EducationLevelService service = new EducationLevelService();
EducationLevel o = new EducationLevel
{
Code = NextCode(service.GetNextCode),
Description = value,
EducationTypeID = parentID,
BoardRequired = item.BoardRequired.Value,
FixedSubject = !item.BoardRequired.Value
};
Stamp(o, userID, today);
return service.Save(o);
}
case EnumBulkUploadMasterData.Institution:
{
InstitutionService service = new InstitutionService();
Institution o = new Institution
{
Code = NextCode(service.GetNextCode),
Name = value,
Type = (EnmInstituteType)item.InstituteType,
BoardRequired = item.BoardRequired.Value,
FixedSubject = !item.BoardRequired.Value
};
Stamp(o, userID, today);
return service.Save(o);
}
case EnumBulkUploadMasterData.Discipline:
{
DisciplineService service = new DisciplineService();
Discipline o = new Discipline
{
Code = NextCode(service.GetNextCode),
Description = value,
EducationLevelID = parentID
};
Stamp(o, userID, today);
return service.Save(o);
}
case EnumBulkUploadMasterData.ResultType:
{
ResultTypeService service = new ResultTypeService();
ResultType o = new ResultType
{
Code = NextCode(service.GetNextCode),
Description = value,
Type = (EnmResultType)item.ResultType
};
Stamp(o, userID, today);
return service.Save(o);
}
case EnumBulkUploadMasterData.NominationPurpose:
{
NominationPurpose o = new NominationPurpose { Code = string.Empty, Description = value };
Stamp(o, userID, today);
return new NominationPurposeService().Save(o);
}
case EnumBulkUploadMasterData.TrainingType:
{
TrainingTypeService service = new TrainingTypeService();
TrainingType o = new TrainingType { Code = NextCode(service.GetNextCode), Name = value };
Stamp(o, userID, today);
return service.Save(o);
}
case EnumBulkUploadMasterData.NatureOfTraining:
{
NatureOfTraining o = new NatureOfTraining { Code = string.Empty, Name = value };
Stamp(o, userID, today);
return new NatureOfTrainingService().Save(o);
}
case EnumBulkUploadMasterData.Country:
{
CountryService service = new CountryService();
Country o = new Country { Code = NextCode(service.GetNextCode), Name = value };
Stamp(o, userID, today);
return service.Save(o);
}
default:
throw new ArgumentException(EmployeeBulkUploadMasterInfo.DisplayName(masterData)
+ " cannot be added from the upload page.");
}
}
private static void Stamp(BasicBaseObject record, int userID, DateTime today)
{
record.Status = EnumStatus.Active;
record.CreatedBy = userID;
record.CreatedDate = today;
}
#endregion
/// <summary>A section name with its group, e.g. "Present Address (Basic Information)".</summary>
private string SectionLabel(string subGroupName)
{
EmployeeBulkUploadGroup group = EmployeeBulkUploadTemplate.GetGroups()
.FirstOrDefault(g => g.SubGroups.Any(s => string.Equals(s.Name, subGroupName, StringComparison.OrdinalIgnoreCase)));
return group == null || string.Equals(group.Name, subGroupName, StringComparison.OrdinalIgnoreCase)
? subGroupName
: subGroupName + " (" + group.Name + ")";
}
private string Count(int count, string singular, string plural)
{
return count + " " + (count == 1 ? singular : plural);
}
/// <summary>The text of a row-2 band cell; a merged band carries its text in its first cell.</summary>
private string BandText(IXLCell cell)
{
IXLCell source = cell.IsMerged() ? cell.MergedRange().FirstCell() : cell;
return source.GetString().Trim();
}
/// <summary>A data cell as text: dates as dd-MMM-yyyy, everything else as Excel shows it.</summary>
private string CellText(IXLCell cell)
{
if (cell.IsEmpty())
{
return string.Empty;
}
if (cell.DataType == XLDataType.DateTime)
{
return cell.GetDateTime().ToString("dd-MMM-yyyy");
}
return cell.GetFormattedString().Trim();
}
private string ColumnLetter(int column)
{
return XLHelper.GetColumnLetterFromNumber(column);
}
#endregion
#region Header rows
/// <summary>Writes the three header rows and returns the index of the last column.</summary>
private int BuildHeader(IXLWorksheet sheet, List<EmployeeBulkUploadGroup> groups)
{
XLColor borderColor = XLColor.FromHtml(EmployeeBulkUploadTemplate.BorderColor);
int column = 1;
foreach (EmployeeBulkUploadGroup group in groups)
{
int groupStart = column;
XLColor bandColor = XLColor.FromHtml(group.BandColor);
XLColor subBandColor = XLColor.FromHtml(group.SubBandColor);
XLColor headerColor = XLColor.FromHtml(group.HeaderColor);
XLColor textColor = XLColor.FromHtml(group.TextColor);
foreach (EmployeeBulkUploadSubGroup subGroup in group.SubGroups)
{
int subStart = column;
foreach (EmployeeBulkUploadColumn definition in subGroup.Columns)
{
IXLCell cell = sheet.Cell(EmployeeBulkUploadTemplate.HeadingRow, column);
cell.Value = definition.Heading;
cell.Style.Fill.BackgroundColor = headerColor;
cell.Style.Font.Bold = true;
cell.Style.Font.FontSize = 10;
cell.Style.Font.FontColor = XLColor.FromHtml(definition.IsMandatory
? EmployeeBulkUploadTemplate.MandatoryTextColor
: EmployeeBulkUploadTemplate.OptionalTextColor);
SetHeaderLayout(cell.Style, borderColor);
sheet.Column(column).Width = definition.Width;
column++;
}
int subEnd = column - 1;
if (!string.IsNullOrEmpty(subGroup.Name))
{
IXLRange band = sheet.Range(EmployeeBulkUploadTemplate.SubBandRow, subStart,
EmployeeBulkUploadTemplate.SubBandRow, subEnd);
band.Style.Fill.BackgroundColor = subBandColor;
band.Merge();
SetHeaderLayout(band.Style, borderColor);
IXLCell cell = sheet.Cell(EmployeeBulkUploadTemplate.SubBandRow, subStart);
cell.Value = subGroup.Name;
cell.Style.Font.Bold = true;
cell.Style.Font.FontSize = 11;
cell.Style.Font.FontColor = textColor;
}
}
int groupEnd = column - 1;
int bandLastRow = group.HasSubBands
? EmployeeBulkUploadTemplate.GroupBandRow
: EmployeeBulkUploadTemplate.SubBandRow;
// A group without sub-bands owns both
// header rows, so rows 1 and 2 merge into one tall band.
IXLRange groupBand = sheet.Range(EmployeeBulkUploadTemplate.GroupBandRow, groupStart,
bandLastRow, groupEnd);
groupBand.Style.Fill.BackgroundColor = bandColor;
if (group.HasSubBands)
{
// Row 2 of this group is painted by its own sub-bands above.
groupBand.Style.Fill.BackgroundColor = bandColor;
}
groupBand.Merge();
SetHeaderLayout(groupBand.Style, borderColor);
IXLCell groupCell = sheet.Cell(EmployeeBulkUploadTemplate.GroupBandRow, groupStart);
groupCell.Value = group.Name;
groupCell.Style.Font.Bold = true;
groupCell.Style.Font.FontSize = 14;
groupCell.Style.Font.FontColor = textColor;
}
sheet.Row(EmployeeBulkUploadTemplate.GroupBandRow).Height = 32;
sheet.Row(EmployeeBulkUploadTemplate.SubBandRow).Height = 24;
sheet.Row(EmployeeBulkUploadTemplate.HeadingRow).Height = 22;
return column - 1;
}
/// <summary>
/// Centred, wrapped, with a medium border. For a merged band pass the range's style, so the
/// border runs round the whole band and not just its first cell.
/// </summary>
private void SetHeaderLayout(IXLStyle style, XLColor borderColor)
{
style.Alignment.Horizontal = XLAlignmentHorizontalValues.Center;
style.Alignment.Vertical = XLAlignmentVerticalValues.Center;
style.Alignment.WrapText = true;
style.Border.OutsideBorder = XLBorderStyleValues.Medium;
style.Border.OutsideBorderColor = borderColor;
}
#endregion
#region Data area
/// <summary>
/// Number formats on the first rows of the data area, and the dropdown validations.
/// </summary>
private void ApplyDataArea(IXLWorksheet sheet, List<EmployeeBulkUploadColumn> columns)
{
int firstRow = EmployeeBulkUploadTemplate.FirstDataRow;
int formattedLastRow = firstRow + EmployeeBulkUploadTemplate.FormattedDataRows - 1;
int validatedLastRow = firstRow + EmployeeBulkUploadTemplate.ValidatedDataRows - 1;
for (int i = 0; i < columns.Count; i++)
{
EmployeeBulkUploadColumn definition = columns[i];
int column = i + 1;
string numberFormat = definition.NumberFormat;
if (!string.IsNullOrEmpty(numberFormat))
{
sheet.Range(firstRow, column, formattedLastRow, column)
.Style.NumberFormat.Format = numberFormat;
}
string dropdown = definition.DropdownList;
if (!string.IsNullOrEmpty(dropdown))
{
// An inline list keeps the workbook to the single "Employee Data" sheet —
// no helper sheet to reference, nothing for a user to delete by accident.
IXLDataValidation validation = sheet
.Range(firstRow, column, validatedLastRow, column)
.CreateDataValidation();
validation.IgnoreBlanks = true;
validation.InCellDropdown = true;
validation.ErrorStyle = XLErrorStyle.Stop;
validation.ErrorTitle = "Invalid value";
validation.ErrorMessage = "Please pick a value from the dropdown list.";
validation.InputTitle = "Pick from list";
validation.InputMessage = "Choose one of the listed values.";
validation.List("\"" + dropdown + "\"", true);
}
}
}
private void ApplySheetView(IXLWorksheet sheet)
{
// Keeps the three header rows and Employee No / Employee Name in view while scrolling
// across the remaining columns. No auto-filter on the heading row.
sheet.SheetView.FreezeRows(EmployeeBulkUploadTemplate.HeadingRow);
sheet.SheetView.FreezeColumns(2);
}
#endregion
}
}