using System; using System.Collections.Generic; using System.IO; using System.Linq; using Ease.Core.DataAccess; using Ease.Core.Model; using Ease.Core.Utility; using HRM.BO; using Microsoft.AspNetCore.Hosting; using SkiaSharp; namespace HRM.DA { /// /// Employee Profile Picture Bulk Upload. The pictures are copied into /// EmployeePhotoBulkUploadRules.FolderName on the server, each named by its Employee No. /// /// Each picture is stored the way an approved profile-picture request stores it /// (EssController.ApproveRequest): the original into empFileupload through HREmployeeDA.uplaodFile /// (which deletes the employee's previous picture first) with the profile percentage recalculated, /// and the 100px thumbnail written by EmployeePhotoService.GenerateThumbnail into /// EmployeePhotos\EmpthumbnailsImage, which is what every screen shows. /// /// Saving is all or nothing for the whole folder: every picture goes into the database in ONE /// transaction, and the thumbnails are written before it commits with the old ones kept aside, /// so a failure anywhere rolls the database back and puts every old thumbnail back. /// public class EmployeePhotoBulkUploadService : IEmployeePhotoBulkUploadService { /// Files Windows, macOS and Office leave in a folder by themselves; not pictures, not reported. private static readonly string[] SystemFiles = { "thumbs.db", "desktop.ini", ".ds_store" }; private readonly IWebHostEnvironment _env; private readonly string _folder; private readonly string _displayFolder; public EmployeePhotoBulkUploadService(IWebHostEnvironment env) { _env = env; _folder = Path.Combine(env.ContentRootPath, EmployeePhotoBulkUploadRules.FolderName); // What the screen shows: only the application's own folder and this one under it // (e.g. HRM.UI\EmployeePhotoBulkUpload), never the full path of the server's disk. _displayFolder = Path.Combine(new DirectoryInfo(env.ContentRootPath).Name, EmployeePhotoBulkUploadRules.FolderName); } public EmployeePhotoBulkUploadInfo GetInfo() { EnsureFolder(); return new EmployeePhotoBulkUploadInfo { FolderPath = _displayFolder, Extensions = EmployeePhotoBulkUploadRules.Extensions.ToList(), MaxFileSizeMB = EmployeePhotoBulkUploadRules.MaxFileSizeMB, MinSidePixels = EmployeePhotoBulkUploadRules.MinSidePixels, FileCount = ListFiles().Count }; } public EmployeePhotoBulkUploadReadResult ReadFolder(int payrollTypeID) { if (payrollTypeID <= 0) throw new ServiceException("The payroll type of the login could not be found."); EnsureFolder(); EmployeePhotoBulkUploadReadResult result = new EmployeePhotoBulkUploadReadResult { FolderPath = _displayFolder, Subfolders = Directory.GetDirectories(_folder).Select(Path.GetFileName) .Where(x => !x.StartsWith(".")).OrderBy(x => x).ToList() // not the save's own kept-aside folder }; result.Files = Check(ListFiles(), payrollTypeID); result.ReadyCount = result.Files.Count(x => x.Status == EmployeePhotoBulkUploadFile.Ready); result.WarningCount = result.Files.Count(x => x.Status == EmployeePhotoBulkUploadFile.Warning); result.ErrorCount = result.Files.Count(x => x.Status == EmployeePhotoBulkUploadFile.Error); result.ReplaceCount = result.Files.Count(x => x.HasPhoto && x.Status != EmployeePhotoBulkUploadFile.Error); result.CanSave = result.Files.Count > 0 && result.ErrorCount == 0; return result; } #region Save run (one at a time, followed by the screen) private static readonly object RunLock = new object(); private static readonly Dictionary Runs = new Dictionary(); private static string _activeRunID; public EmployeePhotoBulkUploadRun BeginSave(int payrollTypeID) { if (payrollTypeID <= 0) throw new ServiceException("The payroll type of the login could not be found."); lock (RunLock) { // Two saves of one folder at once would fight over the same files and thumbnails. if (_activeRunID != null) throw new ServiceException("Profile pictures are being saved already. Wait until that save ends."); // Finished runs are only kept long enough for the screen to read the outcome. foreach (string old in Runs.Where(x => x.Value.FinishedAt.HasValue && x.Value.FinishedAt.Value < DateTime.Now.AddHours(-6)) .Select(x => x.Key).ToList()) Runs.Remove(old); EmployeePhotoBulkUploadRun run = new EmployeePhotoBulkUploadRun { RunID = Guid.NewGuid().ToString("N"), Status = EmployeePhotoBulkUploadRun.Running, Step = "Starting", StartedAt = DateTime.Now, StepStartedAt = DateTime.Now }; Runs[run.RunID] = run; _activeRunID = run.RunID; return Copy(run); } } public void RunSave(string runID, int payrollTypeID) { EmployeePhotoBulkUploadRun run; lock (RunLock) { Runs.TryGetValue(runID ?? string.Empty, out run); } if (run == null) return; try { Save(run, payrollTypeID); } catch (Exception e) { ExceptionLog.Write(e); Finish(run, EmployeePhotoBulkUploadRun.Failed, "Nothing was saved: " + e.Message, null); } finally { lock (RunLock) { if (_activeRunID == runID) _activeRunID = null; } } } public EmployeePhotoBulkUploadRun GetRun(string runID) { lock (RunLock) { EmployeePhotoBulkUploadRun run; return Runs.TryGetValue(runID ?? string.Empty, out run) ? Copy(run) : null; } } public EmployeePhotoBulkUploadRun GetActiveRun() { lock (RunLock) { EmployeePhotoBulkUploadRun run; return _activeRunID != null && Runs.TryGetValue(_activeRunID, out run) ? Copy(run) : null; } } /// The run is changed by the save's thread and read by the screen's requests: hand out copies. private static EmployeePhotoBulkUploadRun Copy(EmployeePhotoBulkUploadRun run) { lock (run) { return new EmployeePhotoBulkUploadRun { RunID = run.RunID, Status = run.Status, Step = run.Step, Total = run.Total, Done = run.Done, Message = run.Message, SavedCount = run.SavedCount, ReplacedCount = run.ReplacedCount, Files = run.Files.ToList(), SavedEmployeeIDs = run.SavedEmployeeIDs.ToList(), StartedAt = run.StartedAt, FinishedAt = run.FinishedAt, StepStartedAt = run.StepStartedAt, ElapsedSeconds = (int)((run.FinishedAt ?? DateTime.Now) - run.StartedAt).TotalSeconds, StepElapsedSeconds = (int)(DateTime.Now - run.StepStartedAt).TotalSeconds }; } } private static void SetStep(EmployeePhotoBulkUploadRun run, string step, int total) { lock (run) { run.Step = step; run.Total = total; run.Done = 0; run.StepStartedAt = DateTime.Now; } } private static void Finish(EmployeePhotoBulkUploadRun run, string status, string message, List files) { lock (run) { run.Status = status; run.Message = message; run.Files = files ?? new List(); run.FinishedAt = DateTime.Now; } } #endregion #region Checks /// /// Every check of every file: the file itself (type, name, size, a real picture), the same /// employee twice, and the employee (in the system, in this payroll type, one only). /// Only picture headers are read here, so even a very large folder is checked quickly; /// the whole picture is decoded when it is saved. /// private List Check(List paths, int payrollTypeID) { List files = new List(); Dictionary pathOf = new Dictionary(); foreach (string path in paths) { EmployeePhotoBulkUploadFile file = new EmployeePhotoBulkUploadFile { FileName = Path.GetFileName(path), EmployeeNo = Path.GetFileNameWithoutExtension(path) }; CheckFile(file, path); files.Add(file); pathOf[file] = path; } // The same employee twice (C0107.jpg and C0107.png): neither is taken, the user keeps one. foreach (var group in files.Where(x => !string.IsNullOrWhiteSpace(x.EmployeeNo)) .GroupBy(x => x.EmployeeNo.Trim(), StringComparer.OrdinalIgnoreCase) .Where(g => g.Count() > 1)) { string names = string.Join(", ", group.Select(x => x.FileName)); foreach (EmployeePhotoBulkUploadFile file in group) file.Errors.Add("More than one picture for Employee No " + group.Key + " (" + names + "). Keep only one of them in the folder."); } CheckEmployees(files, payrollTypeID); foreach (EmployeePhotoBulkUploadFile file in files) { file.Status = file.Errors.Count > 0 ? EmployeePhotoBulkUploadFile.Error : file.Warnings.Count > 0 ? EmployeePhotoBulkUploadFile.Warning : EmployeePhotoBulkUploadFile.Ready; } return files.OrderBy(x => x.FileName, StringComparer.OrdinalIgnoreCase).ToList(); } private static void CheckFile(EmployeePhotoBulkUploadFile file, string path) { string extension = Path.GetExtension(path).ToLowerInvariant(); if (!EmployeePhotoBulkUploadRules.Extensions.Contains(extension)) { file.Errors.Add((string.IsNullOrEmpty(extension) ? "The file has no extension" : "'" + extension + "' files are not taken") + ". Only " + string.Join(", ", EmployeePhotoBulkUploadRules.Extensions) + " pictures can be uploaded."); return; } string name = file.EmployeeNo ?? string.Empty; if (name.Trim().Length == 0) { file.Errors.Add("The file name is empty. Name the picture by the Employee No, e.g. C0107" + extension + "."); return; } if (name != name.Trim()) file.Errors.Add("The file name starts or ends with a space. Name it exactly by the Employee No, e.g. " + name.Trim() + extension + "."); FileInfo info = new FileInfo(path); file.SizeBytes = info.Length; if (info.Length == 0) { file.Errors.Add("The file is empty (0 KB)."); return; } if (info.Length > EmployeePhotoBulkUploadRules.MaxFileSizeMB * 1024L * 1024L) file.Errors.Add("The picture is " + (info.Length / 1024d / 1024d).ToString("0.#") + " MB; the most a picture can be is " + EmployeePhotoBulkUploadRules.MaxFileSizeMB + " MB. Make it smaller and copy it again."); try { using (SKCodec codec = SKCodec.Create(path)) { if (codec == null) { file.Errors.Add("The file is not a picture, or it is damaged, although its name ends in " + extension + "."); return; } if (codec.EncodedFormat != SKEncodedImageFormat.Jpeg && codec.EncodedFormat != SKEncodedImageFormat.Png) { file.Errors.Add("The file is a " + codec.EncodedFormat.ToString().ToUpperInvariant() + " picture renamed to " + extension + ". Save it as a real JPG or PNG."); return; } file.Width = codec.Info.Width; file.Height = codec.Info.Height; } } catch (Exception e) when (e is IOException || e is UnauthorizedAccessException) { file.Errors.Add("The file could not be opened (" + e.Message + ")."); return; } if (Math.Min(file.Width, file.Height) < EmployeePhotoBulkUploadRules.MinSidePixels) file.Warnings.Add("The picture is only " + file.Width + " x " + file.Height + " pixels, so it may look blurred. At least " + EmployeePhotoBulkUploadRules.MinSidePixels + " x " + EmployeePhotoBulkUploadRules.MinSidePixels + " is recommended."); } /// /// The employee of each file, by Employee No: in the system, in the current payroll type and /// only one of them. A file that is wrong as a file already has its error and is not looked up. /// private void CheckEmployees(List files, int payrollTypeID) { List toFind = files.Where(x => x.Errors.Count == 0).ToList(); if (toFind.Count == 0) return; EmployeePhotoService photoService = new EmployeePhotoService(_env); // The lookup Salary Master Data Bulk Upload uses: one query per 500 Employee Nos. Dictionary> byNo = new HistorySalaryMonthlyService() .GetEmployeesByNo(toFind.Select(x => x.EmployeeNo).ToList()) .Where(x => !string.IsNullOrWhiteSpace(x.EmployeeNo)) .GroupBy(x => x.EmployeeNo.Trim(), StringComparer.OrdinalIgnoreCase) .ToDictionary(g => g.Key, g => g.ToList(), StringComparer.OrdinalIgnoreCase); foreach (EmployeePhotoBulkUploadFile file in toFind) { List found; if (!byNo.TryGetValue(file.EmployeeNo, out found) || found.Count == 0) { file.Errors.Add("There is no employee with Employee No " + file.EmployeeNo + " in the system. The file name must be exactly the Employee No (not e.g. \"" + file.EmployeeNo + " (1)\" or \"" + file.EmployeeNo + " - Copy\")."); continue; } List inPayrollType = found.Where(x => x.PayrollTypeID == payrollTypeID).ToList(); if (inPayrollType.Count == 0) { file.Errors.Add("Employee " + file.EmployeeNo + " belongs to another payroll type, not the current one."); continue; } if (inPayrollType.Count > 1) { file.Errors.Add("More than one employee of this payroll type has Employee No " + file.EmployeeNo + ", so it is not clear whose picture this is."); continue; } Employee employee = inPayrollType[0]; file.EmployeeID = employee.ID; file.EmployeeNo = employee.EmployeeNo.Trim(); file.EmployeeName = employee.Name; if (employee.Status != EnumEmployeeStatus.Live) file.Warnings.Add("Employee " + file.EmployeeNo + " is not live — the status is " + employee.Status + ". The picture will still be saved."); file.HasPhoto = photoService.ThumbnailExists(file.EmployeeNo); if (photoService.PendingExists(file.EmployeeNo)) file.Warnings.Add("The employee has a picture waiting for approval on Profile Update. " + "If that request is approved later, it will replace this picture."); } } #endregion #region Save /// /// The whole save, all or nothing: /// 1. every check again, on the folder as it is now — one error and nothing is saved; /// 2. every picture read in full, so a damaged one is found before anything is written; /// 3. every picture into the database in ONE transaction, each thumbnail written as it goes /// with the old one kept aside; a failure rolls the database back and puts the old /// thumbnails back, so every employee is exactly as before; /// 4. only after the commit are the pictures taken out of the folder. /// private void Save(EmployeePhotoBulkUploadRun run, int payrollTypeID) { EnsureFolder(); // 1. Checks SetStep(run, "Checking the pictures", 0); List files = Check(ListFiles(), payrollTypeID); if (files.Count == 0) { Finish(run, EmployeePhotoBulkUploadRun.Failed, "There is no picture in the folder. Nothing was saved.", null); return; } List withErrors = files.Where(x => x.Status == EmployeePhotoBulkUploadFile.Error).ToList(); if (withErrors.Count > 0) { Finish(run, EmployeePhotoBulkUploadRun.Failed, withErrors.Count + " picture(s) have errors, so nothing was saved. " + "Fix them in the folder, check again and save again.", withErrors); return; } // 2. Every picture read in full SetStep(run, "Reading every picture", files.Count); foreach (EmployeePhotoBulkUploadFile file in files) { try { using (SKBitmap bitmap = SKBitmap.Decode(File.ReadAllBytes(Path.Combine(_folder, file.FileName)))) { if (bitmap == null) file.Errors.Add("The picture is damaged and could not be read. Copy a good copy of it into the folder."); } } catch (Exception e) { file.Errors.Add("The picture could not be read (" + e.Message + ")."); } if (file.Errors.Count > 0) file.Status = EmployeePhotoBulkUploadFile.Error; run.Done++; } withErrors = files.Where(x => x.Status == EmployeePhotoBulkUploadFile.Error).ToList(); if (withErrors.Count > 0) { Finish(run, EmployeePhotoBulkUploadRun.Failed, withErrors.Count + " picture(s) could not be read, so nothing was saved. " + "Replace them in the folder, check again and save again.", withErrors); return; } // 3. Database (one transaction) + thumbnails (old ones kept aside) SetStep(run, "Saving the pictures", files.Count); EmployeePhotoService photoService = new EmployeePhotoService(_env); string backupFolder = Path.Combine(_folder, ".backup-" + run.RunID); Directory.CreateDirectory(backupFolder); // Employee No -> (thumbnail path, the old one kept aside or null when the employee had none). Dictionary> replaced = new Dictionary>(StringComparer.OrdinalIgnoreCase); TransactionContext tc = null; EmployeePhotoBulkUploadFile current = null; try { tc = TransactionContext.Begin(true); foreach (EmployeePhotoBulkUploadFile file in files) { current = file; byte[] bytes = File.ReadAllBytes(Path.Combine(_folder, file.FileName)); HREmployeeDA.uplaodFile(tc, ToUpload(file, bytes)); ProfileUpdateRequestDA.UpdateProfilePercentage(tc, file.EmployeeID); string oldThumbnail = photoService.FindApprovedFile(file.EmployeeNo); string keptAside = null; if (oldThumbnail != null) { keptAside = Path.Combine(backupFolder, Path.GetFileName(oldThumbnail)); File.Copy(oldThumbnail, keptAside, true); } // Noted before writing, so even a half-written thumbnail is put back. replaced[file.EmployeeNo] = Tuple.Create(oldThumbnail, keptAside); photoService.GenerateThumbnail(file.EmployeeNo, bytes); run.Done++; } current = null; SetStep(run, "Finishing", files.Count); run.Done = files.Count; tc.End(); tc = null; } catch (Exception e) { #region Handle Exception if (tc != null) tc.HandleError(); ExceptionLog.Write(e); List notRestored = RestoreThumbnails(photoService, replaced); TryDeleteFolder(backupFolder); List stopped = new List(); if (current != null) { current.Errors.Add("Saving stopped at this picture: " + e.Message); current.Status = EmployeePhotoBulkUploadFile.Error; stopped.Add(current); } Finish(run, EmployeePhotoBulkUploadRun.Failed, "Saving stopped (" + e.Message + "). Nothing was saved — every employee's picture is as it was." + (notRestored.Count > 0 ? " The old on-screen picture of " + string.Join(", ", notRestored) + " could not be put back; it is still in the database." : string.Empty), stopped); return; #endregion } // 4. Committed: the pictures leave the folder. List notes = new List(); foreach (EmployeePhotoBulkUploadFile file in files) { try { File.Delete(Path.Combine(_folder, file.FileName)); } catch (Exception e) when (e is IOException || e is UnauthorizedAccessException) { // Saved all the same; only the leftover copy needs removing by hand. file.Warnings.Add("Saved, but the file could not be removed from the folder (" + e.Message + "). Delete it by hand."); notes.Add(file); } } TryDeleteFolder(backupFolder); lock (run) { run.SavedCount = files.Count; run.ReplacedCount = replaced.Count(x => x.Value.Item2 != null); run.SavedEmployeeIDs = files.Select(x => x.EmployeeID).ToList(); } Finish(run, EmployeePhotoBulkUploadRun.Saved, "All " + files.Count + " profile picture(s) were saved.", notes); } /// referenceID is the employee id, as the approval and every photo reader use it. private static empFileupload ToUpload(EmployeePhotoBulkUploadFile file, byte[] bytes) { return new empFileupload { employeeID = file.EmployeeID, referenceID = file.EmployeeID, filetype = enumEmpFileUploadType.photo, FileFormat = EnumFileFormat.Picture, fileName = "Profile Picture" + Path.GetExtension(file.FileName).ToLowerInvariant(), fileData = bytes }; } /// /// After a rollback: every thumbnail this save wrote goes back to the one kept aside, or is /// removed when the employee had none. Returns the Employee Nos that could not be put back. /// private static List RestoreThumbnails(EmployeePhotoService photoService, Dictionary> replaced) { List notRestored = new List(); foreach (KeyValuePair> item in replaced) { try { string oldThumbnail = item.Value.Item1; string keptAside = item.Value.Item2; if (keptAside != null) { File.Copy(keptAside, oldThumbnail, true); } else { string written = photoService.FindApprovedFile(item.Key); if (written != null) File.Delete(written); } } catch (Exception e) { ExceptionLog.Write(e); notRestored.Add(item.Key); } } return notRestored; } private static void TryDeleteFolder(string folder) { try { if (Directory.Exists(folder)) Directory.Delete(folder, true); } catch (Exception e) when (e is IOException || e is UnauthorizedAccessException) { // Only a leftover folder of old thumbnails; it is never read as pictures. } } #endregion #region Folder private void EnsureFolder() { if (!Directory.Exists(_folder)) Directory.CreateDirectory(_folder); } /// The top-level files of the folder, without the system files and Office lock files. private List ListFiles() { return Directory.EnumerateFiles(_folder, "*", SearchOption.TopDirectoryOnly) .Where(path => { string name = Path.GetFileName(path); if (SystemFiles.Contains(name.ToLowerInvariant()) || name.StartsWith("~$") || name.StartsWith(".")) return false; return (File.GetAttributes(path) & (FileAttributes.Hidden | FileAttributes.System)) == 0; }) .ToList(); } #endregion } }