Add SharePoint manual source handling and finance status

2026-05-11 08:43:52 +02:00
parent 57cb09bc50
commit 819a023163
16 changed files with 983 additions and 28 deletions
@@ -11,4 +11,10 @@ public sealed class DataSourceFetchResult
    /// SiteExportService erzeugt dann keine neue Excel-Datei.
    /// </summary>
    public string? ReferenceFilePath { get; init; }
+
+    public string? LocalOutputDirectoryOverride { get; init; }
+
+    public string? SharePointUploadFolderOverride { get; init; }
+
+    public string? SharePointUploadLandOverride { get; init; }
 }
@@ -29,12 +29,15 @@ public sealed class ManualExcelDataSourceAdapter : IDataSourceAdapter

        var manualImportPath = site.ManualImportFilePath.Trim();
        string filePath;
+        string? localOutputDirectory = null;
+        string? sharePointUploadFolder = null;
        string? tempManualImportPath = null;
        try
        {
            if (File.Exists(manualImportPath))
            {
                filePath = manualImportPath;
+                localOutputDirectory = Path.GetDirectoryName(Path.GetFullPath(manualImportPath));
            }
            else if (LooksLikeSharePointReference(manualImportPath))
            {
@@ -55,10 +58,22 @@ public sealed class ManualExcelDataSourceAdapter : IDataSourceAdapter
                await _appEventLogService.WriteAsync("Export", "Manuelle Excel von SharePoint laden",
                    siteId: site.Id, land: site.Land, details: manualImportPath);

+                var sharePointFileReference = manualImportPath;
+                if (LooksLikeSharePointFolderReference(manualImportPath))
+                {
+                    var latestFile = await _sharePointService.ResolveLatestFileInFolderAsync(
+                        spConfig.TenantId, spConfig.ClientId, spConfig.ClientSecret,
+                        spConfig.SiteUrl, manualImportPath, site.TSC);
+                    sharePointFileReference = latestFile.FileReference;
+                    await _appEventLogService.WriteAsync("Export", "Neueste SharePoint-Datei ausgewaehlt",
+                        siteId: site.Id, land: site.Land, details: sharePointFileReference);
+                }
+
                tempManualImportPath = await _sharePointService.DownloadToTempFileAsync(
                    spConfig.TenantId, spConfig.ClientId, spConfig.ClientSecret,
-                    spConfig.SiteUrl, manualImportPath);
-                filePath = manualImportPath;
+                    spConfig.SiteUrl, sharePointFileReference);
+                filePath = sharePointFileReference;
+                sharePointUploadFolder = ResolveSharePointParentFolder(sharePointFileReference, spConfig.SiteUrl);
            }
            else
            {
@@ -75,7 +90,9 @@ public sealed class ManualExcelDataSourceAdapter : IDataSourceAdapter
            return new DataSourceFetchResult
            {
                Records = records,
-                ReferenceFilePath = filePath
+                LocalOutputDirectoryOverride = localOutputDirectory,
+                SharePointUploadFolderOverride = sharePointUploadFolder,
+                SharePointUploadLandOverride = sharePointUploadFolder is null ? null : string.Empty
            };
        }
        finally
@@ -90,4 +107,25 @@ public sealed class ManualExcelDataSourceAdapter : IDataSourceAdapter
           path.StartsWith("https://", StringComparison.OrdinalIgnoreCase) ||
           path.StartsWith("/Shared Documents/", StringComparison.OrdinalIgnoreCase) ||
           path.StartsWith("Shared Documents/", StringComparison.OrdinalIgnoreCase);
+
+    private static bool LooksLikeSharePointFolderReference(string path)
+        => LooksLikeSharePointReference(path) &&
+           string.IsNullOrWhiteSpace(Path.GetExtension(path.TrimEnd('/')));
+
+    private static string ResolveSharePointParentFolder(string fileReference, string siteUrl)
+    {
+        var remotePath = fileReference.Trim('/').Trim();
+        if (Uri.TryCreate(fileReference, UriKind.Absolute, out var fileUri) &&
+            Uri.TryCreate(siteUrl, UriKind.Absolute, out var siteUri))
+        {
+            var absolutePath = Uri.UnescapeDataString(fileUri.AbsolutePath);
+            var sitePath = siteUri.AbsolutePath.TrimEnd('/');
+            if (absolutePath.StartsWith(sitePath, StringComparison.OrdinalIgnoreCase))
+                absolutePath = absolutePath[sitePath.Length..];
+            remotePath = absolutePath.Trim('/').Trim();
+        }
+
+        var lastSlash = remotePath.LastIndexOf('/');
+        return lastSlash <= 0 ? string.Empty : remotePath[..lastSlash];
+    }
 }
@@ -13,6 +13,7 @@ public class DatabaseSeedService : IDatabaseSeedService
        EnsureSourceSystemDefinitions(db);
        EnsureCentralHanaServerRecords(db);
        EnsureSpainManualExcelSite(db);
+        EnsureUkManualExcelFolder(db);
        EnsureSapODataDachSite(db);
        EnsureFinanceReferenceDefaults(db);
        EnsureBudgetExchangeRateDefaults(db);
@@ -287,6 +288,36 @@ public class DatabaseSeedService : IDatabaseSeedService
        db.SaveChanges();
    }

+    private static void EnsureUkManualExcelFolder(AppDbContext db)
+    {
+        var existing = db.Sites
+            .OrderBy(x => x.Id)
+            .FirstOrDefault(x =>
+                x.TSC == "TRUK" ||
+                x.Land == "England" ||
+                x.Land == "UK");
+
+        if (existing is null)
+            return;
+
+        var changed = false;
+        if (string.IsNullOrWhiteSpace(existing.SourceSystem))
+        {
+            existing.SourceSystem = "MANUAL_EXCEL";
+            changed = true;
+        }
+
+        if (string.Equals(existing.SourceSystem, "MANUAL_EXCEL", StringComparison.OrdinalIgnoreCase) &&
+            string.IsNullOrWhiteSpace(existing.ManualImportFilePath))
+        {
+            existing.ManualImportFilePath = "https://trafagag.sharepoint.com/sites/WorldwideBIPlatform/Import/Finance/UK_B1";
+            changed = true;
+        }
+
+        if (changed)
+            db.SaveChanges();
+    }
+
    private static void EnsureSapODataDachSite(AppDbContext db)
    {
        if (db.Sites.Count() <= 1)
@@ -60,16 +60,7 @@ public sealed class FinanceReconciliationService : IFinanceReconciliationService
                rows => BuildNetSalesActual(rows, budgetRatesToChf, intercompanyRules),
                StringComparer.OrdinalIgnoreCase);

-        var activeSiteKeys = (await db.Sites
-                .AsNoTracking()
-                .Where(s => s.IsActive)
-                .Select(s => new { s.Land, s.TSC })
-                .ToListAsync())
-            .Select(s => ResolveReferenceKey(s.Land, s.TSC))
-            .ToHashSet(StringComparer.OrdinalIgnoreCase);
-
        return financeReferences
-            .Where(reference => activeSiteKeys.Contains(reference.Key) || groupedActuals.ContainsKey(reference.Key))
            .Select(reference => BuildReferenceRow(reference, groupedActuals))
            .OrderBy(row => row.Label, StringComparer.OrdinalIgnoreCase)
            .ToList();
@@ -282,6 +273,8 @@ public sealed class FinanceReconciliationService : IFinanceReconciliationService
        var normalizedLand = (land ?? string.Empty).Trim().ToUpperInvariant();
        var normalizedTsc = (tsc ?? string.Empty).Trim().ToUpperInvariant();

+        if (normalizedLand is "AT" or "AUT" || normalizedLand.Contains("OESTER") || normalizedLand.Contains("OSTER") || normalizedLand.Contains("AUSTRIA")) return "AT";
+        if (normalizedLand is "CH" or "CHE" || normalizedLand.Contains("SCHWE") || normalizedLand.Contains("SWITZER")) return "CH";
        if (normalizedLand.Contains("FRANK") || normalizedTsc.Contains("FR")) return "FR";
        if (normalizedLand.Contains("IND") || normalizedTsc.Contains("IN")) return "IN";
        if (normalizedLand.Contains("ITAL") || normalizedTsc.Contains("IT")) return "IT";
@@ -289,7 +282,6 @@ public sealed class FinanceReconciliationService : IFinanceReconciliationService
        if (normalizedLand.Contains("USA") || normalizedLand.Contains("UNITED STATES") || normalizedTsc.Contains("US")) return "US";
        if (normalizedLand.Contains("DEUT") || normalizedTsc.Contains("DE")) return "DE";
        if (normalizedLand.Contains("SPAN") || normalizedTsc is "SE" or "ES") return "ES";
-        if (normalizedLand.Contains("SCHWE") || normalizedTsc.Contains("CH")) return "CH";

        return normalizedTsc.Replace("TR", string.Empty);
    }
@@ -4,5 +4,8 @@ public interface ISharePointUploadService
 {
    Task UploadAsync(string tenantId, string clientId, string clientSecret, string siteUrl, string exportFolder, string land, string localFilePath);
    Task<string> DownloadToTempFileAsync(string tenantId, string clientId, string clientSecret, string siteUrl, string fileReference);
+    Task<SharePointFileReference> ResolveLatestFileInFolderAsync(string tenantId, string clientId, string clientSecret, string siteUrl, string folderReference, string siteTsc);
    Task TestConnectionAsync(string tenantId, string clientId, string clientSecret, string siteUrl);
 }
+
+public sealed record SharePointFileReference(string FileReference, DateTimeOffset? LastModifiedUtc);
@@ -1,6 +1,9 @@
 using Azure.Core;
 using Azure.Identity;
 using Microsoft.Graph;
+using Microsoft.Graph.Models;
+using System.Globalization;
+using System.Text.RegularExpressions;

 namespace TrafagSalesExporter.Services;

@@ -82,6 +85,64 @@ public class SharePointUploadService : ISharePointUploadService
        return tempPath;
    }

+    public async Task<SharePointFileReference> ResolveLatestFileInFolderAsync(
+        string tenantId,
+        string clientId,
+        string clientSecret,
+        string siteUrl,
+        string folderReference,
+        string siteTsc)
+    {
+        var normalizedTenantId = Normalize(tenantId);
+        var normalizedClientId = Normalize(clientId);
+        var normalizedClientSecret = Normalize(clientSecret);
+        var normalizedSiteUrl = Normalize(siteUrl);
+        var normalizedReference = Normalize(folderReference);
+        var normalizedTsc = Normalize(siteTsc).ToUpperInvariant();
+
+        if (string.IsNullOrWhiteSpace(normalizedReference))
+            throw new InvalidOperationException("SharePoint-Ordnerreferenz fehlt.");
+
+        var credential = new ClientSecretCredential(normalizedTenantId, normalizedClientId, normalizedClientSecret);
+        var graphClient = new GraphServiceClient(credential, ["https://graph.microsoft.com/.default"]);
+
+        var siteUri = new Uri(normalizedSiteUrl);
+        var sitePath = siteUri.AbsolutePath.TrimEnd('/');
+        var site = await graphClient.Sites[$"{siteUri.Host}:{sitePath}"].GetAsync();
+
+        if (site?.Id is null)
+            throw new InvalidOperationException("SharePoint Site konnte nicht gefunden werden.");
+
+        var drive = await graphClient.Sites[site.Id].Drive.GetAsync();
+        if (drive?.Id is null)
+            throw new InvalidOperationException("SharePoint Dokumentenbibliothek konnte nicht gefunden werden.");
+
+        var folderPath = ResolveRemotePath(normalizedReference, siteUri);
+        var children = await graphClient.Drives[drive.Id].Root.ItemWithPath(folderPath).Children.GetAsync();
+        var candidates = children?.Value?
+            .Where(item => item.File is not null)
+            .Where(item => IsSupportedManualImportFile(item.Name))
+            .Where(item => MatchesTsc(item.Name, normalizedTsc))
+            .Select(item => new
+            {
+                Item = item,
+                FileDate = TryParseDatedSiteFileName(item.Name, normalizedTsc, out var fileDate) ? fileDate : (DateTime?)null
+            })
+            .OrderByDescending(x => x.FileDate ?? x.Item.LastModifiedDateTime?.UtcDateTime ?? DateTime.MinValue)
+            .ThenByDescending(x => x.Item.LastModifiedDateTime?.UtcDateTime ?? DateTime.MinValue)
+            .ToList() ?? [];
+
+        var selected = candidates.FirstOrDefault()
+            ?? throw new InvalidOperationException(
+                string.IsNullOrWhiteSpace(normalizedTsc)
+                    ? $"Im SharePoint-Ordner '{folderPath}' wurde keine Excel-/CSV-Datei gefunden."
+                    : $"Im SharePoint-Ordner '{folderPath}' wurde keine Excel-/CSV-Datei fuer '{normalizedTsc}' gefunden.");
+
+        return new SharePointFileReference(
+            string.Join("/", folderPath.Trim('/'), selected.Item.Name).Trim('/'),
+            selected.Item.LastModifiedDateTime);
+    }
+
    public async Task TestConnectionAsync(string tenantId, string clientId, string clientSecret, string siteUrl)
    {
        var normalizedTenantId = Normalize(tenantId);
@@ -143,6 +204,41 @@ public class SharePointUploadService : ISharePointUploadService
        return fileReference.Trim('/').Trim();
    }

+    private static bool IsSupportedManualImportFile(string? fileName)
+    {
+        var extension = Path.GetExtension(fileName ?? string.Empty);
+        return extension.Equals(".xlsx", StringComparison.OrdinalIgnoreCase) ||
+               extension.Equals(".csv", StringComparison.OrdinalIgnoreCase);
+    }
+
+    private static bool MatchesTsc(string? fileName, string normalizedTsc)
+    {
+        if (string.IsNullOrWhiteSpace(normalizedTsc))
+            return true;
+
+        var nameWithoutExtension = Path.GetFileNameWithoutExtension(fileName ?? string.Empty);
+        return nameWithoutExtension.EndsWith($"_{normalizedTsc}", StringComparison.OrdinalIgnoreCase);
+    }
+
+    private static bool TryParseDatedSiteFileName(string? fileName, string normalizedTsc, out DateTime fileDate)
+    {
+        fileDate = default;
+        var nameWithoutExtension = Path.GetFileNameWithoutExtension(fileName ?? string.Empty);
+        var pattern = string.IsNullOrWhiteSpace(normalizedTsc)
+            ? @"^(?<date>\d{6})_[A-Z0-9]+$"
+            : $"^(?<date>\\d{{6}})_{Regex.Escape(normalizedTsc)}$";
+        var match = Regex.Match(nameWithoutExtension, pattern, RegexOptions.IgnoreCase);
+        if (!match.Success)
+            return false;
+
+        return DateTime.TryParseExact(
+            match.Groups["date"].Value,
+            "ddMMyy",
+            CultureInfo.InvariantCulture,
+            DateTimeStyles.None,
+            out fileDate);
+    }
+
    private static string BuildInputPreview(string tenantId, string clientId, string clientSecret, string siteUrl)
    {
        var maskedSecret = string.IsNullOrEmpty(clientSecret)
@@ -56,8 +56,6 @@ public class SiteExportService : ISiteExportService
                details: $"Quelle={sourceSystem} | TSC={site.TSC}");

            var (settings, spConfig, sourceDefinition, rules) = await LoadExportConfigAsync(site, sourceSystem);
-            var outputDir = ResolveSiteOutputDirectory(settings, site);
-
            var adapter = _dataSourceResolver.Resolve(sourceDefinition.ConnectionKind);
            var fetchResult = await adapter.FetchAsync(new DataSourceFetchContext
            {
@@ -69,6 +67,7 @@ public class SiteExportService : ISiteExportService
            });

            var records = fetchResult.Records;
+            var outputDir = fetchResult.LocalOutputDirectoryOverride ?? ResolveSiteOutputDirectory(settings, site);

            updateStatus?.Invoke("Transformationen anwenden...");
            await _appEventLogService.WriteAsync("Export", "Transformationen anwenden",
@@ -94,7 +93,7 @@ public class SiteExportService : ISiteExportService
                details: $"Records={records.Count}");
            await _centralSalesRecordService.ReplaceForSiteAsync(site, records, updateStatus);

-            await UploadToSharePointIfConfiguredAsync(site, spConfig, filePath, updateStatus);
+            await UploadToSharePointIfConfiguredAsync(site, spConfig, filePath, updateStatus, fetchResult);

            sw.Stop();
            log.Status = "OK";
@@ -156,7 +155,11 @@ public class SiteExportService : ISiteExportService
    }

    private async Task UploadToSharePointIfConfiguredAsync(
-        Site site, SharePointConfig? spConfig, string filePath, Action<string>? updateStatus)
+        Site site,
+        SharePointConfig? spConfig,
+        string filePath,
+        Action<string>? updateStatus,
+        DataSourceFetchResult fetchResult)
    {
        if (spConfig is null ||
            string.IsNullOrWhiteSpace(spConfig.TenantId) ||
@@ -165,12 +168,16 @@ public class SiteExportService : ISiteExportService
            return;

        updateStatus?.Invoke("SharePoint Upload...");
+        var uploadFolder = string.IsNullOrWhiteSpace(fetchResult.SharePointUploadFolderOverride)
+            ? spConfig.ExportFolder
+            : fetchResult.SharePointUploadFolderOverride;
+        var uploadLand = fetchResult.SharePointUploadLandOverride ?? site.Land;
        await _appEventLogService.WriteAsync("Export", "SharePoint Upload gestartet",
            siteId: site.Id, land: site.Land,
-            details: $"{spConfig.SiteUrl} | {spConfig.ExportFolder}");
+            details: $"{spConfig.SiteUrl} | {uploadFolder}");
        await _sharePointService.UploadAsync(
            spConfig.TenantId, spConfig.ClientId, spConfig.ClientSecret,
-            spConfig.SiteUrl, spConfig.ExportFolder, site.Land, filePath);
+            spConfig.SiteUrl, uploadFolder, uploadLand, filePath);
    }

    private static string NormalizeSourceSystem(string? sourceSystem)
@@ -409,13 +409,14 @@ public sealed class StandortePageService : IStandortePageService
        var trimmedPath = manualImportFilePath.Trim();
        if (string.IsNullOrWhiteSpace(trimmedPath))
            throw new InvalidOperationException("Bitte zuerst einen Dateipfad eintragen.");
-        if (!IsSupportedManualImportFile(trimmedPath))
+        var isSharePointReference = LooksLikeSharePointReference(trimmedPath);
+        if (!isSharePointReference && !IsSupportedManualImportFile(trimmedPath))
            throw new InvalidOperationException("Bitte eine Excel- oder CSV-Datei mit Endung .xlsx oder .csv angeben.");

        if (File.Exists(trimmedPath))
            return File.GetLastWriteTimeUtc(trimmedPath);

-        if (!LooksLikeSharePointReference(trimmedPath))
+        if (!isSharePointReference)
            throw new InvalidOperationException($"Datei nicht gefunden oder nicht erreichbar: {trimmedPath}");

        await using var db = await _dbFactory.CreateDbContextAsync();
@@ -429,8 +430,16 @@ public sealed class StandortePageService : IStandortePageService
            throw new InvalidOperationException("Fuer SharePoint-Pruefung fehlt eine vollstaendige SharePoint-Konfiguration in Settings.");
        }

+        var sharePointFileReference = trimmedPath;
+        if (!IsSupportedManualImportFile(trimmedPath))
+        {
+            var latestFile = await _sharePointService.ResolveLatestFileInFolderAsync(
+                spConfig.TenantId, spConfig.ClientId, spConfig.ClientSecret, spConfig.SiteUrl, trimmedPath, string.Empty);
+            sharePointFileReference = latestFile.FileReference;
+        }
+
        var tempPath = await _sharePointService.DownloadToTempFileAsync(
-            spConfig.TenantId, spConfig.ClientId, spConfig.ClientSecret, spConfig.SiteUrl, trimmedPath);
+            spConfig.TenantId, spConfig.ClientId, spConfig.ClientSecret, spConfig.SiteUrl, sharePointFileReference);
        try
        {
            return File.GetLastWriteTimeUtc(tempPath);
@@ -448,7 +457,7 @@ public sealed class StandortePageService : IStandortePageService
        var deleteAfterRead = !string.Equals(filePath, manualImportFilePath?.Trim(), StringComparison.OrdinalIgnoreCase);
        try
        {
-            return string.Equals(Path.GetExtension(manualImportFilePath?.Trim()), ".csv", StringComparison.OrdinalIgnoreCase)
+        return string.Equals(Path.GetExtension(filePath), ".csv", StringComparison.OrdinalIgnoreCase)
                ? LoadCsvHeaders(filePath)
                : LoadExcelHeaders(filePath);
        }
@@ -482,8 +491,16 @@ public sealed class StandortePageService : IStandortePageService
            throw new InvalidOperationException("Fuer SharePoint-Pruefung fehlt eine vollstaendige SharePoint-Konfiguration in Settings.");
        }

+        var sharePointFileReference = trimmedPath;
+        if (!IsSupportedManualImportFile(trimmedPath))
+        {
+            var latestFile = await _sharePointService.ResolveLatestFileInFolderAsync(
+                spConfig.TenantId, spConfig.ClientId, spConfig.ClientSecret, spConfig.SiteUrl, trimmedPath, string.Empty);
+            sharePointFileReference = latestFile.FileReference;
+        }
+
        return await _sharePointService.DownloadToTempFileAsync(
-            spConfig.TenantId, spConfig.ClientId, spConfig.ClientSecret, spConfig.SiteUrl, trimmedPath);
+            spConfig.TenantId, spConfig.ClientId, spConfig.ClientSecret, spConfig.SiteUrl, sharePointFileReference);
    }

    private static void ApplyServer(HanaServer target, HanaServer source)