Use credentials in Structure Query and add endpoint for testing it

Source/Applications/openXDA/openXDA/Controllers/SystemCenter/SystemCenterController.cs

@@ -23,20 +23,40 @@
 
 using FaultData.DataResources;
 using FaultData.DataSets;
+using GSF.Configuration;
 using GSF.Data;
+using GSF.Data.Model;
 using log4net;
 using Newtonsoft.Json.Linq;
+using openXDA.Model;
+using openXDA.Model.SystemCenter;
 using openXDA.Nodes;
 using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Data;
+using System.Net;
+using System.Net.Http;
 using System.Threading;
 using System.Threading.Tasks;
 using System.Web.Http;
+using static FaultData.DataOperations.TVA.StructureQueryOperation;
+using ConfigurationLoader = openXDA.Nodes.ConfigurationLoader;
 
 namespace openXDA.Controllers.Config
 {
     [RoutePrefix("api/SystemCenter")]
     public class SystemCenterController : ApiController
     {
+        #region [ Members ]
+        private class Settings
+        {
+            [Category]
+            [SettingName(StructureQuerySection.CategoryName)]
+            public StructureQuerySection StructureQuerySettings { get; } = new StructureQuerySection();
+        }
+        #endregion
+
         #region [ Constructors ]
 
         public SystemCenterController(Host nodeHost)
@@ -111,6 +131,80 @@ public IHttpActionResult GetSCADAPointHealth()
             return Ok(scadaDataResource.GetHistorianHealth());
         }
 
+        [Route("StructureCrawler/Health")]
+        public IHttpActionResult GetStructureCrawlerHealth()
+        {
+
+            AppStatus status = new AppStatus()
+            {
+                Status = "N/A"
+            };
+
+            Settings settings = new Settings();
+            GetConfigurator()(settings);
+
+            if (!settings.StructureQuerySettings.Enabled)
+                return Ok(status);
+
+            int stationKey;
+            string lineKey;
+
+            string query = @"
+                SELECT TOP (1) *,
+                (SELECT TOP (1) assetLocation.LocationID FROM AssetLocation assetLocation WHERE assetLocation.AssetID = asset.ID) AS LocationID
+                FROM Asset asset
+                    WHERE AssetTypeID = 1
+            ";
+
+            using (AdoDataConnection connection = CreateDbConnection())
+            {
+                DataTable result = connection.RetrieveData(query);
+
+                if (result.Rows.Count == 0)
+                    return Ok(status);
+
+                if (result.Rows[0].IsNull("LocationID") || result.Rows[0].IsNull("AssetKey"))
+                    return Ok(status);
+
+                stationKey = result.Rows[0].Field<int>("LocationID");
+                lineKey = result.Rows[0].Field<string>("AssetKey");
+            }
+            try
+            {
+                string url = string.Format(settings.StructureQuerySettings.URLFormat, stationKey, lineKey, 1);
+
+                ICredentials credentials = null;
+                if (settings.StructureQuerySettings.UserName != null && settings.StructureQuerySettings.Password != null && settings.StructureQuerySettings.Domain != null)
+                {
+                    NetworkCredential networkCredential = new NetworkCredential(settings.StructureQuerySettings.UserName, settings.StructureQuerySettings.Password, settings.StructureQuerySettings.Domain);
+                    CredentialCache cache = new CredentialCache();
+                    cache.Add(new Uri(url), "NTLM", networkCredential);
+                    credentials = cache;
+                }
+
+                GetStructureInfo(url, credentials);
+                status.Status = "Success";
+                status.Details.Add(new StatusItem() { Status = "Success", Description = "Successful response received from Structure Crawler." });
+            }
+            catch(Exception ex)
+            {
+                status.Status = "Error";
+                if (ex is HttpRequestException httpRequestException)
+                    status.Details.Add(new StatusItem() { Status = "Error", Description = "Structure query received invalid response. Check the logs for full details." });
+                else if (ex is UriFormatException uriFormatException)
+                    status.Details.Add(new StatusItem() { Status = "Error", Description = "Url formatting failed. Check StructureQuery.URLFormat in openXDA settings." });
+                else if (ex is UnauthorizedAccessException)
+                    status.Details.Add(new StatusItem() { Status = "Error", Description = "Failed to authorize structure query. Check the StructureQuery.UserName and StructureQuery.Password in openXDA settings." });
+                else
+                {
+                    Log.Error($"Unexpected exception thrown during Structure Crawler Query.", ex);
+                    status.Details.Add(new StatusItem() { Status = "Error", Description = "Unexpected exception thrown during Structure Crawler query. Full exception message is available in openXDA logs." });
+                }
+            }
+
+            return Ok(status);
+        }
+
         private Action<object> GetConfigurator()
         {
             ConfigurationLoader configurationLoader = new ConfigurationLoader(NodeHost.ID, CreateDbConnection);

Source/Libraries/FaultData/DataOperations/TVA/StructureQueryOperation.cs

@@ -35,7 +35,9 @@
 using GSF.Data;
 using GSF.Data.Model;
 using HtmlAgilityPack;
+using log4net;
 using openXDA.Model;
+using System.Net.Http;
 
 namespace FaultData.DataOperations.TVA
 {
@@ -55,6 +57,22 @@ public class StructureQuerySection
             [Setting]
             [DefaultValue("StrNumber:AssetKey,Latitude:Latitude,Longitude:Longitude")]
             public string FieldMappings { get; set; }
+
+            [Setting]
+            [DefaultValue("")]
+            public string UserName { get; set; }
+
+            [Setting]
+            [DefaultValue("")]
+            public string Password { get; set; }
+
+            [Setting]
+            [DefaultValue("")]
+            public string Domain { get; set; }
+
+            [Setting]
+            [DefaultValue(false)]
+            public bool Enabled { get; set; } = false;
         }
 
         // Constants
@@ -94,6 +112,12 @@ private Func<string, string> FieldMappingLookup
 
         public override void Execute(MeterDataSet meterDataSet)
         {
+            if (!Settings.Enabled)
+            {
+                Log.Warn("Structure Query Operation is not enabled.");
+                return;
+            }
+
             FaultDataResource faultDataResource = meterDataSet.GetResource<FaultDataResource>();
             string stationKey = meterDataSet.Meter.Location.LocationKey;
 
@@ -121,7 +145,18 @@ public override void Execute(MeterDataSet meterDataSet)
                         return;
 
                     string url = string.Format(Settings.URLFormat, stationKey, lineKey, distance);
-                    string structureInfo = GetStructureInfo(url);
+
+                    ICredentials credentials = null;
+                    if (Settings.UserName != null && Settings.Password != null && Settings.Domain != null)
+                    {
+                        NetworkCredential networkCredential = new NetworkCredential(Settings.UserName, Settings.Password, Settings.Domain);
+                        CredentialCache cache = new CredentialCache();
+                        cache.Add(new Uri(url), "NTLM", networkCredential);
+                        credentials = cache;
+                    }
+
+                    string structureInfo = GetStructureInfo(url, credentials);
+
                     DataTable structureData = ToDataTable(structureInfo);
 
                     if (structureData.Rows.Count == 0)
@@ -178,19 +213,39 @@ public override void Execute(MeterDataSet meterDataSet)
                 }
             }
         }
+        public static string GetStructureInfo(string url, ICredentials credentials = null)
+        {
+            HttpClientHandler handler;
+
+            if (credentials is null)
+                handler = new HttpClientHandler() { UseDefaultCredentials = true };
+            else
+                handler = new HttpClientHandler() { Credentials = credentials };
 
-        private static string GetStructureInfo(string url)
+            HttpClient client = new HttpClient(handler);
+            string html;
+            HttpResponseMessage response = client.GetAsync(url).GetAwaiter().GetResult();
+
+            if (response.StatusCode is HttpStatusCode.Forbidden)
         {
-            bool HandlePreRequest(HttpWebRequest request)
+                string responseContent = response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
+                throw new HttpRequestException($"Structure query received 'Forbidden' response: {responseContent}");
+            }
+
+            if (response.StatusCode is HttpStatusCode.InternalServerError)
             {
-                request.UseDefaultCredentials = true;
-                return true;
+                string responseContent = response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
+                throw new HttpRequestException($"Structure query received 'InternalServerError' response: {responseContent}");
             }
 
-            HtmlWeb webClient = new HtmlWeb();
-            webClient.PreRequest += HandlePreRequest;
-            HtmlDocument doc = webClient.Load(url);
-            return doc.DocumentNode.InnerText.Trim();
+            response.EnsureSuccessStatusCode();
+
+            html = response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
+            HtmlDocument doc = new HtmlDocument();
+            doc.LoadHtml(html);
+
+            string result = doc.DocumentNode.Descendants("html").FirstOrDefault().InnerText.Trim();
+            return result;
         }
 
         private DataTable ToDataTable(string csvInput)
@@ -225,5 +280,12 @@ private double ToDouble(string csvValue) =>
                 : 0.0D;
 
         #endregion
+
+        #region [ Static ]
+
+        // Static Fields
+        private static readonly ILog Log = LogManager.GetLogger(typeof(StructureQueryOperation));
+
+        #endregion
     }
 }