Merge pull request #14552 from ppekrol/v5.4

5.3 to 5.4 merge

src/Raven.Client/Documents/Linq/LinqPathProvider.cs

@@ -80,10 +80,19 @@ public Result GetPath(Expression expression, bool isFilterActive = false)
 
                 if (callExpression.Method.Name == "get_Item")
                 {
-                    var parent = GetPath(callExpression.Object);
-
                     var itemKey = GetValueFromExpression(callExpression.Arguments[0], callExpression.Method.GetParameters()[0].ParameterType).ToString();
 
+                    if (callExpression.Object?.NodeType == ExpressionType.Parameter)
+                    {
+                        return new Result
+                        {
+                            MemberType = callExpression.Method.ReturnType,
+                            IsNestedPath = false,
+                            Path = itemKey
+                        };
+                    }
+
+                    var parent = GetPath(callExpression.Object);
                     return new Result
                     {
                         MemberType = callExpression.Method.ReturnType,

src/Raven.Client/Documents/Linq/RavenQueryProviderProcessor.cs

@@ -510,6 +510,8 @@ private bool IsMemberAccessForQuerySource(Expression node)
             }
             if (node.NodeType == ExpressionType.Parameter)
                 return true;
+            if (node.NodeType == ExpressionType.Call && node is MethodCallExpression callExpressionNode && callExpressionNode.Method.Name == "get_Item" && callExpressionNode.Object?.NodeType == ExpressionType.Parameter)
+                return true;
             if (node.NodeType != ExpressionType.MemberAccess)
                 return false;
             var memberExpression = ((MemberExpression)node);

src/Raven.Client/Documents/Queries/Explanation/Explanations.cs

@@ -6,6 +6,8 @@ public class Explanations
     {
         private Dictionary<string, string[]> _explanations;
 
+        internal bool ShouldBeIncluded { get; set; } = false;
+
         public string[] GetExplanations(string key)
         {
             if (_explanations.TryGetValue(key, out var results) == false)

src/Raven.Client/Documents/Session/AbstractDocumentQuery.Explanations.cs

@@ -6,7 +6,7 @@ namespace Raven.Client.Documents.Session
 {
     public abstract partial class AbstractDocumentQuery<T, TSelf>
     {
-        protected Explanations Explanations;
+        protected Explanations Explanations = new Explanations();
 
         protected ExplanationToken ExplanationToken;
 
@@ -17,7 +17,9 @@ public void IncludeExplanations(ExplanationOptions options, out Explanations exp
 
             var optionsParameterName = options != null ? AddQueryParameter(options) : null;
             ExplanationToken = ExplanationToken.Create(optionsParameterName);
-            Explanations = explanations = new Explanations();
+            Explanations.ShouldBeIncluded = true;
+            explanations = Explanations;
         }
+
     }
 }

src/Raven.Client/Documents/Session/AbstractDocumentQuery.cs

@@ -1484,8 +1484,10 @@ private void UpdateStatsHighlightingsAndExplanations(QueryResult queryResult)
         {
             QueryStats.UpdateQueryStats(queryResult);
             QueryHighlightings.Update(queryResult);
-            Explanations?.Update(queryResult);
-            QueryTimings?.Update(queryResult);
+            if (Explanations.ShouldBeIncluded)
+                Explanations.Update(queryResult);
+            if (QueryTimings.ShouldBeIncluded)
+                QueryTimings.Update(queryResult);
         }
 
         private void BuildSelect(StringBuilder writer)

src/Raven.Client/Http/NodeSelector.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Threading;
 using Raven.Client.Exceptions;
 using Raven.Client.Exceptions.Database;
@@ -66,12 +67,19 @@ public bool OnUpdateTopology(Topology topology, bool forceUpdate = false)
             var state = _state;
             var stateFailures = state.Failures;
             var serverNodes = state.Nodes;
-            var len = Math.Min(serverNodes.Count, stateFailures.Length);
-            for (var i = 0; i < len; i++)
+
+            Debug.Assert(serverNodes.Count == stateFailures.Length, $"Expected equals {serverNodes.Count}, but got {stateFailures.Length}");
+
+            if (serverNodes.Count == 1 && serverNodes[0].ClusterTag == nodeTag) // If this is a cluster with 1 node return it without checking it's failure.
+                return (0, serverNodes[0]);
+
+            for (var i = 0; i < serverNodes.Count; i++)
             {
                 if (serverNodes[i].ClusterTag == nodeTag)
                 {
-                    if (stateFailures[i] == 0 && string.IsNullOrEmpty(serverNodes[i].Url) == false)
+                    Debug.Assert(string.IsNullOrEmpty(serverNodes[i].Url) == false, $"Expected serverNodes Url not null or empty but got: \'{serverNodes[i].Url}\'");
+
+                    if (stateFailures[i] == 0)
                         return (i, serverNodes[i]);
 
                     throw new RequestedNodeUnavailableException($"Requested node {nodeTag} currently unavailable, please try again later.");

src/Raven.Server/Dashboard/ThreadsInfo.cs

@@ -9,18 +9,23 @@ namespace Raven.Server.Dashboard
 {
     public class ThreadsInfo : IDynamicJson
     {
+        private readonly int? _take;
+
         public DateTime Date => SystemTime.UtcNow;
 
         public SortedSet<ThreadInfo> List { get; }
 
         public double CpuUsage { get; set; }
+
+        public double ProcessCpuUsage { get; set; }
 
         public long ActiveCores { get; set; }
 
         public long ThreadsCount => List.Count;
 
-        public ThreadsInfo()
+        public ThreadsInfo(int? take)
         {
+            _take = take;
             List = new SortedSet<ThreadInfo>(new ThreadsInfoComparer());
         }
 
@@ -47,9 +52,10 @@ public DynamicJsonValue ToJson()
             {
                 [nameof(Date)] = Date,
                 [nameof(CpuUsage)] = CpuUsage,
+                [nameof(ProcessCpuUsage)] = ProcessCpuUsage,
                 [nameof(ActiveCores)] = ActiveCores,
                 [nameof(ThreadsCount)] = ThreadsCount,
-                [nameof(List)] = new DynamicJsonArray(List.Select(x => x.ToJson()))
+                [nameof(List)] = new DynamicJsonArray(List.Take(_take ?? int.MaxValue).Select(x => x.ToJson()))
             };
         }
     }

src/Raven.Server/Documents/Handlers/Admin/AdminLogsHandler.cs

@@ -1,12 +1,19 @@
-using System.Threading.Tasks;
+using System;
+using System.IO;
+using System.IO.Compression;
+using System.Threading.Tasks;
 using Raven.Client.ServerWide.Operations.Logs;
+using Raven.Server.Indexing;
 using Raven.Server.Json;
 using Raven.Server.Routing;
+using Raven.Server.ServerWide;
 using Raven.Server.Web;
 using Sparrow;
 using Sparrow.Json;
 using Sparrow.Json.Parsing;
 using Sparrow.Logging;
+using Sparrow.Server.Platform.Posix;
+using Sparrow.Utils;
 
 namespace Raven.Server.Documents.Handlers.Admin
 {
@@ -77,5 +84,68 @@ public async Task RegisterForLogs()
                 await LoggingSource.Instance.Register(socket, context, ServerStore.ServerShutdown);
             }
         }
+
+        [RavenAction("/admin/logs/download", "GET", AuthorizationStatus.Operator)]
+        public async Task Download()
+        {
+            var contentDisposition = $"attachment; filename={DateTime.UtcNow:yyyy-MM-dd H:mm:ss} - Node [{ServerStore.NodeTag}] - Logs.zip";
+            HttpContext.Response.Headers["Content-Disposition"] = contentDisposition;
+            HttpContext.Response.Headers["Content-Type"] = "application/zip";
+
+            var adminLogsFileName = $"admin.logs.download.{Guid.NewGuid():N}";
+            var adminLogsFilePath = ServerStore._env.Options.DataPager.Options.TempPath.Combine(adminLogsFileName);
+
+            var from = GetDateTimeQueryString("from", required: false);
+            var to = GetDateTimeQueryString("to", required: false);
+
+            using (var stream = SafeFileStream.Create(adminLogsFilePath.FullPath, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.ReadWrite, 4096,
+                       FileOptions.DeleteOnClose | FileOptions.SequentialScan))
+            {
+                using (var archive = new ZipArchive(stream, ZipArchiveMode.Create, true))
+                {
+                    foreach (var filePath in Directory.GetFiles(ServerStore.Configuration.Logs.Path.FullPath))
+                    {
+                        var fileName = Path.GetFileName(filePath);
+                        if (fileName.EndsWith(LoggingSource.LogInfo.LogExtension, StringComparison.OrdinalIgnoreCase) == false &&
+                            fileName.EndsWith(LoggingSource.LogInfo.FullCompressExtension, StringComparison.OrdinalIgnoreCase) == false)
+                            continue;
+
+                        var hasLogDateTime = LoggingSource.LogInfo.TryGetDate(filePath, out var logDateTime);
+                        if (hasLogDateTime)
+                        {
+                            if (from != null && logDateTime < from)
+                                continue;
+
+                            if (to != null && logDateTime > to)
+                                continue;
+                        }
+
+                        try
+                        {
+                            var entry = archive.CreateEntry(fileName);
+                            if (hasLogDateTime)
+                                entry.LastWriteTime = logDateTime;
+
+                            using (var fs = File.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
+                            {
+                                entry.ExternalAttributes = ((int)(FilePermissions.S_IRUSR | FilePermissions.S_IWUSR)) << 16;
+
+                                await using (var entryStream = entry.Open())
+                                {
+                                    await fs.CopyToAsync(entryStream);
+                                }
+                            }
+                        }
+                        catch (Exception e)
+                        {
+                            await DebugInfoPackageUtils.WriteExceptionAsZipEntryAsync(e, archive, fileName);
+                        }
+                    }
+                }
+
+                stream.Position = 0;
+                await stream.CopyToAsync(ResponseBodyStream());
+            }
+        }
     }
 }

src/Raven.Server/Documents/Handlers/Debugging/ThreadsHandler.cs

@@ -8,7 +8,9 @@
 using System.Threading.Tasks;
 using Newtonsoft.Json.Linq;
 using Raven.Client.Documents.Conventions;
+using Raven.Server.Dashboard;
 using Raven.Server.Routing;
+using Raven.Server.ServerWide;
 using Raven.Server.ServerWide.Context;
 using Raven.Server.Utils;
 using Raven.Server.Web;
@@ -46,7 +48,7 @@ await using (var sw = new StringWriter())
                     await Task.Delay((int)wait);
                 }
 
-                var threadStats = threadsUsage.Calculate(threadIdsAsString.Count == 0 ? null : threadIdsAsString.Select(int.Parse).ToHashSet());
+                var threadStats = threadsUsage.Calculate(threadIds: threadIdsAsString.Count == 0 ? null : threadIdsAsString.Select(int.Parse).ToHashSet());
                 result["Threads"] = JArray.FromObject(threadStats.List);
 
                 using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
@@ -62,23 +64,47 @@ await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBody
         [RavenAction("/admin/debug/threads/runaway", "GET", AuthorizationStatus.Operator, IsDebugInformationEndpoint = true)]
         public async Task RunawayThreads()
         {
+            var samplesCount = GetIntValueQueryString("samplesCount", required: false) ?? 1;
+            var interval = GetIntValueQueryString("intervalInMs", required: false) ?? ServerMetricCacher.DefaultCpuRefreshRateInMs;
+            var maxTopThreads = GetIntValueQueryString("maxTopThreads", required: false);
+
+            if (samplesCount <= 0)
+                throw new ArgumentException("Must be positive", "samplesCount");
+
+            if (interval <= 0)
+                throw new ArgumentException("Must be positive", "interval");
+
             using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
             {
                 await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream()))
                 {
                     try
                     {
-                        var threadsUsage = new ThreadsUsage();
+                        var threadsInfos = await GetThreadsInfos();
 
-                        // need to wait to get a correct measure of the cpu
-                        await Task.Delay(100);
+                        if (samplesCount == 1)
+                        {
+                            context.Write(writer,
+                                new DynamicJsonValue
+                                {
+                                    ["Runaway Threads"] = threadsInfos.First().ToJson()
+                                });
+                            return;
+                        }
 
-                        var result = threadsUsage.Calculate();
-                        context.Write(writer,
-                            new DynamicJsonValue
-                            {
-                                ["Runaway Threads"] = result.ToJson()
-                            });
+                        writer.WriteStartObject();
+                        writer.WritePropertyName("Results");
+
+                        var dja = new DynamicJsonArray();
+
+                        foreach (var threadInfo in threadsInfos)
+                        {
+                            dja.Add(threadInfo.ToJson());
+                        }
+
+                        context.Write(writer, dja);
+                        writer.WriteEndObject();
+
                     }
                     catch (Exception e)
                     {
@@ -90,6 +116,21 @@ await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBody
                     }
                 }
             }
+
+            async Task<List<ThreadsInfo>> GetThreadsInfos()
+            {
+                var results = new List<ThreadsInfo>();
+
+                var threadsUsage = new ThreadsUsage();
+
+                for (var i = 0; i < samplesCount; i++)
+                {
+                    await Task.Delay(interval, ServerStore.ServerShutdown);
+                    results.Add(threadsUsage.Calculate(maxTopThreads));
+                }
+
+                return results;
+            }
         }
 
         [MethodImpl(MethodImplOptions.Synchronized)]

src/Raven.Server/Documents/Indexes/Debugging/IndexDebugExtensions.cs

@@ -434,7 +434,15 @@ private static BlittableJsonReaderObject GetReduceResult(ulong reduceKeyHash, In
                     return context.ReadObject(new DynamicJsonValue(), "debug-reduce-result");
 
                 if (result.Count > 1)
-                    throw new InvalidOperationException("Cannot have multiple reduce results for a single reduce key");
+                {
+                    var dvj = new DynamicJsonValue
+                    {
+                        ["MergedResults"] = true,
+                        ["TotalNumberOfResults"] = result.Count,
+                        ["Results"] = new DynamicJsonArray(result.Select(x=>x.Result.Data))
+                    };
+                    return context.ReadObject(dvj, "merged-map-reduce");
+                }
 
                 return result[0].Result.Data;
             }

src/Raven.Server/Documents/Indexes/Index.cs

@@ -18,6 +18,7 @@
 using Raven.Client.Documents.Queries;
 using Raven.Client.Exceptions.Database;
 using Raven.Client.Exceptions.Documents.Indexes;
+using Raven.Client.Extensions;
 using Raven.Client.ServerWide.Operations;
 using Raven.Client.Util;
 using Raven.Server.Config;
@@ -2114,6 +2115,11 @@ private void FlushAndSync(StorageEnvironment storageEnvironment, int timeToWaitI
                 // index was deleted or database was shutdown
                 return;
             }
+            catch (AggregateException ae) when (ae.ExtractSingleInnerException() is OperationCanceledException)
+            {
+                // index was deleted or database was shutdown
+                return;
+            }
 
             storageEnvironment.Cleanup(tryCleanupRecycledJournals);
         }

src/Raven.Server/NotificationCenter/OutOfMemoryNotifications.cs

@@ -6,6 +6,7 @@
 using Raven.Server.NotificationCenter.Notifications.Details;
 using Sparrow;
 using Sparrow.LowMemory;
+using Sparrow.Utils;
 using Voron;
 
 namespace Raven.Server.NotificationCenter
@@ -75,14 +76,7 @@ private static MessageDetails OutOfMemoryDetails(Exception exception)
 
             return new MessageDetails
             {
-                Message = $"Managed memory: {new Size(AbstractLowMemoryMonitor.GetManagedMemoryInBytes(), SizeUnit.Bytes)}, " +
-                          $"Unmanaged allocations: {new Size(AbstractLowMemoryMonitor.GetUnmanagedAllocationsInBytes(), SizeUnit.Bytes)}, " +
-                          $"Shared clean: {memoryInfo.SharedCleanMemory}, " +
-                          $"Working set: {memoryInfo.WorkingSet}, " +
-                          $"Available memory: {memoryInfo.AvailableMemory}, " +
-                          $"Calculated Available memory: {memoryInfo.AvailableMemoryForProcessing}, " +
-                          $"Total Scratch Dirty memory: {memoryInfo.TotalScratchDirtyMemory}, " +
-                          $"Total memory: {memoryInfo.TotalPhysicalMemory} {Environment.NewLine}" +
+                Message = $"{MemoryUtils.GetExtendedMemoryInfo(memoryInfo)} {Environment.NewLine}" +
                           $"Error: {exception}"
             };
         }