ELK Rollup data

Scenario:

Roll up data from Elk index at minute level and store in DB

Solution:

1. The data is stored in ELK index "logs". The data to be rolled up to Category/Sub Category at minute level.


2. Pull the data from ELK index for today and some day back and roll up at minute level using below query:

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
GET logs-*2020.01.01&expand_wildcards=all/_search
{
  "size": 0,
  "query": {
    "bool": {
      "must": [
        {
          "range": {
            "DateCreated": {
              "gte": "2020-01-01T15:05:16.6759102-07:00",
              "lt": "2020-01-01T15:08:51.0818714-07:00"
            }
          }
        }
      ]
    }
  },
  "aggs": {
    "Time": {
      "date_histogram": {
        "field": "DateCreated",
        "interval": "minute",
        "time_zone": "America/LA"
      },
      "aggs": {
        "CategoryId": {
          "terms": {
            "field": "CategoryId",
            "size": 500
          },
          "aggs": {
            "SubCategory": {
              "terms": {
                "field": "SubCategory.keyword",
                "size": 500
              },
              "aggs": {
                "SubCategory-Errors": {
                  "filter": {
                    "bool": {
                      "must": [
                        {
                          "term": {
                            "Error": {
                              "value": true
                            }
                          }
                        }
                      ]
                    }
                  }
                },
                "SubCategory-Avg": {
                  "avg": {
                    "field": "ResponseTime"
                  }
                },
                "SubCategory-Percstile": {
                  "percentiles": {
                    "field": "ResponseTime",
                    "percents": [
                      50,
                      75,
                      98
                    ]
                  }
                }
              }
            },
            "ProviderId-Errors": {
              "filter": {
                "exists": {
                  "field": "Error.keyword"
                }
              }
            },
            "ProviderId-Avg": {
              "avg": {
                "field": "ResponseTime"
              }
            },
            "ProviderId-Percstile": {
              "percentiles": {
                "field": "ResponseTime",
                "percents": [
                  50,
                  75,
                  98
                ]
              }
            }
          }
        },
        "Time-Errors": {
          "filter": {
            "bool": {
              "must": [
                {
                  "term": {
                    "Error": {
                      "value": true
                    }
                  }
                }
              ]
            }
          }
        },
        "Time-Avg": {
          "avg": {
            "field": "ResponseTime"
          }
        },
        "Time-Percstile": {
          "percentiles": {
            "field": "ResponseTime",
            "percents": [
              50,
              75,
              98
            ]
          }
        }
      }
    }
  }
}



3. The response would look something like below: 

   
    
   
    1
    2
    3
    4
    5
    6
    7
    8
    9
   10
   11
   12
   13
   14
   15
   16
   17
   18
   19
   20
   21
   22
   23
   24
   25
   26
   27
   28
   29
   30
   31
   32
   33
   34
   35
   36
   37
   38
   39
   40
   41
   42
   43
   44
   45
   46
   47
   48
   49
   50
   51
   52
   53
   54
   55
   56
   57
   58
   59
   60
   61
   62
   63
   64
   65
   66
   67
   68
   69
   70
   71
   72
   73
   74
   {
     "took": 50,
     "timed_out": false,
     "_shards": {
       "total": 1,
       "successful": 1,
       "skipped": 0,
       "failed": 0
     },
     "hits": {
       "total": {
         "value": 1,
         "relation": ""
       },
       "max_score": null,
       "hits": []
     },
     "aggregations": {
       "Time": {
         "buckets": [
           {
             "key_as_string": "2020-01-01T19:00:00.000-05:00",
             "key": 158682241111,
             "doc_count": 6,
             "CategoryId": {
               "doc_count_error_upper_bound": 0,
               "sum_other_doc_count": 0,
               "buckets": [
                 {
                   "key": 10001,
                   "doc_count": 6,
                   "CategoryId-Avg": { "value": 527.6666666666666 },
                   "CategoryId-Percstile": {
                     "values": {
                       "50.0": 695.5,
                       "75.0": 763.0,
                       "98.0": 815.0
                     }
                   },
                   "CategoryId-Errors": { "doc_count": 0 },
                   "SubCategory": {
                     "doc_count_error_upper_bound": 0,
                     "sum_other_doc_count": 0,
                     "buckets": [
                       {
                         "key": "10001.1",
                         "doc_count": 6,
                         "SubCategory-Percstile": {
                           "values": {
                             "50.0": 695.5,
                             "75.0": 763.0,
                             "98.0": 815.0
                           }
                         },
                         "SubCategory-Avg": { "value": 527.6666666666666 },
                         "SubCategory-Errors": { "doc_count": 0 }
                       }
                     ]
                   }
                 }
               ]
             },
             "Time-Percstile": {
               "values": {
                 "50.0": 695.5,
                 "75.0": 763.0,
                 "98.0": 815.0
               }
             },
             "Time-Avg": { "value": 527.6666666666666 },
             "Time-Errors": { "doc_count": 0 }
           }
       }
     }
   
   
   
   
   



4. Below is the code to parse the above response and get into a model:

   
    
   
    1
    2
    3
    4
    5
    6
    7
    8
    9
   10
   11
   12
   13
   14
   15
   16
   17
   18
   19
   20
   21
   22
   23
   24
   25
   26
   27
   28
   29
   30
   31
   32
   33
   34
   35
   36
   37
   38
   39
   40
   41
   42
   43
   44
   45
   46
   47
   48
   49
   50
   51
   52
   53
   54
   55
   56
   57
   58
   59
   60
   61
   62
   63
   64
   65
   66
   67
   68
   69
   70
   71
   72
   73
   74
   75
   76
   77
   78
   79
   80
   81
   82
   83
   84
   85
   86
   87
   88
   89
   90
   91
   92
   93
   94
   95
   96
   97
   98
   99
   public class RollupUser
   {
       public void Process(HttpContext httpContext)
       {
           var model = CreateRequest(httpContext);
   
           using (var outputStream = requestDoc.CreateOutputStream())
           {
               var content = Encoding.Default.GetBytes(requestBody);
               outputStream.Write(content, 0, content.Length);
           }
   
           var request = new Httprequest
           {
               RequestUri = ElkUri,
               Method = HttpMethod.Post,
               Content = new StringContent(requestBody, Encoding.UTF8, "application/json")
           };
   
           var response = new HttpClient().SendAsync(request).Result;
   
           var response = response.Content.ReadAsStringAsync().Result;
   
           using (var outputStream = responseDoc.CreateOutputStream())
           {
               var content = Encoding.Default.GetBytes(response);
               outputStream.Write(content, 0, content.Length);
           }
   
           var output = Read(response);
       }
   
       private ResponseModel Read(string input)
       {
           using (var reader = new JsonTextReader(new StringReader(input)))
           {
               reader.DateParseHandling = DateParseHandling.None;
               r = JObject.Load(reader);
           }
   
           var buckets = (JArray)r["aggregations"]["Time"]["buckets"];
   
           foreach (var bucket in buckets)
           {
               var bucket1 = bucket;
   
               logs.Add(
                   {
                   Date = date,
                       CategoryId = -1,
                       SubCategory = "_ALL"
                       Calls = (int)bucket["doc_count"],
                       Errors = (int)bucket["Time-Errors"]["doc_count"],
                       AvgTimeElapsed = GetValue(() => (int)bucket1["Time-Avg"]["value"]),
                       P50TimeElapsed = GetValue(() => (int)bucket1["Time-Percstile"]["values"]["50.0"]),
                       P75TimeElapsed = GetValue(() => (int)bucket1["Time-Percstile"]["values"]["75.0"]),
                       P98TimeElapsed = GetValue(() => (int)bucket1["Time-Percstile"]["values"]["98.0"])
                   });
   
               foreach (var categoryIdBucket in (JArray)bucket["CategoryId"]["buckets"])
               {
                   var bucket2 = categoryIdBucket;
                   logs.Add(
                       {
                       Date = date,
                           CategoryId = (int)categoryIdBucket["key"],
                           SubCategory = "_ALL",
                           Calls = (int)categoryIdBucket["doc_count"],
                           Errors = (int)categoryIdBucket["CategoryId-Errors"]["doc_count"],
                           AvgTimeElapsed = GetValue(() => (int)bucket2["CategoryId-Avg"]["value"]),
                           P50TimeElapsed = GetValue(() => (int)bucket2["CategoryId-Percstile"]["values"]["50.0"]),
                           P75TimeElapsed = GetValue(() => (int)bucket2["CategoryId-Percstile"]["values"]["75.0"]),
                           P98TimeElapsed = GetValue(() => (int)bucket2["CategoryId-Percstile"]["values"]["98.0"])
                       });
   
                   foreach (var SubCategoryBucket in (JArray)categoryIdBucket["SubCategory"]["buckets"])
                   {
                       var bucket3 = SubCategoryBucket;
                       logs.Add(
                           {
                           Date = date,
                               CategoryId = CategoryId,
                               SubCategory = (string)SubCategoryBucket["key"],
                               Calls = (int)SubCategoryBucket["doc_count"],
                               Errors = (int)SubCategoryBucket["SubCategory-Errors"]["doc_count"],
                               AvgTimeElapsed = GetValue(() => (int)bucket3["SubCategory-Avg"]["value"]),
                               P50TimeElapsed = GetValue(() =>
                                   (int)bucket3["SubCategory-Percstile"]["values"]["50.0"]),
                               P75TimeElapsed = GetValue(() =>
                                   (int)bucket3["SubCategory-Percstile"]["values"]["75.0"]),
                               P98TimeElapsed = GetValue(() =>
                                   (int)bucket3["SubCategory-Percstile"]["values"]["98.0"])
                           });
                   }
               }
           }
           return logs;
       }
   }
   
   
   
   
   

Session management service [cache based]

Scenario:

Create a Sign on service to allow creation of users in our app. Add data added details to session (Cache based)

Solution:

1. Add SessionServ



1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
public class SessionServ
{
    private readonly ISessionRepo _repo;

    public SessionServ(ISessionRepo repository)
    {
        _repo = repository;
    }


    public string StartSession(string sessionId)
    {
        sessInfo sessInfo;

        // if new sessionId not set
        sessInfo = new sessInfo(_repo.NewId());
        _repo.Update(sessInfo);
        sessInfo.SetCurrent(sessInfo);
        return sessInfo.SessionId;
    }

    public void StopSession()
    {
        if (sessInfo.IsInitialized)
        {
            _repo.Remove(sessInfo.Current);
        }
    }

    public sessInfo GetSession(string sessionId)
    {
        sessInfo sessInfo;

        if (!_repo.Find(sessionId, out sessInfo))
            throw exception;

        return sessInfo;
    }

    public void SaveSession(sessInfo sessInfo)
    {
        _repo.Update(sessInfo);
    }

    public void SetProperty(string name, string value)
    {
        if (sessInfo.IsInitialized)
        {
            sessInfo.Current.SetProperty(name, value);
        }
    }

    public string GetPropertyBySessionId(string sessionId, string name)
    {
        sessInfo sessInfo;
        return _repo.Find(sessionId, out sessInfo)
                   ? sessInfo.GetProperty(name)
                   : null;
    }

    public void RemoveProperty(string name)
    {
        if (sessInfo.IsInitialized)
        {
            sessInfo.Current.RemoveProperty(name);
        }
    }
}



2. Add CacheSesRepo



 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
public class CacheSesRepo : ISessionRepo
{
    private readonly ICache<sessInfo> _session;

    public CacheSesRepo([AppSettingsBinding(Name = "CacheServ")] ICacheSer cs)
    {
        _session = ..GetCache<sessInfo>(sessinfo);
    }

    private const string keypref = "_session_";

    public string NewId()
    {
        return Guid.NewGuid().ToString("N"); ;
    }

    public bool Find(string sessionId, out sessInfo sessInfo)
    {
        !_session.TryGet(GetCache(keypref + sessionId), out sessInfo);
    }

    public bool Update(sessInfo sessInfo)
    {
        _session.Add(cacheKey, sessInfo);
    }

    public bool Remove(sessInfo sessInfo)
    {
        _session.Remove(cacheKey);
    }
}



3. Add SessHttpModule 



 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
public class SessHttpModule : BaseHttpModule
{
    protected override void OnBeginRequest(HttpContext context, EventArgs e)
    {
        BeginSession(context);
    }

    protected override void OnEndRequest(HttpContext context, EventArgs e)
    {
        EndSession(context);
    }
}



4. Use session as below



1 2 3 4 5

//Set _sessionServ.SetProperty(Constants.UserId, "123"); //Get var userId = _sessionServ.GetProperty(Constants.UserId);

Baretail log viewer

Baretail is a log viewing  editor where particular type of logs can be be color coded for easier analysis as well as it supports auto load of data as log is being written to it..

1. It can be dowloaded ffrom https://www.baremetalsoft.com/baretail/
2. Click on Highlighting menu -> set string and color -> Add. Example: string as error -> color as Red. Now in the log all lines with error text would be highlighted with red color.

API data validation

Scenario:

Create a Sign on service to allow creation of users in our app. Add validation for max/min length for UserName mode

Solution:

1. 
   

1
2
3
[MaxLength(100, ErrorMessage = "Invalidparam*maximum length of 100*userName")]
[MinLength(10)]
public string UserName { get; set; }



2. 
   

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
var errors = modelState.Where(ms => ms.Value.Errors != null && ms.Value.Errors.Count > 0)
                .SelectMany(ms => ms.Value.Errors)
                .Select(e =>
                {
                    if (e.message.Contains("*"))
                    {
                        var details = e.message.Split('*');
                        return new Error
                        {
                            Code = details[0],
                            Description = details[1],
                            Parameter = details[2]
                        };
                    }

                    return new Error
                    { Code = "InvalidRequest", Details = e.message };
                }).ToList();

Session - Set/Get session in service

Scenario:

Create a Sign on service to allow creation of users in our app. Add data added details to session

Solution:

1. Add NuGet Microsoft.AspNet.Core.Session
2. In Startup.cs under Configure method



//add before UseEndpoints or if using Mvc as before UseMvc
app.UseSession();



3. In Startup.cs under ConfigureServices method



1
2
services.AddDistributedMemoryCache();
services.AddSession(options => { options.Cookie.Name = ".solutionName.Session"; options.IdleTimeout = TimeSpan.FromMinutes(5); });



4. Set IHttpContextAccessor



1
2
3
4
5
6
7
private readonly IHttpContextAccessor _httpContextAccessor;

public SignOnService(ILogger<SignOnService> logger, IHttpContextAccessor httpContextAccessor)
{
 _logger = logger;
 _httpContextAccessor = httpContextAccessor;
}



5. Set session data as below



1 2 3 4 5 6 7 8 9 10 11 12 13

//return back to client one user at a time foreach (var user in users) { try { _httpContextAccessor.HttpContext.Session.SetString("message", "Processed request at : " + DateTime.Now.ToString()); await responseStream.WriteAsync(user); } catch (Exception e) { throw; } }

Session - Set/Get session in Web

Scenario:

Create a Sign on service to allow creation of users in our app. Add user details to session

Solution:

1. Add NuGet Microsoft.AspNet.Core.Session
2. In Startup.cs under Configure method



//add before UseEndpoints or if using Mvc as before UseMvc
app.UseSession();



3. In Startup.cs under ConfigureServices method



1
2
services.AddDistributedMemoryCache();
services.AddSession(options => { options.Cookie.Name = ".solutionName.Session"; options.IdleTimeout = TimeSpan.FromMinutes(5); });



4. Set session as below



 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
try
{
 var userList = string.Join(",", result.Item2.ToArray());
 HttpContext.Session.SetString("users", userList + " in Home controller");
 ViewBag.success = true;
 ViewBag.Name = userList;
}
catch (Exception e)
{
 throw;
}



5. Read session data as below



1 2 var users = HttpContext.Session.GetString("users"); ViewBag.Name = users;

ELK 5.6 queries guide

Scenarios:

--Get all records { "query": { "match_all": {} } } --Single term { "query": { "term": { "UserName.raw": "Jerry" } } } --date range { "query": { "range": { "Date": { "gt": "2020-01-22T20:00:00.3638639-06:00", "lt": "2020-01-22T21:00:00.3638639-06:00" } } } } -- "range": { "DateRequested": { "gte": "now-30m" } --multiple terms { "query": { "terms": { "UserName.raw": [ "Jerry", "Terry" ] } }, "sort": [ { "Date": { "order": "desc" } } ] } --Single term wild card { "query": { "wildcard": { "UserName.raw": { "value": "*rry" } } } } --Single term exists { "query": { "exists": { "field": "Local" } } } --multiple conditions and select columns { "query": { "filtered": { "query": { "wildcard": { "UserName.raw": { "value": "*rry" } } }, "filter": { "term": { "City.raw": "NeverLand" } } } }, "fields": ["UserId", "Name"] } { "query": { "bool": { "filter": [ { "terms": { "groupIds": [ "123" ] } }, { "term": { "userid": { "value": "1" } } }, { "term": { "name": { "value": "Ram" } } } ] } }, "size": 9999 } --true match { "query": { "bool": { "must": [ { "term": { "City.raw": "NeverLand" } } } ] } } } --false match { "query": { "bool": { "must_not": [ { "term": { "City.raw": "NeverLand" } } } ] } } } --aggregate { "query": { "term": { "City.raw": "NeverLand" } }, "query": { "wildcard": { "UserName.raw": { "value": "*rry" } } }, "aggs": { "Avg" : { "avg": { "field":"Time" } } } } --group by aggregate { "query": { "filtered": { "query": { "bool": { "must": [ { "range": { "Date": { "gte": "now-30m" } } }, { "wildcard": { UserName.raw": { "value": "*rry" } } } ] } } } }, "aggs": { "Hourly": { "date_histogram": { "field": "Date", "interval": "hour" }, "aggs": { "Group By City": { "terms": { "field": "City", "size": 0 } } } } } } --List all cities { "aggs": { "group_by_City": { "terms": { "field": "City.raw", "size": 0 } } } } --ELK query with multiple conditions: { "query": { "bool": { "must": { "bool": { "should": [ { "match": { "Category": "Test" } }, { "match": { "Category": "Test1" } } ] } }, "filter": { "terms": { "Id": [ "111", "222" ] } } } }, "fields": ["Id", "Category","@time"], } --Delete document by id DELETE /index/[_type]/doc Id

Plant UML using Visual Studio code

Scenario:

Create uml diagram declarative using Plant UML in Visual Studio code.

Solution:

You need to install.

1. Install:
1. Visual studio code -> Extension -> PlantUML
2. To create UML's other than sequenc diagram, download Graphviz (https://graphviz.gitlab.io/_pages/Download/Download_windows.html) and install.
3. Visual studio code > Preferences > Settings > Plantuml ->  Command Args -> Edit [settings.json].
4. In settings.json add "plantuml.commandArgs": [

           "-DGRAPHVIZ_DOT=..graphviz-2.38\\..\\bin\\dot.exe",
       ]

5. Close and reopen Visual studio code 

2. File -> New File -> Save as (with .wsd extension)


3. Add below to generate Class diagram for User class



@startuml UserRoles abstract class BaseUser class Shopper #Yellow class User #Yellow {     + CreateUser (name, hashPassword): id     + UpdateUser (id, userData) : id } class Role {    + CreateRole (name) id  } class UserRole #Yellow {      - User user      + AssociateRole (userId, roleId) void } UserRole .up.> Role UserRole .up.> User BaseUser <|-- User User <|-- Shopper @enduml



4. Ctrl + Shift + p -> PlantUML: Preview Current Diagram 
   
5. To save to file Ctrl + Shift + p -> Export Current File Diagrams