ELK Rollup data


Scenario:

Roll up data from Elk index at minute level and store in DB

Solution:

  1. The data is stored in ELK index "logs". The data to be rolled up to Category/Sub Category at minute level.

  2. Pull the data from ELK index for today and some day back and roll up at minute level using below query:
    3. 

      1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
    

    GET logs-*2020.01.01&expand_wildcards=all/_search
{
  "size": 0,
  "query": {
    "bool": {
      "must": [
        {
          "range": {
            "DateCreated": {
              "gte": "2020-01-01T15:05:16.6759102-07:00",
              "lt": "2020-01-01T15:08:51.0818714-07:00"
            }
          }
        }
      ]
    }
  },
  "aggs": {
    "Time": {
      "date_histogram": {
        "field": "DateCreated",
        "interval": "minute",
        "time_zone": "America/LA"
      },
      "aggs": {
        "CategoryId": {
          "terms": {
            "field": "CategoryId",
            "size": 500
          },
          "aggs": {
            "SubCategory": {
              "terms": {
                "field": "SubCategory.keyword",
                "size": 500
              },
              "aggs": {
                "SubCategory-Errors": {
                  "filter": {
                    "bool": {
                      "must": [
                        {
                          "term": {
                            "Error": {
                              "value": true
                            }
                          }
                        }
                      ]
                    }
                  }
                },
                "SubCategory-Avg": {
                  "avg": {
                    "field": "ResponseTime"
                  }
                },
                "SubCategory-Percstile": {
                  "percentiles": {
                    "field": "ResponseTime",
                    "percents": [
                      50,
                      75,
                      98
                    ]
                  }
                }
              }
            },
            "ProviderId-Errors": {
              "filter": {
                "exists": {
                  "field": "Error.keyword"
                }
              }
            },
            "ProviderId-Avg": {
              "avg": {
                "field": "ResponseTime"
              }
            },
            "ProviderId-Percstile": {
              "percentiles": {
                "field": "ResponseTime",
                "percents": [
                  50,
                  75,
                  98
                ]
              }
            }
          }
        },
        "Time-Errors": {
          "filter": {
            "bool": {
              "must": [
                {
                  "term": {
                    "Error": {
                      "value": true
                    }
                  }
                }
              ]
            }
          }
        },
        "Time-Avg": {
          "avg": {
            "field": "ResponseTime"
          }
        },
        "Time-Percstile": {
          "percentiles": {
            "field": "ResponseTime",
            "percents": [
              50,
              75,
              98
            ]
          }
        }
      }
    }
  }
}

  3. The response would look something like below:  
    

     
    

    

     1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
    

    {
  "took": 50,
  "timed_out": false,
  "_shards": {
    "total": 1,
    "successful": 1,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": {
      "value": 1,
      "relation": ""
    },
    "max_score": null,
    "hits": []
  },
  "aggregations": {
    "Time": {
      "buckets": [
        {
          "key_as_string": "2020-01-01T19:00:00.000-05:00",
          "key": 158682241111,
          "doc_count": 6,
          "CategoryId": {
            "doc_count_error_upper_bound": 0,
            "sum_other_doc_count": 0,
            "buckets": [
              {
                "key": 10001,
                "doc_count": 6,
                "CategoryId-Avg": { "value": 527.6666666666666 },
                "CategoryId-Percstile": {
                  "values": {
                    "50.0": 695.5,
                    "75.0": 763.0,
                    "98.0": 815.0
                  }
                },
                "CategoryId-Errors": { "doc_count": 0 },
                "SubCategory": {
                  "doc_count_error_upper_bound": 0,
                  "sum_other_doc_count": 0,
                  "buckets": [
                    {
                      "key": "10001.1",
                      "doc_count": 6,
                      "SubCategory-Percstile": {
                        "values": {
                          "50.0": 695.5,
                          "75.0": 763.0,
                          "98.0": 815.0
                        }
                      },
                      "SubCategory-Avg": { "value": 527.6666666666666 },
                      "SubCategory-Errors": { "doc_count": 0 }
                    }
                  ]
                }
              }
            ]
          },
          "Time-Percstile": {
            "values": {
              "50.0": 695.5,
              "75.0": 763.0,
              "98.0": 815.0
            }
          },
          "Time-Avg": { "value": 527.6666666666666 },
          "Time-Errors": { "doc_count": 0 }
        }
    }
  }

  4. Below is the code to parse the above response and get into a model: 
    

     
    

    

     1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
    

    public class RollupUser
{
    public void Process(HttpContext httpContext)
    {
        var model = CreateRequest(httpContext);

        using (var outputStream = requestDoc.CreateOutputStream())
        {
            var content = Encoding.Default.GetBytes(requestBody);
            outputStream.Write(content, 0, content.Length);
        }

        var request = new Httprequest
        {
            RequestUri = ElkUri,
            Method = HttpMethod.Post,
            Content = new StringContent(requestBody, Encoding.UTF8, "application/json")
        };

        var response = new HttpClient().SendAsync(request).Result;

        var response = response.Content.ReadAsStringAsync().Result;

        using (var outputStream = responseDoc.CreateOutputStream())
        {
            var content = Encoding.Default.GetBytes(response);
            outputStream.Write(content, 0, content.Length);
        }

        var output = Read(response);
    }

    private ResponseModel Read(string input)
    {
        using (var reader = new JsonTextReader(new StringReader(input)))
        {
            reader.DateParseHandling = DateParseHandling.None;
            r = JObject.Load(reader);
        }

        var buckets = (JArray)r["aggregations"]["Time"]["buckets"];

        foreach (var bucket in buckets)
        {
            var bucket1 = bucket;

            logs.Add(
                {
                Date = date,
                    CategoryId = -1,
                    SubCategory = "_ALL"
                    Calls = (int)bucket["doc_count"],
                    Errors = (int)bucket["Time-Errors"]["doc_count"],
                    AvgTimeElapsed = GetValue(() => (int)bucket1["Time-Avg"]["value"]),
                    P50TimeElapsed = GetValue(() => (int)bucket1["Time-Percstile"]["values"]["50.0"]),
                    P75TimeElapsed = GetValue(() => (int)bucket1["Time-Percstile"]["values"]["75.0"]),
                    P98TimeElapsed = GetValue(() => (int)bucket1["Time-Percstile"]["values"]["98.0"])
                });

            foreach (var categoryIdBucket in (JArray)bucket["CategoryId"]["buckets"])
            {
                var bucket2 = categoryIdBucket;
                logs.Add(
                    {
                    Date = date,
                        CategoryId = (int)categoryIdBucket["key"],
                        SubCategory = "_ALL",
                        Calls = (int)categoryIdBucket["doc_count"],
                        Errors = (int)categoryIdBucket["CategoryId-Errors"]["doc_count"],
                        AvgTimeElapsed = GetValue(() => (int)bucket2["CategoryId-Avg"]["value"]),
                        P50TimeElapsed = GetValue(() => (int)bucket2["CategoryId-Percstile"]["values"]["50.0"]),
                        P75TimeElapsed = GetValue(() => (int)bucket2["CategoryId-Percstile"]["values"]["75.0"]),
                        P98TimeElapsed = GetValue(() => (int)bucket2["CategoryId-Percstile"]["values"]["98.0"])
                    });

                foreach (var SubCategoryBucket in (JArray)categoryIdBucket["SubCategory"]["buckets"])
                {
                    var bucket3 = SubCategoryBucket;
                    logs.Add(
                        {
                        Date = date,
                            CategoryId = CategoryId,
                            SubCategory = (string)SubCategoryBucket["key"],
                            Calls = (int)SubCategoryBucket["doc_count"],
                            Errors = (int)SubCategoryBucket["SubCategory-Errors"]["doc_count"],
                            AvgTimeElapsed = GetValue(() => (int)bucket3["SubCategory-Avg"]["value"]),
                            P50TimeElapsed = GetValue(() =>
                                (int)bucket3["SubCategory-Percstile"]["values"]["50.0"]),
                            P75TimeElapsed = GetValue(() =>
                                (int)bucket3["SubCategory-Percstile"]["values"]["75.0"]),
                            P98TimeElapsed = GetValue(() =>
                                (int)bucket3["SubCategory-Percstile"]["values"]["98.0"])
                        });
                }
            }
        }
        return logs;
    }
}
at
Labels: , , , , , ,

No comments:

Post a Comment

Subscribe to: Post Comments (Atom)

Move Github Sub Repository back to main repo

 -- delete .gitmodules git rm --cached MyProject/Core git commit -m 'Remove myproject_core submodule' rm -rf MyProject/Core git remo...

  • Move Github Sub Repository back to main repo
     -- delete .gitmodules git rm --cached MyProject/Core git commit -m 'Remove myproject_core submodule' rm -rf MyProject/Core git remo...
  • Azure - CL & Commands
               Scenario:  Use Azure CLI to execute commands & perform operations. Solution: Per Azure docs: The Azure Command-Line Interface...
  • Web protocol evolution
      Scenario: The  end-to-end principle  is a network design method in which application-specific features are kept at communication end point...