Skip to content

Sean/0731_make the Prometheus query handles missing entries automatically #80

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ProcrastinateGuy wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Sean/0731_make the Prometheus query handles missing entries automatically #80

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
40d4134
testing query for detection missing entries
ProcrastinateGuy Aug 6, 2025
b7f96b4
change the query to fill in the gaps when an entry is missing
ProcrastinateGuy Aug 10, 2025
264adeb
change the query to fill in the gaps when an entry is missing
ProcrastinateGuy Aug 10, 2025
ebcfdea
change the scraping timestamp back to normal
ProcrastinateGuy Aug 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 18 additions & 5 deletions system_status/server.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,6 @@ def get_args() -> argparse.Namespace:

args = get_args()


def get_prometheus_data() -> list[PrometheusData]:
"""Sends a PromQL query to Prometheus and returns the results."""
"""
Expand All @@ -89,9 +88,11 @@ def get_prometheus_data() -> list[PrometheusData]:
}
"""
url = urljoin(args.target, "api/v1/query_range")
#the below date has a rare data for testing purpose
#now = datetime.datetime(2025, 8, 6, 11, 38, 49)
now = datetime.datetime.now()
params = {
"query": 'min_over_time(up{job!=""}[1h])',
"query": '(min_over_time(up{job!=""}[1h]))',
"start": int((now - datetime.timedelta(hours=23)).timestamp()),
"end": int(now.timestamp()),
"step": "1h",
Expand All @@ -103,6 +104,12 @@ def get_prometheus_data() -> list[PrometheusData]:
response_json = response.json()
result_list = response_json.get("data", {}).get("result", [])

key_list = [] # make a list of expected timestamps
for i in range(24):
key_list.append(int(now.timestamp()) - (23 - i) * 3600)
key_list[i] = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(key_list[i]))
#print(key_list)

for service_dict in result_list:
maybe_instance = service_dict.get("metric", {}).get(
"instance", "NO INSTANCE AVAILABLE"
Expand All @@ -111,9 +118,15 @@ def get_prometheus_data() -> list[PrometheusData]:
maybe_values = service_dict.get("values", [])

timestamps_and_values = []
for epoch_time, value in maybe_values:
timestamp = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(epoch_time))
timestamps_and_values.append(TimestampAndValuePair(timestamp, value))
value_ptr = 0
for i in range(len(key_list)):
timestamp = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(maybe_values[value_ptr][0]))
if timestamp == key_list[i]:
timestamps_and_values.append(TimestampAndValuePair(key_list[i], maybe_values[value_ptr][1]))
value_ptr += 1
else:
#fill in the gaps
timestamps_and_values.append(TimestampAndValuePair(key_list[i], -1))

# the service is up if the maximum timestamp's value is "1"
# prometheus returns data with the greatest timestamp last
Expand Down
6 changes: 4 additions & 2 deletions system_status/templates/my_template.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,8 +124,10 @@ <h2 id="fetch_time">Fetch Time: {{ fetch_time }}</h2>
{% for value in item.values %}
{% if value.value == "1" %}
✅
{% else %}
{% elif value.value == "0" %}
❌
{% else %}
⚠️
{% endif %}
{# the below if-statement adds a vertical line for every four entries #}
{% if (not loop.index % 4) and loop.index < 24%}
Expand Down Expand Up @@ -165,7 +167,7 @@ <h2 id="fetch_time">Fetch Time: {{ fetch_time }}</h2>
// Format the string with leading zeroes
const dayStr = `${year.toString()}-${month.toString().padStart(2, '0')}-${date.toString().padStart(2, '0')}`
const clockStr = `${hours.toString().padStart(2, '0')}:${minutes.toString().padStart(2, '0')}:${seconds.toString().padStart(2, '0')}`;
timeElement.innerText = `${dayStr} ${clockStr} - PDT/PST`;
timeElement.innerText = `${dayStr} ${clockStr} - Local Time`;
}

// Call initially to display time immediately
Expand Down