Skip to content

Optimizes report queries and improves session pairing logic #172

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dkmstr merged 1 commit into from
May 8, 2026
Merged

Optimizes report queries and improves session pairing logic #172

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 26 additions & 27 deletions server/src/uds/reports/stats/pool_users_summary.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,14 @@
"""
Author: Adolfo Gómez, dkmaster at dkmon dot com
"""
import collections
import csv
import io
import logging
import typing

from django.db.models import F, Window
from django.db.models.functions import Lag
from django.utils.translation import gettext
from django.utils.translation import gettext_lazy as _

Expand Down Expand Up @@ -76,48 +79,44 @@ def get_pool_data(self, pool: 'ServicePool') -> tuple[list[dict[str, typing.Any]
end = self.end_date.as_timestamp()
logger.debug(self.pool.value)

login = events.types.stats.EventType.LOGIN
logout = events.types.stats.EventType.LOGOUT

# LOGIN/LOGOUT pairing pushed to DB via window LAG over fld4 (single pool, no need
# to partition by owner_id). Preserves "last login wins" semantics: only LOGOUT
# rows whose previous event for the same user is a LOGIN are accepted as a pair.
items = (
StatsManager.manager()
.enumerate_events(
events.types.stats.EventOwnerType.SERVICEPOOL,
(events.types.stats.EventType.LOGIN, events.types.stats.EventType.LOGOUT),
(login, logout),
owner_id=pool.id,
since=start,
to=end,
)
.order_by('stamp')
.annotate(
prev_type=Window(Lag('event_type'), partition_by=[F('fld4')], order_by=[F('stamp')]),
prev_stamp=Window(Lag('stamp'), partition_by=[F('fld4')], order_by=[F('stamp')]),
)
.values('event_type', 'stamp', 'fld4', 'prev_type', 'prev_stamp')
)

logins: dict[str, int] = {}
users: dict[str, dict[str, typing.Any]] = {}
users: dict[str, dict[str, int]] = collections.defaultdict(
lambda: {'sessions': 0, 'time': 0}
)
for i in items:
# if '\\' in i.fld1:
# continue
username = i.fld4
if i.event_type == events.types.stats.EventType.LOGIN:
logins[username] = i.stamp
else:
if username in logins:
stamp = logins[username]
del logins[username]
total = i.stamp - stamp
if username not in users:
users[username] = {'sessions': 0, 'time': 0}
users[username]['sessions'] += 1
users[username]['time'] += total
# data.append({
# 'name': i.fld4,
# 'date': datetime.datetime.fromtimestamp(stamp),
# 'time': total
# })

# Extract different number of users
if i['event_type'] != logout or i['prev_type'] != login:
continue
entry = users[i['fld4']]
entry['sessions'] += 1
entry['time'] += i['stamp'] - i['prev_stamp']

data = [
{
'user': k,
'sessions': v['sessions'],
'hours': '{:.2f}'.format(float(v['time']) / 3600),
'average': '{:.2f}'.format(float(v['time']) / 3600 / v['sessions']),
'hours': '{:.2f}'.format(v['time'] / 3600),
'average': '{:.2f}'.format(v['time'] / 3600 / v['sessions']),
}
for k, v in users.items()
]
Expand Down
111 changes: 60 additions & 51 deletions server/src/uds/reports/stats/pools_performance.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,9 @@
import io
import logging
import typing
import collections.abc

import django.template.defaultfilters as filters
from django.db.models import Count
from django.utils import timezone
from django.utils.translation import gettext
from django.utils.translation import gettext_lazy as _

Expand Down Expand Up @@ -77,14 +76,12 @@ def init_gui(self) -> None:
]
self.pools.set_choices(vals)

def list_pools(self) -> collections.abc.Iterable[tuple[int, str]]:
def list_pools(self) -> list[tuple[int, str]]:
if '0-0-0-0' in self.pools.value:
pools = ServicePool.objects.all()
qs = ServicePool.objects.all()
else:
pools = ServicePool.objects.filter(uuid__in=self.pools.value)

for p in pools:
yield (p.id, p.name)
qs = ServicePool.objects.filter(uuid__in=self.pools.value)
return list(qs.values_list('id', 'name'))

def get_range_data(
self,
Expand All @@ -93,69 +90,81 @@ def get_range_data(
]: # pylint: disable=too-many-locals
start = self.start_date.as_timestamp()
end = self.end_date.as_timestamp()
if self.sampling_points.as_int() < 2:
self.sampling_points.value = 2
if self.sampling_points.as_int() > 128:
self.sampling_points.value = 128

sampling_points = self.sampling_points.as_int()
sampling_points = max(2, min(128, self.sampling_points.as_int()))

# x axis label format
if end - start > 3600 * 24 * 2:
x_label_format = 'SHORT_DATE_FORMAT'
else:
x_label_format = 'SHORT_DATETIME_FORMAT'

sampling_intervals: list[tuple[int, int]] = []
sampling_interval_seconds = (end - start) / sampling_points
# Precompute per-bucket (start, end, midpoint) once.
bucket_bounds: list[tuple[int, int, int]] = []
for i in range(sampling_points):
sampling_intervals.append(
(int(start + i * sampling_interval_seconds), int(start + (i + 1) * sampling_interval_seconds))
)

# Store dataUsers for all pools
pools_data: list[dict[str, typing.Any]] = []
b_start = int(start + i * sampling_interval_seconds)
b_end = int(start + (i + 1) * sampling_interval_seconds)
bucket_bounds.append((b_start, b_end, (b_start + b_end) // 2))

fld = StatsManager.manager().get_event_field_for('username')
pools = self.list_pools()
last_idx = sampling_points - 1

# Single query covering all selected pools. Bucketize in Python keyed by owner_id.
rows = (
StatsManager.manager()
.enumerate_events(
events.types.stats.EventOwnerType.SERVICEPOOL,
events.types.stats.EventType.ACCESS,
since=start,
to=end,
owner_id=[p[0] for p in pools],
)
.values('owner_id', 'stamp', fld)
)

distinct_users: dict[int, list[set[str]]] = {
p[0]: [set() for _ in range(sampling_points)] for p in pools
}
accesses_count: dict[int, list[int]] = {
p[0]: [0] * sampling_points for p in pools
}
for row in rows:
idx = int((row['stamp'] - start) // sampling_interval_seconds)
if idx < 0:
continue
if idx > last_idx:
idx = last_idx
owner_id = row['owner_id']
distinct_users[owner_id][idx].add(row[fld])
accesses_count[owner_id][idx] += 1

pools_data: list[dict[str, typing.Any]] = []
report_data: list[dict[str, typing.Any]] = []
for p in self.list_pools():
for pool_id, pool_name in pools:
users_buckets = distinct_users[pool_id]
access_buckets = accesses_count[pool_id]
data_users: list[tuple[int, int]] = []
data_accesses: list[tuple[int, int]] = []
for interval in sampling_intervals:
key = (interval[0] + interval[1]) // 2
q = (
StatsManager.manager()
.enumerate_events(
events.types.stats.EventOwnerType.SERVICEPOOL,
events.types.stats.EventType.ACCESS,
since=interval[0],
to=interval[1],
owner_id=p[0],
)
.values(fld)
.annotate(cnt=Count(fld))
)
accesses = 0
for v in q:
accesses += v['cnt']

data_users.append((key, len(q))) # Store number of users
for i, (b_start, b_end, key) in enumerate(bucket_bounds):
users_n = len(users_buckets[i])
accesses = access_buckets[i]
data_users.append((key, users_n))
data_accesses.append((key, accesses))
report_data.append(
{
'name': p[1],
'date': utils.timestamp_as_str(interval[0], 'SHORT_DATETIME_FORMAT')
'name': pool_name,
'date': utils.timestamp_as_str(b_start, 'SHORT_DATETIME_FORMAT')
+ ' - '
+ utils.timestamp_as_str(interval[1], 'SHORT_DATETIME_FORMAT'),
'users': len(q),
+ utils.timestamp_as_str(b_end, 'SHORT_DATETIME_FORMAT'),
'users': users_n,
'accesses': accesses,
}
)
pools_data.append(
{
'pool': p[0],
'name': p[1],
'pool': pool_id,
'name': pool_name,
'dataUsers': data_users,
'dataAccesses': data_accesses,
}
Expand All @@ -178,7 +187,7 @@ def generate(self) -> bytes:
# l is the index of the x value
# returns the date in the x value to be used as label on the x axis
def _tick_fnc1(l: int) -> str:
return filters.date(datetime.datetime.fromtimestamp(x[l]), x_label_format) if int(x[l]) >= 0 else ''
return filters.date(timezone.make_aware(datetime.datetime.fromtimestamp(x[l])), x_label_format) if int(x[l]) >= 0 else ''

data = {
'title': _('Distinct Users'),
Expand All @@ -194,7 +203,7 @@ def _tick_fnc1(l: int) -> str:
x = [v[0] for v in pools_data[0]['dataAccesses']]

def _tick_fnc2(l: int) -> str:
return filters.date(datetime.datetime.fromtimestamp(x[l]), x_label_format) if int(x[l]) >= 0 else ''
return filters.date(timezone.make_aware(datetime.datetime.fromtimestamp(x[l])), x_label_format) if int(x[l]) >= 0 else ''

data = {
'title': _('Accesses'),
Expand All @@ -213,10 +222,10 @@ def _tick_fnc2(l: int) -> str:
'uds/reports/stats/pools-performance.html',
dct={
'data': report_data,
'pools': [i[1] for i in self.list_pools()],
'pools': [p['name'] for p in pools_data],
'beginning': self.start_date.as_date(),
'ending': self.end_date.as_date(),
'intervals': self.sampling_points.as_int(),
'intervals': max(2, min(128, self.sampling_points.as_int())),
},
header=gettext('UDS Pools Performance Report'),
water=gettext('Pools Performance'),
Expand Down
89 changes: 46 additions & 43 deletions server/src/uds/reports/stats/usage_by_pool.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,10 @@
import io
import logging
import typing
from collections import defaultdict

from django.db.models import F, Window
from django.db.models.functions import Lag
from django.utils import timezone
from django.utils.translation import gettext
from django.utils.translation import gettext_lazy as _

Expand Down Expand Up @@ -70,64 +72,65 @@ def init_gui(self) -> None:
self.pool.set_choices(vals)

def get_data(self) -> tuple[list[dict[str, typing.Any]], str]:
# Generate the sampling intervals and get dataUsers from db
start = self.start_date.as_timestamp()
end = self.end_date.as_timestamp()
logger.debug(self.pool.value)
if '0-0-0-0' in self.pool.value:
pools = ServicePool.objects.all()
qs = ServicePool.objects.all()
else:
pools = ServicePool.objects.filter(uuid__in=self.pool.value)

# Build pool id -> pool map for fast lookup
pool_map: dict[int, ServicePool] = {p.id: p for p in pools}

# Single query for ALL pools (eliminates N+1) + values() avoids model instantiation overhead
qs = ServicePool.objects.filter(uuid__in=self.pool.value)

# (uuid, name) per pool id. values_list avoids instantiating ServicePool
# rows just to read 2 fields.
pool_map: dict[int, tuple[str, str]] = {
p_id: (p_uuid, p_name)
for p_id, p_uuid, p_name in qs.values_list('id', 'uuid', 'name')
}

login = stats.events.types.stats.EventType.LOGIN
logout = stats.events.types.stats.EventType.LOGOUT

# LOGIN/LOGOUT pairing pushed to DB via window LAG over (owner_id, fld4) ordered by stamp.
# Preserves "last login wins" semantics: if prev event of same (pool, user) is LOGIN -> pair.
# Portable across MySQL 8+, PostgreSQL, SQLite 3.25+, Oracle (ANSI window functions).
partition = [F('owner_id'), F('fld4')]
items = (
StatsManager.manager()
.enumerate_events(
stats.events.types.stats.EventOwnerType.SERVICEPOOL,
(stats.events.types.stats.EventType.LOGIN, stats.events.types.stats.EventType.LOGOUT),
(login, logout),
owner_id=list(pool_map.keys()),
since=start,
to=end,
)
.order_by('stamp')
.values('owner_id', 'event_type', 'stamp', 'fld2', 'fld4')
.annotate(
prev_type=Window(Lag('event_type'), partition_by=partition, order_by=[F('stamp')]),
prev_stamp=Window(Lag('stamp'), partition_by=partition, order_by=[F('stamp')]),
)
.values('owner_id', 'event_type', 'stamp', 'fld2', 'fld4', 'prev_type', 'prev_stamp')
)

# Group events by pool_id for correct LOGIN/LOGOUT pairing per pool
pool_events: dict[int, list[dict[str, typing.Any]]] = defaultdict(list)
for item in items:
pool_events[item['owner_id']].append(item)

data: list[dict[str, typing.Any]] = []
for pool_id, evts in pool_events.items():
pool = pool_map[pool_id]
logins: dict[str, int] = {}
for i in evts:
username = i['fld4'] # full_username
if i['event_type'] == stats.events.types.stats.EventType.LOGIN:
logins[username] = i['stamp']
else:
stamp = logins.pop(username, None) # pop avoids double lookup + del
if stamp is not None:
total = i['stamp'] - stamp
# src_ip logic: if IPv6 (contains '[') return as-is, else split port
fld2 = i['fld2']
origin = fld2 if '[' in fld2 else fld2.split(':')[0]
data.append(
{
'name': username,
'origin': origin,
'date': datetime.datetime.fromtimestamp(stamp),
'time': total,
'pool': pool.uuid,
'pool_name': pool.name,
}
)

return data, ','.join([p.name for p in pools])
for i in items:
if i['event_type'] != logout or i['prev_type'] != login:
continue
pool_uuid, pool_name = pool_map[i['owner_id']]
login_stamp = i['prev_stamp']
fld2 = i['fld2']
# ipv6 handled inline (was StatsEvents.src_ip property; we use .values()).
origin = fld2 if '[' in fld2 else fld2.split(':')[0]
data.append(
{
'name': i['fld4'],
'origin': origin,
'date': timezone.make_aware(datetime.datetime.fromtimestamp(login_stamp)),
'time': i['stamp'] - login_stamp,
'pool': pool_uuid,
'pool_name': pool_name,
}
)

return data, ','.join(name for _uuid, name in pool_map.values())

def generate(self) -> bytes:
items, poolname = self.get_data()
Expand Down
Loading
Loading