API stats frontend revamp

api/server/build.gradle.kts

@@ -3,6 +3,7 @@ import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
 plugins {
     application
     alias(libs.plugins.buildlogic.kotlin.jvm)
+    alias(libs.plugins.kotlin.dataframe)
     alias(libs.plugins.kotlin.ksp)
     alias(libs.plugins.androidx.room)
     alias(libs.plugins.shadow)
@@ -14,6 +15,8 @@ dependencies {
     implementation(projects.util.fileDB)
     implementation(projects.util.logger)
 
+    implementation(libs.kotlin.reflect)
+    implementation(libs.kotlinx.dataframe)
     implementation(libs.decoroutinator)
 
     implementation(platform(libs.ktor.bom))
@@ -62,14 +65,25 @@ tasks.withType<ShadowJar> {
     minimize {
         exclude(dependency(libs.ktor.serialization.kotlinx.json.get()))
         exclude(dependency(libs.slf4j.simple.get()))
+        exclude(dependency(libs.decoroutinator.get()))
+        exclude(dependency(libs.kotlin.reflect.get()))
     }
     exclude(
-        "**/*.kotlin_*",
-        "**/*.pro",
+        "/*.css",
+        "/*.html",
+        "/*.js",
         "/*/default/linkdata/",
         "/*/default/manifest",
         "/DebugProbesKt.bin",
+        "/META-INF/**/*.kotlin_*",
+        "/META-INF/**/*.pro",
+        "/META-INF/**/*.version*",
+        "/META-INF/**/*LICENSE*",
+        "/META-INF/**/*NOTICE*",
+        "/META-INF/*jupyter*/",
+        "/META-INF/maven/",
         "/META-INF/native-image/",
-        "/META-INF/maven/"
+        "/META-INF/versions/",
+        "/org/apache/commons/codec/language/**/*.txt"
     )
 }

api/server/src/main/kotlin/ApiServer.kt

@@ -1,17 +1,13 @@
-@file:OptIn(ExperimentalSerializationApi::class, ExperimentalTime::class)
-
 package de.binarynoise.captiveportalautologin.server
 
 import java.nio.file.Path
 import kotlin.io.path.Path
 import kotlin.time.Clock
-import kotlin.time.ExperimentalTime
 import kotlin.time.Instant
 import kotlinx.coroutines.runBlocking
 import kotlinx.datetime.TimeZone
 import kotlinx.datetime.number
 import kotlinx.datetime.toLocalDateTime
-import kotlinx.serialization.ExperimentalSerializationApi
 import kotlinx.serialization.json.Json
 import de.binarynoise.captiveportalautologin.api.Api
 import de.binarynoise.captiveportalautologin.api.json.har.HAR
@@ -38,7 +34,7 @@ class ApiServer(root: Path = Path(".")) : Api {
         runBlocking {
             database.useConnection(isReadOnly = true) {}
         }
-        log("Database initialized")
+        log("Database initialized at root ${root.toAbsolutePath()}")
     }
 
     override val har: Api.Har = object : Api.Har {

api/server/src/main/kotlin/LoggingPlugin.kt

@@ -18,15 +18,20 @@ val LoggingPlugin: ApplicationPlugin<Unit> = createApplicationPlugin(name = "Log
             append(call.request.httpMethod.value)
             append(" ")
             append(call.request.origin.uri)
-            append(" with body ")
+            append(" with status code ")
+            append(call.response.status()?.value ?: 200)
+            append(" and body '")
             append(body.toString().substringBefore("\n").take(100))
+            append("'")
         })
     }
 
     on(CallFailed, handler = object : suspend (ApplicationCall, Throwable) -> Unit {
         override suspend fun invoke(call: ApplicationCall, cause: Throwable) {
-            if (cause is CancellationException) throw cause
-            Logger.log("call failed", cause)
+            when (cause) {
+                is CancellationException -> throw cause
+                else -> Logger.log("call failed", cause)
+            }
         }
     })
 

api/server/src/main/kotlin/Main.kt

@@ -1,19 +1,18 @@
-@file:OptIn(ExperimentalSerializationApi::class)
-
 package de.binarynoise.captiveportalautologin.server
 
 import kotlin.io.path.Path
 import kotlin.io.path.exists
 import kotlin.io.path.readText
+import kotlin.reflect.jvm.javaMethod
 import kotlinx.coroutines.CancellationException
-import kotlinx.serialization.ExperimentalSerializationApi
 import kotlinx.serialization.json.Json
 import com.github.mustachejava.DefaultMustacheFactory
 import com.github.mustachejava.FragmentKey
 import com.github.mustachejava.Mustache
 import de.binarynoise.captiveportalautologin.server.ApiServer.Companion.api
 import de.binarynoise.captiveportalautologin.server.routes.configureRouting
 import de.binarynoise.logger.Logger.log
+import dev.reformator.stacktracedecoroutinator.jvm.DecoroutinatorJvmApi
 import io.ktor.http.*
 import io.ktor.serialization.kotlinx.json.*
 import io.ktor.server.application.*
@@ -27,9 +26,15 @@ import io.ktor.server.response.*
 
 val hostname = Path("/proc/sys/kernel/hostname").takeIf { it.exists() }?.readText()?.trim()
 val isDevelopment = hostname != "captiveportalautologin"
+val isRunningFromJar = ::main.javaMethod!!.declaringClass.protectionDomain.codeSource.location.path.endsWith(".jar")
 
 fun main() {
-    val server = createServer("::", 8080)
+    if (isDevelopment) DecoroutinatorJvmApi.install()
+
+    val port = System.getenv("API_SERVER_PORT")?.toInt() ?: 8080
+    val host = System.getenv("API_SERVER_HOST") ?: "::"
+    log("launching server at $host:$port")
+    val server = createServer(host, port)
     server.start(wait = true)
 }
 
@@ -39,18 +44,18 @@ fun createServer(host: String, port: Int): EmbeddedServer<*, *> {
         factory = Netty,
         port = port,
         host = host,
-        watchPaths = listOf("classes", "resources"),
+        watchPaths = if (isRunningFromJar) emptyList() else listOf("classes", "resources"),
         module = Application::module,
     )
     with(server.engineConfig) {
         shutdownTimeout = 1000
         enableHttp2 = false
-        enableH2c = false
+//        enableH2c = false
     }
     return server
 }
 
-suspend fun Application.module() {
+/*suspend*/ fun Application.module() { // TODO: make this suspend again for ktor >=3.2.0
     api = ApiServer(Path(System.getenv("API_SERVER_PATH") ?: "."))
 
     check(developmentMode == isDevelopment) { "developmentMode != isDevelopment" }
@@ -83,11 +88,11 @@ suspend fun Application.module() {
         })
     }
     install(StatusPages) {
-        exception<CancellationException> { call, cause ->
-            throw cause
-        }
-        exception<IllegalArgumentException> { call, cause ->
-            call.respond(HttpStatusCode.BadRequest, cause.message ?: "Illegal Arguments")
+        exception<Throwable> { call, cause ->
+            when (cause) {
+                is CancellationException -> throw cause
+                else -> call.respondText(text = "500: $cause", status = HttpStatusCode.InternalServerError)
+            }
         }
         unhandled { call ->
             System.err.println("unhandled call: ${call.request.httpMethod.value} ${call.request.uri}")

api/server/src/main/kotlin/database/ErrorDao.kt

@@ -1,6 +1,5 @@
 package de.binarynoise.captiveportalautologin.server.database
 
-import kotlin.time.Instant
 import androidx.room.Dao
 import androidx.room.Insert
 import androidx.room.Query
@@ -10,11 +9,11 @@ interface ErrorDao {
     @Insert
     suspend fun insert(error: ErrorEntity)
 
-    @Query("SELECT * FROM errors ORDER BY timestamp DESC LIMIT :limit")
-    suspend fun getAllErrors(limit: Int): List<ErrorEntity>
+    @Query("SELECT * FROM errors ORDER BY timestamp DESC")
+    suspend fun getAll(): List<ErrorEntity>
 
-    @Query("SELECT * FROM errors WHERE message LIKE 'unknown portal' ORDER BY timestamp DESC LIMIT :limit")
-    suspend fun getUnknownPortalErrors(limit: Int): List<ErrorEntity>
+    @Query("SELECT * FROM errors WHERE message LIKE 'unknown portal' ORDER BY timestamp DESC")
+    suspend fun getUnknownPortals(): List<ErrorEntity>
 
     @Query(
         """
@@ -27,27 +26,9 @@ interface ErrorDao {
         AND message NOT LIKE 'Binding socket to network % failed: %'
         AND message NOT LIKE 'Chain validation failed'
         AND message NOT LIKE 'java.security.cert.CertPathValidatorException: %'
+        AND message NOT LIKE 'Socket is closed'
         ORDER BY timestamp DESC
-        LIMIT :limit
         """
     )
-    suspend fun getNoNoiseErrors(limit: Int): List<ErrorEntity>
-
-    @Query(
-        """
-        SELECT * FROM errors 
-        WHERE timestamp BETWEEN :start AND :end
-        AND version = :version
-        AND message = :message
-        AND url LIKE '%' || :domain || '%'
-        ORDER BY timestamp DESC
-        """
-    )
-    suspend fun getErrorDetails(
-        start: Instant,
-        end: Instant,
-        version: String,
-        message: String,
-        domain: String,
-    ): List<ErrorEntity>
+    suspend fun getNoNoise(): List<ErrorEntity>
 }

api/server/src/main/kotlin/database/ErrorEntity.kt

@@ -1,11 +1,16 @@
 package de.binarynoise.captiveportalautologin.server.database
 
 import kotlin.time.Instant
+import kotlinx.datetime.TimeZone.Companion.UTC
+import kotlinx.datetime.number
+import kotlinx.datetime.toLocalDateTime
 import androidx.room.Entity
 import androidx.room.PrimaryKey
+import org.jetbrains.kotlinx.dataframe.annotations.DataSchema
 
 @Entity(tableName = "errors")
-data class ErrorEntity(
+@DataSchema
+open class ErrorEntity(
     @PrimaryKey(autoGenerate = true) val id: Long = 0,
     val version: String,
     val timestamp: Instant,
@@ -14,4 +19,35 @@ data class ErrorEntity(
     val message: String,
     val solver: String,
     val stackTrace: String,
-)
+) {
+    fun toExtendedErrorEntity() = ExtendedErrorEntity(
+        id = id,
+        version = version,
+        timestamp = timestamp,
+        ssid = ssid,
+        url = url,
+        message = message,
+        solver = solver,
+        stackTrace = stackTrace,
+        domain = url.getUrlDomain(),
+        majorVersion = version.getMajorVersion(),
+        year = timestamp.toLocalDateTime(UTC).year,
+        month = timestamp.toLocalDateTime(UTC).month.number,
+    )
+}
+
+@DataSchema
+class ExtendedErrorEntity(
+    id: Long = 0,
+    version: String,
+    timestamp: Instant,
+    ssid: String,
+    url: String,
+    message: String,
+    solver: String,
+    stackTrace: String,
+    val domain: String,
+    val majorVersion: Int,
+    val year: Int,
+    val month: Int,
+) : ErrorEntity(id, version, timestamp, ssid, url, message, solver, stackTrace)

api/server/src/main/kotlin/database/SuccessDao.kt

@@ -20,17 +20,5 @@ interface SuccessDao {
     suspend fun insertOrIncrement(version: String, year: Int, month: Int, ssid: String, url: String, solver: String)
 
     @Query("SELECT * FROM successes")
-    suspend fun getAllSuccesses(): List<SuccessEntity>
-
-    @Query(
-        """
-        SELECT * FROM successes 
-        WHERE year = :year 
-        AND month = :month 
-        AND version LIKE '%' || :version || '%'
-        AND url LIKE '%' || :domain || '%'
-        ORDER BY count DESC, ssid ASC, url ASC
-    """
-    )
-    suspend fun getSuccessDetails(year: Int, month: Int, version: String, domain: String): List<SuccessEntity>
+    suspend fun getAll(): List<SuccessEntity>
 }

api/server/src/main/kotlin/database/SuccessEntity.kt

@@ -1,17 +1,44 @@
 package de.binarynoise.captiveportalautologin.server.database
 
 import androidx.room.Entity
+import org.jetbrains.kotlinx.dataframe.annotations.DataSchema
 
 @Entity(
     tableName = "successes",
     primaryKeys = ["version", "year", "month", "ssid", "url", "solver"],
 )
-data class SuccessEntity(
+@DataSchema
+open class SuccessEntity(
     val version: String,
     val year: Int,
     val month: Int,
     val ssid: String,
     val url: String,
     val solver: String,
     val count: Int,
-)
+) {
+    fun toExtendedSuccessEntity() = ExtendedSuccessEntity(
+        version,
+        year,
+        month,
+        ssid,
+        url,
+        solver,
+        count,
+        url.getUrlDomain(),
+        version.getMajorVersion(),
+    )
+}
+
+@DataSchema
+class ExtendedSuccessEntity(
+    version: String,
+    year: Int,
+    month: Int,
+    ssid: String,
+    url: String,
+    solver: String,
+    count: Int,
+    val domain: String,
+    val majorVersion: Int,
+) : SuccessEntity(version, year, month, ssid, url, solver, count)

api/server/src/main/kotlin/database/utils.kt

@@ -0,0 +1,8 @@
+package de.binarynoise.captiveportalautologin.server.database
+
+import io.ktor.http.URLBuilder
+
+
+fun String.getUrlDomain(): String = if (isNotEmpty()) URLBuilder(urlString = this).host else ""
+
+fun String.getMajorVersion(): Int = this.split('-', '+').first().toInt()