Refactor and restructure policy and store generators

.github/workflows/release-sqlite-db.yml

@@ -35,19 +35,12 @@ jobs:
 
       - name: Build project and export SQLite DBs
         run: |
-          # Build and export policies database
-          ./gradlew :generator:build --quiet
-          ./gradlew :generator:jvmRun \
-            -DmainClass=io.github.kdroidfilter.database.generator.SqlLiteExtractorKt \
-            --quiet
-          ls -l generator/build/policies.db
+          # Build and run both generators using the new task
+          ./gradlew runGenerators --quiet
 
-          # Build and export store database
-          ./gradlew :storegenerator:build --quiet
-          ./gradlew :storegenerator:jvmRun \
-            -DmainClass=io.github.kdroidfilter.database.storegenerator.SqliteStoreBuilderKt \
-            --quiet
-          ls -l storegenerator/build/store-database.db
+          # Verify the generated database files
+          ls -l generators/policies/build/policies-database.db
+          ls -l generators/store/build/store-database.db
 
       - name: Define release name
         id: relname
@@ -68,5 +61,5 @@ jobs:
           tag_name: ${{ env.RELEASE_NAME }}
           name: ${{ env.RELEASE_NAME }}
           files: |
-            generator/build/policies.db
-            storegenerator/build/store-database.db
+            generators/policies/build/policies-database.db
+            generators/store/build/store-database.db

build.gradle.kts

@@ -5,3 +5,11 @@ plugins {
     alias(libs.plugins.android.application).apply(false)
     alias(libs.plugins.vannitktech.maven.publish).apply(false)
 }
+
+// Task to run both generators
+tasks.register("runGenerators") {
+    group = "extraction"
+    description = "Runs both the SQLite policy extractor and the SQLite store extractor"
+
+    dependsOn(":generators:policies:runPolicyExtractor", ":generators:store:runStoreExtractor")
+}

generator/build.gradle.kts → generators/policies/build.gradle.kts

@@ -10,13 +10,16 @@ kotlin {
     jvmToolchain(17)
 
     jvm()
+
+
     sourceSets {
         commonMain.dependencies {
             implementation(project(":core"))
             implementation(libs.kotlinx.coroutines.core)
             implementation(libs.kotlinx.coroutines.test)
             implementation(libs.kotlinx.serialization.json)
             implementation(libs.kermit)
+            implementation(libs.platform.tools.release.fetcher)
         }
 
         commonTest.dependencies {
@@ -42,3 +45,16 @@ kotlin {
     }
 
 }
+
+tasks.register<JavaExec>("runPolicyExtractor") {
+    group = "extraction"
+    description = "Runs the SQLite policy extractor"
+
+    dependsOn(kotlin.jvm().compilations["main"].compileTaskProvider)
+    classpath = files(
+        kotlin.jvm().compilations["main"].output.allOutputs,
+        kotlin.jvm().compilations["main"].runtimeDependencyFiles
+    )
+    mainClass.set("SqlitePolicyExtractorKt")
+
+}

generator/src/commonMain/kotlin/io/github/kdroidfilter/database/generator/JsonExtractor.kt → generators/policies/src/jvmMain/kotlin/JsonExtractor.kt

@@ -1,9 +1,8 @@
-import io.github.kdroidfilter.database.generator.PolicyRepository
 import java.nio.file.Paths
 
 fun main() {
     val projectDir = Paths.get("").toAbsolutePath()
-    val root       = projectDir.parent.resolve("app-policies")
+    val root       = projectDir.parent.resolve("../app-policies")
     val output = Paths.get("build","all-policies.json")
     PolicyRepository.exportAll(root, output)
 }

generator/src/commonMain/kotlin/io/github/kdroidfilter/database/generator/PolicyRepository.kt → generators/policies/src/jvmMain/kotlin/PolicyRepository.kt

@@ -1,5 +1,3 @@
-package io.github.kdroidfilter.database.generator
-
 import io.github.kdroidfilter.database.core.ModeSpec
 import io.github.kdroidfilter.database.core.policies.AppPolicy
 import io.github.kdroidfilter.database.core.policies.FixedPolicy

generators/policies/src/jvmMain/kotlin/SqlitePolicyBuilder.kt

@@ -0,0 +1,98 @@
+import co.touchlab.kermit.Logger
+import io.github.kdroidfilter.database.core.policies.AppPolicy
+import kotlinx.serialization.json.Json
+import java.nio.file.Files
+import java.nio.file.Path
+import java.sql.Connection
+import java.sql.DriverManager
+import java.time.LocalDateTime
+import java.time.format.DateTimeFormatter
+
+object SqlitePolicyBuilder {
+    private val logger = Logger.withTag("SqlitePolicyBuilder")
+    private val json = Json {
+        classDiscriminator = "type"
+        ignoreUnknownKeys = true
+        encodeDefaults = false
+        prettyPrint = false
+        serializersModule = PolicyRepository.json.serializersModule
+    }
+
+    fun buildDatabase(policiesDir: Path, outputDbPath: Path) {
+        // Always create the database from scratch
+        logger.i { "🔄 Creating database from scratch..." }
+
+        // Delete existing database file if it exists
+        if (Files.exists(outputDbPath)) {
+            logger.i { "🗑️ Deleting existing database file..." }
+            Files.delete(outputDbPath)
+        }
+
+        // Get release name from environment variable or generate timestamp
+        val releaseName = System.getenv("RELEASE_NAME") ?: LocalDateTime.now()
+            .format(DateTimeFormatter.ofPattern("yyyyMMddHHmm"))
+        logger.i { "🏷️ Using release name: $releaseName" }
+
+        Class.forName("org.sqlite.JDBC")
+        Files.createDirectories(outputDbPath.parent)
+        val url = "jdbc:sqlite:${outputDbPath.toAbsolutePath()}"
+        DriverManager.getConnection(url).use { conn ->
+            conn.autoCommit = false
+            createTables(conn)
+            insertVersion(conn, releaseName)
+            insertPolicies(conn, policiesDir)
+            conn.commit()
+        }
+        logger.i { "✅ SQLite database created at $outputDbPath" }
+    }
+
+    private fun createTables(conn: Connection) = with(conn.createStatement()) {
+        executeUpdate("""
+            CREATE TABLE IF NOT EXISTS policies (
+              package_name TEXT PRIMARY KEY,
+              data         TEXT NOT NULL
+            )
+        """.trimIndent())
+
+        executeUpdate("""
+            CREATE TABLE IF NOT EXISTS version (
+              id INTEGER PRIMARY KEY AUTOINCREMENT,
+              release_name TEXT NOT NULL
+            )
+        """.trimIndent())
+        close()
+    }
+
+    private fun insertVersion(conn: Connection, releaseName: String) {
+        // Clear existing entries
+        conn.createStatement().use { stmt ->
+            stmt.executeUpdate("DELETE FROM version")
+        }
+
+        // Insert new release name
+        conn.prepareStatement("INSERT INTO version (release_name) VALUES (?)").use { ps ->
+            ps.setString(1, releaseName)
+            ps.executeUpdate()
+        }
+        logger.i { "✅ Inserted version info: $releaseName" }
+    }
+
+    private fun insertPolicies(conn: Connection, policiesDir: Path) {
+        val insertSql = """
+            INSERT OR REPLACE INTO policies(package_name, data) 
+            VALUES(?, ?)
+        """.trimIndent()
+
+        conn.prepareStatement(insertSql).use { ps ->
+            val policies = PolicyRepository.loadAll(policiesDir)
+            policies.forEach { policy ->
+                val jsonStr = json.encodeToString(AppPolicy.serializer(), policy)
+                ps.setString(1, policy.packageName)
+                ps.setString(2, jsonStr)
+                ps.addBatch()
+            }
+            ps.executeBatch()
+            logger.i { "✅ Inserted ${policies.size} policies" }
+        }
+    }
+}

generators/policies/src/jvmMain/kotlin/SqlitePolicyExtractor.kt

@@ -0,0 +1,9 @@
+import java.nio.file.Path
+
+fun main() {
+    val projectDir = Path.of("").toAbsolutePath()
+    val policiesDir = projectDir.parent.resolve("../app-policies")
+    val outputDb = projectDir.resolve("build/policies-database.db")
+
+    SqlitePolicyBuilder.buildDatabase(policiesDir, outputDb)
+}

generator/src/commonTest/kotlin/io/github/kdroidfilter/database/generator/JsonExtractorTest.kt → generators/policies/src/jvmTest/kotlin/JsonExtractorTest.kt

@@ -1,27 +1,25 @@
-package io.github.kdroidfilter.database.generator
-
 import io.github.kdroidfilter.database.core.AppCategory
 import io.github.kdroidfilter.database.core.ModeSpec
-import io.github.kdroidfilter.database.core.UserMode
 import io.github.kdroidfilter.database.core.NetworkMode
 import io.github.kdroidfilter.database.core.NetworkPolicy
+import io.github.kdroidfilter.database.core.UserMode
 import io.github.kdroidfilter.database.core.policies.AppPolicy
 import io.github.kdroidfilter.database.core.policies.FixedPolicy
 import io.github.kdroidfilter.database.core.policies.ModeBasedPolicy
 import kotlinx.serialization.builtins.ListSerializer
+import java.nio.file.Files
+import java.nio.file.Path
 import kotlin.test.Test
 import kotlin.test.assertEquals
 import kotlin.test.assertTrue
-import java.nio.file.Files
-import java.nio.file.Path
 
 class JsonExtractorTest {
 
     @Test
     fun `exportAll should serialize all existing policies without error`() {
         // Arrange: point to the real app-policies folder in the project
         val projectDir = Path.of("").toAbsolutePath()
-        val root = projectDir.parent.resolve("app-policies")
+        val root = projectDir.parent.resolve("../app-policies")
         val output = Files.createTempFile("all-policies", ".json")
 
         // Act: generate the single JSON array
@@ -75,4 +73,4 @@ class JsonExtractorTest {
         val names = policies.map { it.packageName }.toSet()
         assertTrue(names.containsAll(setOf("com.example.foo", "com.example.bar")), "Both package names must be present")
     }
-}
+}

storegenerator/build.gradle.kts → generators/store/build.gradle.kts

@@ -47,3 +47,16 @@ kotlin {
     }
 
 }
+
+tasks.register<JavaExec>("runStoreExtractor") {
+    group = "extraction"
+    description = "Runs the SQLite store extractor"
+
+    dependsOn(kotlin.jvm().compilations["main"].compileTaskProvider)
+    classpath = files(
+        kotlin.jvm().compilations["main"].output.allOutputs,
+        kotlin.jvm().compilations["main"].runtimeDependencyFiles
+    )
+    mainClass.set("SqliteStoreExtractorKt")
+
+}

storegenerator/src/commonMain/kotlin/io/github/kdroidfilter/database/storegenerator/SqliteStoreBuilder.kt → generators/store/src/jvmMain/kotlin/SqliteStoreBuilder.kt

@@ -1,5 +1,3 @@
-package io.github.kdroidfilter.database.storegenerator
-
 import co.touchlab.kermit.Logger
 import com.kdroid.gplayscrapper.services.getGooglePlayApplicationInfo
 import io.github.kdroidfilter.database.core.AppCategory
@@ -224,10 +222,3 @@ object SqliteStoreBuilder {
         }
     }
 }
-
-fun main() {
-    val root = Path.of("").toAbsolutePath()
-    val policies = root.resolve("../app-policies")
-    val output = root.resolve("build/store-database.db")
-    SqliteStoreBuilder.buildDatabase(policies, output)
-}

generators/store/src/jvmMain/kotlin/SqliteStoreExtractor.kt

@@ -0,0 +1,9 @@
+import java.nio.file.Path
+
+fun main() {
+    val projectDir = Path.of("").toAbsolutePath()
+    val policiesDir = projectDir.resolve("../../app-policies")
+    val outputDb = projectDir.resolve("build/store-database.db")
+
+    SqliteStoreBuilder.buildDatabase(policiesDir, outputDb)
+}

gradle.properties

@@ -12,3 +12,5 @@ kotlin.daemon.jvmargs=-Xmx4G
 #Android
 android.useAndroidX=true
 android.nonTransitiveRClass=true
+
+kotlin.native.ignoreDisabledTargets=true