Refactoring the npm installation process for JavaScript (#1891)

* Refactor npm packages checking & installing

* Enhanced package installation for JavaScript

* Add JSON file consistency check

* Add message about time expiration for npm packages installation

* Add packageJson in ServiceContext

* Text corrections

Author: rudolf101

utbot-intellij-js/src/main/kotlin/org/utbot/intellij/plugin/language/js/JsDialogProcessor.kt

@@ -23,16 +23,15 @@ import org.jetbrains.kotlin.idea.util.application.invokeLater
 import org.jetbrains.kotlin.idea.util.application.runReadAction
 import org.jetbrains.kotlin.idea.util.application.runWriteAction
 import org.jetbrains.kotlin.konan.file.File
+import org.utbot.framework.plugin.api.TimeoutException
 import org.utbot.intellij.plugin.ui.utils.showErrorDialogLater
 import org.utbot.intellij.plugin.ui.utils.testModules
 import settings.JsDynamicSettings
 import settings.JsExportsSettings.endComment
 import settings.JsExportsSettings.startComment
-import settings.JsPackagesSettings.mochaData
-import settings.JsPackagesSettings.nycData
-import settings.JsPackagesSettings.ternData
 import settings.JsTestGenerationSettings.dummyClassName
-import settings.PackageData
+import settings.PackageDataService
+import settings.jsPackagesList
 import utils.JsCmdExec
 import utils.OsProvider
 import java.io.IOException
@@ -58,9 +57,10 @@ object JsDialogProcessor {
         ) {
             override fun run(indicator: ProgressIndicator) {
                 invokeLater {
-                    checkAndInstallRequirement(model.project, model.pathToNPM, mochaData)
-                    checkAndInstallRequirement(model.project, model.pathToNPM, nycData)
-                    checkAndInstallRequirement(model.project, model.pathToNPM, ternData)
+                    if (!PackageDataService(
+                            model.containingFilePath, model.project.basePath!!, model.pathToNPM
+                        ).checkAndInstallRequirements(project)
+                    ) return@invokeLater
                     createDialog(model)?.let { dialogWindow ->
                         if (!dialogWindow.showAndGet()) return@invokeLater
                         // Since Tern.js accesses containing file, sync with file system required before test generation.
@@ -76,36 +76,31 @@ object JsDialogProcessor {
         }).queue()
     }
 
-    private fun findNodeAndNPM(): Pair<String, String>? =
-        try {
-            val pathToNode = NodeJsLocalInterpreterManager.getInstance()
-                .interpreters.first().interpreterSystemIndependentPath
-            val (_, errorText) = JsCmdExec.runCommand(
-                shouldWait = true,
-                cmd = arrayOf("\"${pathToNode}\"", "-v")
-            )
-            if (errorText.isNotEmpty()) throw NoSuchElementException()
-            val pathToNPM =
-                pathToNode.substringBeforeLast("/") + "/" + "npm" + OsProvider.getProviderByOs().npmPackagePostfix
-            pathToNode to pathToNPM
-        } catch (e: NoSuchElementException) {
-            Messages.showErrorDialog(
-                "Node.js interpreter is not found in IDEA settings.\n" +
-                        "Please set it in Settings > Languages & Frameworks > Node.js",
-                "Requirement Error"
-            )
-            logger.error { "Node.js interpreter was not found in IDEA settings." }
-            null
-        } catch (e: IOException) {
-            Messages.showErrorDialog(
-                "Node.js interpreter path is corrupted in IDEA settings.\n" +
-                        "Please check Settings > Languages & Frameworks > Node.js",
-                "Requirement Error"
-            )
-            logger.error { "Node.js interpreter path is corrupted in IDEA settings." }
-            null
-        }
-
+    private fun findNodeAndNPM(): Pair<String, String>? = try {
+        val pathToNode =
+            NodeJsLocalInterpreterManager.getInstance().interpreters.first().interpreterSystemIndependentPath
+        val (_, errorText) = JsCmdExec.runCommand(
+            shouldWait = true, cmd = arrayOf("\"${pathToNode}\"", "-v")
+        )
+        if (errorText.isNotEmpty()) throw NoSuchElementException()
+        val pathToNPM =
+            pathToNode.substringBeforeLast("/") + "/" + "npm" + OsProvider.getProviderByOs().npmPackagePostfix
+        pathToNode to pathToNPM
+    } catch (e: NoSuchElementException) {
+        Messages.showErrorDialog(
+            "Node.js interpreter is not found in IDEA settings.\n" + "Please set it in Settings > Languages & Frameworks > Node.js",
+            "Requirement Error"
+        )
+        logger.error { "Node.js interpreter was not found in IDEA settings." }
+        null
+    } catch (e: IOException) {
+        Messages.showErrorDialog(
+            "Node.js interpreter path is corrupted in IDEA settings.\n" + "Please check Settings > Languages & Frameworks > Node.js",
+            "Requirement Error"
+        )
+        logger.error { "Node.js interpreter path is corrupted in IDEA settings." }
+        null
+    }
 
     private fun createJsTestModel(
         project: Project,
@@ -138,7 +133,6 @@ object JsDialogProcessor {
             this.pathToNode = pathToNode
             this.pathToNPM = pathToNPM
         }
-
     }
 
     private fun createDialog(jsTestsModel: JsTestsModel?) = jsTestsModel?.let { JsDialogWindow(it) }
@@ -159,10 +153,8 @@ object JsDialogProcessor {
                 val testDir = PsiDirectoryFactory.getInstance(project).createDirectory(
                     model.testSourceRoot!!
                 )
-                val testFileName = normalizedContainingFilePath.substringAfterLast("/")
-                    .replace(Regex(".js"), "Test.js")
-                val testGenerator = JsTestGenerator(
-                    fileText = editor.document.text,
+                val testFileName = normalizedContainingFilePath.substringAfterLast("/").replace(Regex(".js"), "Test.js")
+                val testGenerator = JsTestGenerator(fileText = editor.document.text,
                     sourceFilePath = normalizedContainingFilePath,
                     projectPath = model.project.basePath?.replace(File.separator, "/")
                         ?: throw IllegalStateException("Can't access project path."),
@@ -267,45 +259,47 @@ object JsDialogProcessor {
     }
 }
 
-fun checkAndInstallRequirement(
-    project: Project,
-    pathToNPM: String,
-    requirement: PackageData,
-) {
-    if (!requirement.findPackageByNpm(project.basePath!!, pathToNPM)) {
-        installMissingRequirement(project, pathToNPM, requirement)
-    }
-}
-
-private fun installMissingRequirement(
-    project: Project,
-    pathToNPM: String,
-    requirement: PackageData,
-) {
+private fun PackageDataService.checkAndInstallRequirements(project: Project): Boolean {
+    val missingPackages = jsPackagesList.filterNot { this.findPackage(it) }
+    if (missingPackages.isEmpty()) return true
     val message = """
-            Requirement is not installed:
-            ${requirement.packageName}
-            Install it?
+            Requirements are not installed:
+            ${missingPackages.joinToString { it.packageName }}
+            Install them?
         """.trimIndent()
     val result = Messages.showOkCancelDialog(
-        project,
-        message,
-        "Requirement Missmatch Error",
-        "Install",
-        "Cancel",
-        null
+        project, message, "Requirements Missmatch Error", "Install", "Cancel", null
     )
 
     if (result == Messages.CANCEL)
-        return
+        return false
 
-    val (_, errorText) = requirement.installPackage(project.basePath!!, pathToNPM)
-
-    if (errorText.isNotEmpty()) {
+    try {
+        val (_, errorText) = this.installMissingPackages(missingPackages)
+        if (errorText.isNotEmpty()) {
+            showErrorDialogLater(
+                project,
+                "Requirements installing failed with some reason:\n${errorText}",
+                "Failed to install requirements"
+            )
+            return false
+        }
+        return true
+    } catch (_: TimeoutException) {
         showErrorDialogLater(
             project,
-            "Requirements installing failed with some reason:\n${errorText}",
-            "Requirements error"
+            """
+                Requirements installing failed due to the exceeded waiting time for the installation, check your internet connection.
+                
+                Try to install missing npm packages manually:
+            ${
+                missingPackages.joinToString(separator = "\n") {
+                    "> npm install ${it.npmListFlag} ${it.packageName}"
+                }
+            }
+            """.trimIndent(),
+            "Failed to install requirements"
         )
+        return false
     }
 }

utbot-intellij-js/src/main/kotlin/org/utbot/intellij/plugin/language/js/NycSourceFileChooser.kt

@@ -5,7 +5,7 @@ import com.intellij.openapi.ui.TextBrowseFolderListener
 import com.intellij.openapi.ui.TextFieldWithBrowseButton
 import com.intellij.openapi.ui.ValidationInfo
 import org.utbot.common.PathUtil.replaceSeparator
-import settings.JsPackagesSettings.nycData
+import settings.PackageDataService
 import utils.OsProvider
 
 
@@ -24,8 +24,7 @@ class NycSourceFileChooser(val model: JsTestsModel) : TextFieldWithBrowseButton(
         addBrowseFolderListener(
             TextBrowseFolderListener(descriptor, model.project)
         )
-        text = (replaceSeparator(nycData.findPackagePath() ?: "Nyc was not found")
-                + OsProvider.getProviderByOs().npmPackagePostfix)
+        text = PackageDataService.nycPath
     }
 
     fun validateNyc(): ValidationInfo? {

utbot-js/src/main/kotlin/api/JsTestGenerator.kt

@@ -30,6 +30,7 @@ import org.utbot.framework.plugin.api.UtTimeoutException
 import org.utbot.fuzzer.FuzzedValue
 import org.utbot.fuzzer.UtFuzzedExecution
 import org.utbot.fuzzing.Control
+import org.utbot.fuzzing.utils.Trie
 import parser.JsClassAstVisitor
 import parser.JsFunctionAstVisitor
 import parser.JsFuzzerAstVisitor
@@ -44,6 +45,7 @@ import parser.JsToplevelFunctionAstVisitor
 import service.CoverageMode
 import service.CoverageServiceProvider
 import service.InstrumentationService
+import service.PackageJsonService
 import service.ServiceContext
 import service.TernService
 import settings.JsDynamicSettings
@@ -56,7 +58,6 @@ import utils.constructClass
 import utils.toJsAny
 import java.io.File
 import java.util.concurrent.CancellationException
-import org.utbot.fuzzing.utils.Trie
 
 private val logger = KotlinLogging.logger {}
 
@@ -100,6 +101,10 @@ class JsTestGenerator(
             parsedFile = parsedFile,
             settings = settings,
         )
+        context.packageJson = PackageJsonService(
+            sourceFilePath,
+            projectPath,
+        ).findClosestConfig()
         val ternService = TernService(context)
         val paramNames = mutableMapOf<ExecutableId, List<String>>()
         val testSets = mutableListOf<CgMethodTestSet>()

utbot-js/src/main/kotlin/service/PackageJsonService.kt

@@ -0,0 +1,42 @@
+package service
+
+import org.json.JSONObject
+import java.io.File
+import java.io.FilenameFilter
+
+data class PackageJson(
+    val isModule: Boolean,
+    val deps: Set<String>
+) {
+    companion object {
+        val defaultConfig = PackageJson(false, emptySet())
+    }
+}
+
+class PackageJsonService(
+    private val filePathToInference: String,
+    private val projectPath: String
+) {
+
+    fun findClosestConfig(): PackageJson {
+        var currDir = File(filePathToInference.substringBeforeLast("/"))
+        do {
+            val matchingFiles: Array<File> = currDir.listFiles(
+                FilenameFilter { _, name ->
+                    return@FilenameFilter name == "package.json"
+                }
+            ) ?: throw IllegalStateException("Error occurred while scanning file system")
+            if (matchingFiles.isNotEmpty()) return parseConfig(matchingFiles.first())
+            currDir = currDir.parentFile
+        } while (currDir.path != projectPath)
+        return PackageJson.defaultConfig
+    }
+
+    private fun parseConfig(configFile: File): PackageJson {
+        val configAsJson = JSONObject(configFile.readText())
+        return PackageJson(
+            isModule = configAsJson.optString("type") == "module",
+            deps = configAsJson.optJSONObject("dependencies")?.keySet() ?: emptySet()
+        )
+    }
+}

utbot-js/src/main/kotlin/service/ServiceContext.kt

@@ -9,12 +9,14 @@ class ServiceContext(
     override val filePathToInference: String,
     override val parsedFile: Node,
     override val settings: JsDynamicSettings,
-): ContextOwner
+    override var packageJson: PackageJson = PackageJson.defaultConfig
+) : ContextOwner
 
 interface ContextOwner {
     val utbotDir: String
     val projectPath: String
     val filePathToInference: String
     val parsedFile: Node
     val settings: JsDynamicSettings
-}
+    var packageJson: PackageJson
+}