Text to vector refactoring

changelog/unreleased/PR#4375-language-model-refactoring.yml

@@ -0,0 +1,9 @@
+title: Refactor of the language-model module to accomodate new "Document Enrichment with LLMs" feature avoiding code duplication.
+type: changed
+authors:
+  - name: Nicolò Rinaldi
+  - name: Anna Ruggero
+  - name: Alessandro benedetti
+links:
+- name: PR#4375
+  url: https://github.com/apache/solr/pull/4375

solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/store/TextToVectorModelException.java → solr/modules/language-models/src/java/org/apache/solr/languagemodels/LanguageModelException.java

@@ -14,17 +14,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.solr.languagemodels.textvectorisation.store;
+package org.apache.solr.languagemodels;
 
-public class TextToVectorModelException extends RuntimeException {
+public class LanguageModelException extends RuntimeException {
 
   private static final long serialVersionUID = 1L;
 
-  public TextToVectorModelException(String message) {
+  public LanguageModelException(String message) {
     super(message);
   }
 
-  public TextToVectorModelException(String message, Exception cause) {
+  public LanguageModelException(String message, Exception cause) {
     super(message, cause);
   }
 }

solr/modules/language-models/src/java/org/apache/solr/languagemodels/model/SolrLanguageModel.java

@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.languagemodels.model;
+
+import java.util.Map;
+
+/**
+ * Abstract base class for Solr-managed wrappers around langchain4j used in {@code language-models}
+ * module
+ */
+public abstract class SolrLanguageModel {
+
+  // common parameters
+  protected static final String TIMEOUT_PARAM = "timeout";
+  protected static final String MAX_RETRIES_PARAM = "maxRetries";
+
+  protected final String name;
+  protected final Map<String, Object> params;
+
+  protected SolrLanguageModel(String name, Map<String, Object> params) {
+    this.name = name;
+    this.params = params;
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public Map<String, Object> getParams() {
+    return params;
+  }
+
+  /** Returns the class name of the underlying langchain4j model instance. */
+  public abstract String getModelClassName();
+}

solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/model/SolrTextToVectorModel.java → solr/modules/language-models/src/java/org/apache/solr/languagemodels/model/SolrTextToVectorModel.java

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.solr.languagemodels.textvectorisation.model;
+package org.apache.solr.languagemodels.model;
 
 import dev.langchain4j.data.embedding.Embedding;
 import dev.langchain4j.model.embedding.EmbeddingModel;
@@ -28,26 +28,21 @@
 import org.apache.lucene.util.RamUsageEstimator;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.core.SolrResourceLoader;
-import org.apache.solr.languagemodels.textvectorisation.store.TextToVectorModelException;
-import org.apache.solr.languagemodels.textvectorisation.store.rest.ManagedTextToVectorModelStore;
+import org.apache.solr.languagemodels.LanguageModelException;
+import org.apache.solr.languagemodels.store.rest.ManagedTextToVectorModelStore;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * This object wraps a {@link dev.langchain4j.model.embedding.EmbeddingModel} to encode text to
- * vector. It's meant to be used as a managed resource with the {@link
- * ManagedTextToVectorModelStore}
+ * This object wraps a {@link EmbeddingModel} to encode text to vector. It's meant to be used as a
+ * managed resource with the {@link ManagedTextToVectorModelStore}
  */
-public class SolrTextToVectorModel implements Accountable {
+public class SolrTextToVectorModel extends SolrLanguageModel implements Accountable {
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
   private static final long BASE_RAM_BYTES =
       RamUsageEstimator.shallowSizeOfInstance(SolrTextToVectorModel.class);
-  private static final String TIMEOUT_PARAM = "timeout";
   private static final String MAX_SEGMENTS_PER_BATCH_PARAM = "maxSegmentsPerBatch";
-  private static final String MAX_RETRIES_PARAM = "maxRetries";
 
-  private final String name;
-  private final Map<String, Object> params;
   private final EmbeddingModel textToVector;
   private final int hashCode;
 
@@ -56,7 +51,7 @@ public static SolrTextToVectorModel getInstance(
       String className,
       String name,
       Map<String, Object> params)
-      throws TextToVectorModelException {
+      throws LanguageModelException {
     try {
       /*
        * The idea here is to build a {@link dev.langchain4j.model.embedding.EmbeddingModel} using inversion
@@ -125,15 +120,14 @@ public static SolrTextToVectorModel getInstance(
       textToVector = (EmbeddingModel) builder.getClass().getMethod("build").invoke(builder);
       return new SolrTextToVectorModel(name, textToVector, params);
     } catch (final Exception e) {
-      throw new TextToVectorModelException("Model loading failed for " + className, e);
+      throw new LanguageModelException("Model loading failed for " + className, e);
     }
   }
 
   public SolrTextToVectorModel(
       String name, EmbeddingModel textToVector, Map<String, Object> params) {
-    this.name = name;
+    super(name, params);
     this.textToVector = textToVector;
-    this.params = params;
     this.hashCode = calculateHashCode();
   }
 
@@ -170,20 +164,12 @@ private int calculateHashCode() {
   @Override
   public boolean equals(Object obj) {
     if (this == obj) return true;
-    if (!(obj instanceof SolrTextToVectorModel)) return false;
-    final SolrTextToVectorModel other = (SolrTextToVectorModel) obj;
+    if (!(obj instanceof SolrTextToVectorModel other)) return false;
     return Objects.equals(textToVector, other.textToVector) && Objects.equals(name, other.name);
   }
 
-  public String getName() {
-    return name;
-  }
-
-  public String getEmbeddingModelClassName() {
+  @Override
+  public String getModelClassName() {
     return textToVector.getClass().getName();
   }
-
-  public Map<String, Object> getParams() {
-    return params;
-  }
 }

solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/store/package-info.java → solr/modules/language-models/src/java/org/apache/solr/languagemodels/model/package-info.java

@@ -15,5 +15,5 @@
  * limitations under the License.
  */
 
-/** Contains model store related classes. */
-package org.apache.solr.languagemodels.textvectorisation.store;
+/** Contains model related classes. */
+package org.apache.solr.languagemodels.model;

solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/model/package-info.java → solr/modules/language-models/src/java/org/apache/solr/languagemodels/package-info.java

@@ -15,5 +15,5 @@
  * limitations under the License.
  */
 
-/** APIs and classes for implementing text to vector logic. */
-package org.apache.solr.languagemodels.textvectorisation.model;
+/** Contains the whole module for Language Models. */
+package org.apache.solr.languagemodels;

solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/search/TextToVectorQParserPlugin.java → solr/modules/language-models/src/java/org/apache/solr/languagemodels/search/TextToVectorQParserPlugin.java

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.solr.languagemodels.textvectorisation.search;
+package org.apache.solr.languagemodels.search;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -26,8 +26,8 @@
 import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.core.SolrResourceLoader;
-import org.apache.solr.languagemodels.textvectorisation.model.SolrTextToVectorModel;
-import org.apache.solr.languagemodels.textvectorisation.store.rest.ManagedTextToVectorModelStore;
+import org.apache.solr.languagemodels.model.SolrTextToVectorModel;
+import org.apache.solr.languagemodels.store.rest.ManagedTextToVectorModelStore;
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.rest.ManagedResource;
 import org.apache.solr.rest.ManagedResourceObserver;

solr/modules/language-models/src/java/org/apache/solr/languagemodels/search/package-info.java

@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** APIs and classes for implementing language models QueryParsers. */
+package org.apache.solr.languagemodels.search;