default.yaml

# Either "cpu" or "cuda"
# NOTE: Cuda requires enough VRAM to load 3 FP16 models (~45 GB for Mistral)
# NOTE 2: The (much slower) CPU mode still requires Cuda capability, but only enough VRAM to load a model once. (~15 GB for Mistral)
device: "cuda"
random_seed: 42 # Random seed to use

directories:
  model_path1: "/home/gryphe/merge-monster/models/Mistral-7B-v0.1" # Path to the base model. Must be a local copy.
  model_directory: "/home/gryphe/merge-monster/models/" # Directory of models to scan, IGNORED if models_to_merge has entries in it
  output_directory: "/home/gryphe/merge-monster/mm-output" # Output directory of the merged model

# A list of models to use as merge candidates - HF syntax, so can be either local directories or repos.
# Overrides model_directory if used
models_to_merge: ["jondurbin/airoboros-m-7b-3.1.2", "Intel/neural-chat-7b-v3-1", "teknium/OpenHermes-2.5-Mistral-7B"]

# Merge ratios used for testing each layer's potential for improvement - Huge impact on total running time
merge_ratios: [0.2, 0.4, 0.6, 0.8]

# Choose from the following methods. Defaults to "lerp".
# "lerp" - Linear interpolation
# "slerp" - Spherical linear interpolation
# "slice" - Highly experimental. The tensor weights shifts from one model to another. [Model 1 > 10% blend > Model 2]
# "cyclic" - Highly experimental. Ignores merge ratios as these are predefined. [Model 1 > 10% blend > 10% Model 2 > 10% blend > Model 1]
merge_method: "lerp"

# If set to true, the lm_head and embed_token tensors (located outside the layers) will also be optimized
# Models that have a different vocab size from model1 will skip this phase automatically as it tends to cause model stability issues
merge_headers: true

# Strategies:
# "cumulative" - Default strategy. If there's a chance of reducing the combined probability, accept the merge.
# "all_phrases" - Only accept the merge if all phrases show an improvement. (Warning: This rarely happens)
# "quantitive" - Ignores probabilities completely. Only looks at how many phrases show an improvement, as defined by the threshold below.
strategy: "cumulative"
# Threshold is currently only used by the "quantitive" strategy. If 0.6, at least 60% of the number of phrases must show am improvement.
strategy_threshold: 0.6

# Whether or not to automatically balance the weights so all phrases are of equal importance to the "cumulative" strategy.
# The weight value of phrases is ignored if set to true.
auto_weights: false

# Phrase = What to measure, weight = multiplication factor, contexts = proceeding contexts
bad_phrases:
  - phrase: "anticipation"
    weight: 10
    contexts: ["Her body quivers with ", "The atmosphere is thick with "]
  - phrase: "unwavering"
    weight: 1
    contexts: ["Filled with an "]
  - phrase: "determination"
    weight: 1
    contexts: ["Her eyes were filled with "]

# Note - Example of a complex phrase
good_phrases:
  - phrase: "The apple is in the bedroom"
    weight: 1
    contexts: ["Question: If I'm in the living room and pick up the apple, go to the bedroom and drop the apple, then walk to the kitchen, where is the apple? Explain your reasoning. Answer: "]