llama_cpp_router/config.example.json at master · Quozul/llama_cpp_router · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
{
	"owner": "llama.cpp",
	"unloadDuration": 30,
	"concurrentModels": 2,
	"system": {
		"llamaServer": "/path/to/llama-server",
		"ggufParser": "/path/to/gguf-parser-linux-amd64",
		"rocmSmi": "/opt/rocm/bin/rocm-smi",
		"memoryType": "unified"
	},
	"server": {
		"hostname": "0.0.0.0",
		"port": 8080
	},
	"models": {
		"granite-4.0-nano": {
			"modelFilePath": "/path/to/granite-4.0-h-1b-Q4_K_M.gguf",
			"multimodalProjectorFilePath": null,
			"unloadable": true,
			"common": {
				"cacheType": "q8_0",
				"contextSize": 4096,
				"noMmap": true,
				"flashAttention": true,
				"nGpuLayers": 99,
				"threads": -1
			},
			"sampling": {
				"temperature": 0.8,
				"topK": 40,
				"topP": 0.9,
				"minP": 0.1,
				"repeatPenalty": 1.0,
				"mirostat": 2
			},
			"network": {
				"host": "0.0.0.0",
				"port": 8081
			}
		},
		"jina-embeddings-v4-text-retrieval": {
			"modelFilePath": "/path/to/jina-embeddings-v4-text-retrieval-Q4_K_M.gguf",
			"multimodalProjectorFilePath": null,
			"unloadable": true,
			"embeddings": true,
			"pooling": "mean",
			"common": {
				"cacheType": "q8_0",
				"contextSize": 4096,
				"noMmap": true,
				"flashAttention": true,
				"nGpuLayers": 99,
				"threads": -1
			},
			"sampling": {
				"temperature": 0.8,
				"topK": 40,
				"topP": 0.9,
				"minP": 0.1,
				"repeatPenalty": 1.0,
				"mirostat": 2
			},
			"network": {
				"host": "0.0.0.0",
				"port": 8082
			}
		}
	}
}