*** Welcome to KoboldCpp - Version 1.124.2 Loading Chat Completions Adapter: /var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kcpp_adapters/AutoGuess.json Chat Completions Adapter Loaded MacOS detected: Auto GPU layers set to maximum System: Darwin Darwin Kernel Version 24.6.0: Sun Oct 11 23:44:11 PDT 2026; root:xnu-11417.140.69.x.x~2/RELEASE_ARM64_VMAPPLE arm64 arm Unable to determine GPU Memory Unable to determine available RAM Initializing dynamic library: koboldcpp_default.so ========== Namespace(usecuda=None, usevulkan=None, usecpu=False, model=['models/SmolVLM-256M-Instruct-Q8_0.gguf'], model_param='models/SmolVLM-256M-Instruct-Q8_0.gguf', config=None, contextsize=4096, gpulayers=200, host='', launch=False, tensor_split=None, threads=1, port=49653, port_param=49653, version=False, agent=False, agentload='', analyze='', autofit=False, autofitpadding=1024, batchsize=512, ubatchsize=-1, benchmark=None, blasthreads=0, chatcompletionsadapter='AutoGuess', cli=False, debugmode=0, defaultgenamt=2048, device='', downloaddir='', draftamount=4, draftgpulayers=999, draftgpusplit=None, draftmodel='', enableguidance=False, exportconfig='', exporttemplate='', failsafe=False, foreground=False, gendefaults='', gendefaultsoverwrite=False, genlimit=0, highpriority=False, ignoremissing=False, jinja=False, jinja_kwargs='', jinjatemplate='', jinjathink='default', lora=None, loramult=1.0, lowvram=False, maingpu=-1, maxrequestsize=32, mcpfile='', mmproj='models/mmproj-SmolVLM-256M-Instruct-Q8_0.gguf', mmprojcpu=False, lazy_mode='auto', moe_cache_mib=0, moecpu=0, ffncpu=0, moeexperts=-1, multiplayer=False, nocertify=False, nofastforward=False, noflashattention=False, noavx2=False, nobostoken= nommq=False, nomodel=False, noshift=False, noswa=False, onready='', allow_config_onready=False, overridekv='', overridenativecontext=0, overridetensors='', parallelrequests=1, paralleltotalcontext=0, password= preloadstory='', prompt='', quantkv='f16', queuelimit=10, quiet=False, ratelimit=0, reasoningeffort='default', remotetunnel=False, ropeconfig=[0.0, 10000.0], savedatafile='', singleinstance=False, smartcache=0, smartcontext=False, splitmode='layer', ssl=None, swapadding=0, unpack='', usemtp=False, usemlock=False, usemmap=False, usedirectio=False, visionmaxres=1024, visionmaxtokens=-1, visionmintokens=-1, websearch=False, showgui=False, skiplauncher=True, hordemodelname='', hordeworkername='', hordekey='', hordemaxctx=0, hordegenlen=0, sdaudiovae='', sdclamped=0, sdclampedsoft=0, sdclip1='', sdclip2='', sdclipdevice=-2, sdconvdirect='off', sdflashattention=False, sdlora=[], sdloramult=[1.0], sdmaingpu=None, sdmodel='', sdoffloadcpu=False, sdphotomaker='', sdquant=0, sdllm='', sdthreads=0, sdtiledvae=512, sdupscaler='', sdvae='', sdvaeauto=False, sdvaedevice=-1, sdvramlimit=0, whispermodel='models/ggml-tiny.en.bin', ttsmodel='', ttswavtokenizer='', ttsgpu=False, ttsmaxlen=4096, ttsthreads=0, ttsdir='', musicllm='', musicembeddings='', musicdiffusion='', musicvae='', musiclowvram=False, embeddingsmodel='models/all-MiniLM-L6-v2-Q8_0.gguf', embeddingsmaxctx=4096, embeddingsgpu=False, rpcmode='disabled', rpcport=5551, rpchost='0.0.0.0', rpcdevice='', rpccache='', rpctargets='', admin=False, adminpassword= admindir='', adminunloadtimeout=0, routermode=False, reqtimeout=600, autoswapmode=False, autoswapthreshold=256, baseconfig='', hordeconfig=None, sdconfig=None, noblas=False, nommap=False, nopipelineparallel=False, pipelineparallel=False, sdnotile=False, sdt5xxl=None, sdvaecpu=False, sdclipgpu=False, forceversion=False, sdgendefaults=False, flashattention=False, useswa=False, multiuser=None, testmemory=False, run_bundled_agent=False, agent_base_url=None, agent_api_key= proxy_port=None) ========== Loading Text Model: /Users//work/test-KoboldCpp/test-KoboldCpp/models/SmolVLM-256M-Instruct-Q8_0.gguf ggml_metal_device_init: tensor API disabled for pre-M5 and pre-A19 devices ggml_metal_library_init: default.metallib not found, loading kernel sources ggml_metal_library_init: GGML_METAL_PATH_RESOURCES = /private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/fa_aux.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/fa_f16.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/fa_f32.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/fa_q4_0.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/fa_q4_1.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/fa_q5_0.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/fa_q5_1.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/fa_q8_0.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/fa_tensor.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/fa_vec_f16.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/fa_vec_f32.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/fa_vec_q4_0.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/fa_vec_q4_1.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/fa_vec_q5_0.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/fa_vec_q5_1.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/fa_vec_q8_0.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/mul_mv.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/mul_mv_mma.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/mul_mm.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/quantize.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/softmax.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/norm.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/unary.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/binbcast.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/reduce.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/tri.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/ssm.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/wkv.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/gated_delta_net.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/solve_tri.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/rope.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/conv.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/upscale.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/argsort.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/pool.metal' ggml_metal_library_init: loading '/private/var/folders/2s/vty41l455yv698g4xkgfv7z00000gn/T/_MEI00006a4el6m5vT/kernels/misc.metal' ggml_metal_library_compile_all: compiled 'fa_aux' library in 0.012 sec ggml_metal_library_compile_all: compiled 'fa_f16' library in 0.016 sec ggml_metal_library_compile_all: compiled 'fa_f32' library in 0.014 sec ggml_metal_library_compile_all: compiled 'fa_q4_0' library in 0.011 sec ggml_metal_library_compile_all: compiled 'fa_q4_1' library in 0.012 sec ggml_metal_library_compile_all: compiled 'fa_q5_0' library in 0.022 sec ggml_metal_library_compile_all: compiled 'fa_q5_1' library in 0.014 sec ggml_metal_library_compile_all: compiled 'fa_q8_0' library in 0.012 sec ggml_metal_library_compile_all: compiled 'fa_tensor' library in 0.003 sec ggml_metal_library_compile_all: compiled 'fa_vec_f16' library in 0.019 sec ggml_metal_library_compile_all: compiled 'fa_vec_f32' library in 0.012 sec ggml_metal_library_compile_all: compiled 'fa_vec_q4_0' library in 0.023 sec ggml_metal_library_compile_all: compiled 'fa_vec_q4_1' library in 0.020 sec ggml_metal_library_compile_all: compiled 'fa_vec_q5_0' library in 0.019 sec ggml_metal_library_compile_all: compiled 'fa_vec_q5_1' library in 0.019 sec ggml_metal_library_compile_all: compiled 'fa_vec_q8_0' library in 0.016 sec ggml_metal_library_compile_all: compiled 'mul_mv' library in 0.025 sec ggml_metal_library_compile_all: compiled 'mul_mv_mma' library in 0.027 sec ggml_metal_library_compile_all: compiled 'mul_mm' library in 0.025 sec ggml_metal_library_compile_all: compiled 'quantize' library in 0.013 sec ggml_metal_library_compile_all: compiled 'softmax' library in 0.008 sec ggml_metal_library_compile_all: compiled 'norm' library in 0.005 sec ggml_metal_library_compile_all: compiled 'unary' library in 0.007 sec ggml_metal_library_compile_all: compiled 'binbcast' library in 0.010 sec ggml_metal_library_compile_all: compiled 'reduce' library in 0.003 sec ggml_metal_library_compile_all: compiled 'tri' library in 0.002 sec ggml_metal_library_compile_all: compiled 'ssm' library in 0.006 sec ggml_metal_library_compile_all: compiled 'wkv' library in 0.002 sec ggml_metal_library_compile_all: compiled 'gated_delta_net' library in 0.005 sec ggml_metal_library_compile_all: compiled 'solve_tri' library in 0.004 sec ggml_metal_library_compile_all: compiled 'rope' library in 0.003 sec ggml_metal_library_compile_all: compiled 'conv' library in 0.003 sec ggml_metal_library_compile_all: compiled 'upscale' library in 0.002 sec ggml_metal_library_compile_all: compiled 'argsort' library in 0.003 sec ggml_metal_library_compile_all: compiled 'pool' library in 0.002 sec ggml_metal_library_compile_all: compiled 'misc' library in 0.003 sec ggml_metal_library_compile_all: loaded 36 libraries from source in 0.144 sec (max single = 0.027 sec) ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s) ggml_metal_device_init: GPU name: MTL0 (Apple Paravirtual device) ggml_metal_device_init: GPU family: MTLGPUFamilyApple5 (1005) ggml_metal_device_init: simdgroup reduction = false ggml_metal_device_init: simdgroup matrix mul. = false ggml_metal_device_init: has unified memory = true ggml_metal_device_init: has bfloat = false ggml_metal_device_init: has tensor = false ggml_metal_device_init: use residency sets = true ggml_metal_device_init: use shared buffers = true ggml_metal_device_init: recommendedMaxWorkingSetSize = 5010.80 MB llama_model_loader: loaded meta data with 49 key-value pairs and 273 tensors from /Users//work/test-KoboldCpp/test-KoboldCpp/models/SmolVLM-256M-Instruct-Q8_0.gguf (version GGUF V3 (latest)) print_info: file format = GGUF V3 (latest) print_info: file type = unknown, may not work print_info: file size = 165.24 MiB (8.51 BPW) llama_prepare_model_devices: using device MTL0 (Apple Paravirtual device) (unknown id) - 4778 MiB free init_tokenizer: initializing tokenizer for type 2 load: 0 unused tokens load: printing all EOG tokens: load: - 0 ('<|endoftext|>') load: - 2 ('<|im_end|>') load: - 4 ('') load: - 49279 ('') load: special tokens cache size = 145 load: token to piece cache size = 0.3199 MB print_info: arch = llama print_info: vocab_only = 0 print_info: no_alloc = 0 print_info: n_ctx_train = 8192 print_info: n_embd_inp = 576 print_info: n_embd = 576 print_info: n_embd_out = 576 print_info: n_layer = 30 print_info: n_layer_all = 30 print_info: n_head = 9 print_info: n_head_kv = 3 print_info: n_rot = 64 print_info: n_swa = 0 print_info: is_swa_any = 0 print_info: non_causal_type = 0 print_info: n_embd_head_k = 64 print_info: n_embd_head_v = 64 print_info: n_gqa = 3 print_info: n_embd_k_gqa = 192 print_info: n_embd_v_gqa = 192 print_info: f_norm_eps = 0.0e+00 print_info: f_norm_rms_eps = 1.0e-05 print_info: f_clamp_kqv = 0.0e+00 print_info: f_max_alibi_bias = 0.0e+00 print_info: f_logit_scale = 0.0e+00 print_info: f_attn_scale = 0.0e+00 print_info: f_attn_value_scale = 0.0000 print_info: n_ff = 1536 print_info: n_expert = 0 print_info: n_expert_used = 0 print_info: n_expert_groups = 0 print_info: n_group_used = 0 print_info: causal attn = 1 print_info: pooling type = -1 print_info: rope type = 0 print_info: rope scaling = linear print_info: freq_base_train = 100000.0 print_info: freq_scale_train = 1 print_info: n_ctx_orig_yarn = 8192 print_info: rope_yarn_log_mul = 0.0000 print_info: rope_finetuned = unknown print_info: model type = 256M print_info: model params = 162.97 M print_info: general.name = SmolVLM 256M Instruct print_info: vocab type = BPE print_info: n_vocab = 49280 print_info: n_merges = 48900 print_info: BOS token = '<|im_start|>' print_info: EOS token = '' print_info: EOT token = '' print_info: UNK token = '<|endoftext|>' print_info: PAD token = '<|im_end|>' print_info: LF token = 'Ċ' print_info: FIM REP token = '' print_info: EOG token = '<|endoftext|>' print_info: EOG token = '<|im_end|>' print_info: EOG token = '' print_info: EOG token = '' print_info: max token length = 162 load_tensors: loading model tensors, this can take a while... (load_mode = none) done_getting_tensors: tensor 'token_embd.weight' (q8_0) (and 211 others) moved from CPU_REPACK, using CPU instead load_tensors: offloading output layer to GPU load_tensors: offloading 29 repeating layers to GPU load_tensors: offloaded 31/31 layers to GPU load_tensors: CPU model buffer size = 28.76 MiB load_tensors: CPU_REPACK model buffer size = 136.34 MiB load_tensors: MTL0 model buffer size = 0.13 MiB ................................................................... llama_context: constructing llama_context llama_context: n_seq_max = 1 llama_context: n_ctx = 4352 llama_context: n_ctx_seq = 4352 llama_context: n_batch = 512 llama_context: n_ubatch = 512 llama_context: causal_attn = 1 llama_context: flash_attn = enabled llama_context: kv_unified = true llama_context: freq_base = 100000.0 llama_context: freq_scale = 1 llama_context: n_rs_seq = 0 llama_context: n_outputs_max = 512 llama_context: n_outputs_max_per_seq = 1 llama_context: n_ctx_seq (4352) < n_ctx_train (8192) -- the full capacity of the model will not be utilized ggml_metal_init: allocating ggml_metal_init: picking default device: Apple Paravirtual device ggml_metal_init: use fusion = true ggml_metal_init: use concurrency = true ggml_metal_init: use graph optimize = true set_abort_callback: call llama_context: CPU output buffer size = 0.19 MiB llama_kv_cache: MTL0 KV buffer size = 95.62 MiB llama_kv_cache: size = 95.62 MiB ( 4352 cells, 30 layers, 1/1 seqs), K (f16): 47.81 MiB, V (f16): 47.81 MiB llama_kv_cache: n_rot_k = 0, n_embd_head_k_all = 64 llama_kv_cache: n_rot_v = 0, n_embd_head_k_all = 64 llama_context: enumerating backends llama_context: backend_ptrs.size() = 3 sched_reserve: reserving ... sched_reserve: max_nodes = 2184 resolve_fused_ops: resolving fused DeepSeek V4 HC support: resolve_fused_ops: fused DeepSeek V4 HC pre enabled resolve_fused_ops: fused DeepSeek V4 HC comb enabled resolve_fused_ops: fused DeepSeek V4 HC post enabled sched_reserve: MTL0 compute buffer size = 4.50 MiB sched_reserve: CPU compute buffer size = 98.50 MiB sched_reserve: graph: nodes = 939, splits = 305, input objects = 4, input tensors = 9 sched_reserve: reserve took 4.09 ms, sched copies = 1 attach_threadpool: call clip_model_loader: model name: SmolVLM 256M Instruct clip_model_loader: description: clip_model_loader: GGUF version: 3 clip_model_loader: alignment: 32 clip_model_loader: n_tensors: 198 clip_model_loader: n_kv: 40 clip_model_loader: has vision encoder ggml_metal_init: allocating ggml_metal_init: picking default device: Apple Paravirtual device ggml_metal_init: use fusion = true ggml_metal_init: use concurrency = true ggml_metal_init: use graph optimize = true clip_ctx: CLIP using MTL0 backend load_hparams: projector: idefics3 load_hparams: n_embd: 768 load_hparams: n_head: 12 load_hparams: n_ff: 3072 load_hparams: n_layer: 12 load_hparams: ffn_op: gelu load_hparams: projection_dim: 576 --- vision hparams --- load_hparams: image_size: 512 load_hparams: patch_size: 16 load_hparams: has_llava_proj: 0 load_hparams: minicpmv_version: 0 load_hparams: n_merge: 4 load_hparams: n_wa_pattern: 0 load_hparams: model size: 98.95 MiB load_hparams: metadata size: 0.09 MiB load_tensors: ffn up/down are swapped load_tensors: loaded 198 tensors from /Users//work/test-KoboldCpp/test-KoboldCpp/models/mmproj-SmolVLM-256M-Instruct-Q8_0.gguf get_dummy_batch: warmup with image size = 512 x 512 ggml_metal_library_compile_pipeline: compiling pipeline: base = 'kernel_bin_fuse_f32_f32_f32_4', name = 'kernel_bin_fuse_f32_f32_f32_4_op=2_nf=1_rb=1_cb=0' ggml_metal_library_compile_pipeline: loaded kernel_bin_fuse_f32_f32_f32_4_op=2_nf=1_rb=1_cb=0 0x6000016b41e0 | th_max = 1024 | th_width = 32 ggml_metal_library_compile_pipeline: compiling pipeline: base = 'kernel_rope_norm_f32', name = 'kernel_rope_norm_f32_imrope=0_is_back=0' ggml_metal_library_compile_pipeline: loaded kernel_rope_norm_f32_imrope=0_is_back=0 0x6000016acbe0 | th_max = 1024 | th_width = 32 ggml_metal_library_compile_pipeline: compiling pipeline: base = 'kernel_set_rows_f32_i64_f16', name = 'kernel_set_rows_f32_i64_f16' ggml_metal_library_compile_pipeline: loaded kernel_set_rows_f32_i64_f16 0x6000016b4280 | th_max = 1024 | th_width = 32 ggml_metal_library_compile_pipeline: compiling pipeline: base = 'kernel_bin_fuse_f32_f32_f32_4', name = 'kernel_bin_fuse_f32_f32_f32_4_op=0_nf=1_rb=0_cb=0' ggml_metal_library_compile_pipeline: loaded kernel_bin_fuse_f32_f32_f32_4_op=0_nf=1_rb=0_cb=0 0x6000016b4320 | th_max = 1024 | th_width = 32 ggml_metal_library_compile_pipeline: compiling pipeline: base = 'kernel_get_rows_f32', name = 'kernel_get_rows_f32' ggml_metal_library_compile_pipeline: loaded kernel_get_rows_f32 0x6000016acc80 | th_max = 1024 | th_width = 32 ggml_metal_library_compile_pipeline: compiling pipeline: base = 'kernel_bin_fuse_f32_f32_f32_4', name = 'kernel_bin_fuse_f32_f32_f32_4_op=0_nf=1_rb=1_cb=0' ggml_metal_library_compile_pipeline: loaded kernel_bin_fuse_f32_f32_f32_4_op=0_nf=1_rb=1_cb=0 0x6000016acd20 | th_max = 1024 | th_width = 32 ggml_metal_init: allocating ggml_metal_init: picking default device: Apple Paravirtual device ggml_metal_init: use fusion = true ggml_metal_init: use concurrency = true ggml_metal_init: use graph optimize = true ggml_metal_free: deallocating ggml_metal_init: allocating ggml_metal_init: picking default device: Apple Paravirtual device ggml_metal_init: use fusion = true ggml_metal_init: use concurrency = true ggml_metal_init: use graph optimize = true ggml_metal_free: deallocating llama_model_loader: loaded meta data with 24 key-value pairs and 101 tensors from /Users//work/test-KoboldCpp/test-KoboldCpp/models/all-MiniLM-L6-v2-Q8_0.gguf (version GGUF V3 (latest)) print_info: file format = GGUF V3 (latest) print_info: file type = (guessed) unknown, may not work print_info: file size = 23.13 MiB (8.60 BPW) llama_prepare_model_devices: using device MTL0 (Apple Paravirtual device) (unknown id) - 4578 MiB free init_tokenizer: initializing tokenizer for type 3 load: 0 unused tokens load: special_eos_id is not in special_eog_ids - the tokenizer config may be incorrect load: printing all EOG tokens: load: - 0 ('[PAD]') load: - 102 ('[SEP]') load: special tokens cache size = 5 load: token to piece cache size = 0.2032 MB print_info: arch = bert print_info: vocab_only = 0 print_info: no_alloc = 0 print_info: n_ctx_train = 512 print_info: n_embd_inp = 384 print_info: n_embd = 384 print_info: n_embd_out = 384 print_info: n_layer = 6 print_info: n_layer_all = 6 print_info: n_head = 12 print_info: n_head_kv = 12 print_info: n_rot = 32 print_info: n_swa = 0 print_info: is_swa_any = 0 print_info: non_causal_type = 0 print_info: n_embd_head_k = 32 print_info: n_embd_head_v = 32 print_info: n_gqa = 1 print_info: n_embd_k_gqa = 384 print_info: n_embd_v_gqa = 384 print_info: f_norm_eps = 1.0e-12 print_info: f_norm_rms_eps = 0.0e+00 print_info: f_clamp_kqv = 0.0e+00 print_info: f_max_alibi_bias = 0.0e+00 print_info: f_logit_scale = 0.0e+00 print_info: f_attn_scale = 0.0e+00 print_info: f_attn_value_scale = 0.0000 print_info: n_ff = 1536 print_info: n_expert = 0 print_info: n_expert_used = 0 print_info: n_expert_groups = 0 print_info: n_group_used = 0 print_info: causal attn = 0 print_info: pooling type = 1 print_info: rope type = 2 print_info: rope scaling = linear print_info: freq_base_train = 10000.0 print_info: freq_scale_train = 1 print_info: n_ctx_orig_yarn = 512 print_info: rope_yarn_log_mul = 0.0000 print_info: rope_finetuned = unknown print_info: model type = 22M print_info: model params = 22.57 M print_info: general.name = all-MiniLM-L6-v2 print_info: vocab type = WPM print_info: n_vocab = 30522 print_info: n_merges = 0 print_info: BOS token = '[CLS]' print_info: EOS token = '[SEP]' print_info: UNK token = '[UNK]' print_info: SEP token = '[SEP]' print_info: PAD token = '[PAD]' print_info: MASK token = '[MASK]' print_info: LF token = '[PAD]' print_info: FIM PAD token = '[PAD]' print_info: EOG token = '[PAD]' print_info: EOG token = '[SEP]' print_info: max token length = 21 load_tensors: loading model tensors, this can take a while... (load_mode = none) done_getting_tensors: tensor 'token_embd.weight' (q8_0) (and 64 others) moved from CPU_REPACK, using CPU instead load_tensors: offloading 0 repeating layers to GPU load_tensors: offloaded 0/7 layers to GPU load_tensors: CPU model buffer size = 12.37 MiB load_tensors: CPU_REPACK model buffer size = 10.76 MiB .............................. llama_context: constructing llama_context llama_context: n_seq_max = 1 llama_context: n_ctx = 512 llama_context: n_ctx_seq = 512 llama_context: n_batch = 512 llama_context: n_ubatch = 512 llama_context: causal_attn = 0 llama_context: flash_attn = enabled llama_context: kv_unified = true llama_context: freq_base = 10000.0 llama_context: freq_scale = 1 llama_context: n_rs_seq = 0 llama_context: n_outputs_max = 512 llama_context: n_outputs_max_per_seq = 1 ggml_metal_init: allocating ggml_metal_init: picking default device: Apple Paravirtual device ggml_metal_init: use fusion = true ggml_metal_init: use concurrency = true ggml_metal_init: use graph optimize = true set_abort_callback: call llama_context: CPU output buffer size = 0.12 MiB llama_context: enumerating backends llama_context: backend_ptrs.size() = 3 sched_reserve: reserving ... sched_reserve: max_nodes = 1024 init: embeddings required but some input tokens were not marked as outputs -> overriding Load Text Model OK: True Chat template heuristics failed to identify chat completions format. Jinja will be used. Load Audio Transcription Model OK: True Load Embeddings Model OK: True Embedded KoboldAI Lite loaded. Embedded API docs loaded. Llama.cpp UI loaded. Embedded MusicUI loaded. ====== Active Modules: TextGeneration VoiceRecognition MultimodalVision VectorEmbeddings Inactive Modules: ImageGeneration MultimodalAudio NetworkMultiplayer ApiKeyPassword WebSearchProxy TextToSpeech AdminControl MCPBridge MusicGen RouterMode Enabled APIs: KoboldCppApi OpenAiApi OllamaApi AnthropicApi WhisperTranscribeApi Note: For third party Ollama API Emulation, you should set the port to 11434. Starting Kobold API on port 49653 at http://localhost:49653/api/ Starting OpenAI Compatible API on port 49653 at http://localhost:49653/v1/ Starting llama.cpp secondary WebUI at http://localhost:49653/lcpp/ MusicUI is available at http://localhost:49653/musicui/ ====== Please connect to custom endpoint at http://localhost:49653 mtmd_group_mergeable_bitmaps: no merging for part index 0 add_media: preproc_out has 1 entries, grid_x = 1, grid_y = 1, has_overview = 1 add_media: adding 1 slices (1 rows x 1 cols) add_text: add_media: adding slice image at row 0 col 0 The reported GGUF Arch is: llama Arch Category: 1 --- Identified as GGUF model. Attempting to Load... --- Using automatic RoPE scaling for GGUF. If the model has custom RoPE settings, they'll be used directly instead! System Info: AVX = 0 | AVX_VNNI = 0 | AVX2 = 0 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | AVX512_BF16 = 0 | AMX_INT8 = 0 | FMA = 0 | NEON = 1 | SVE = 0 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | RISCV_VECT = 0 | WASM_SIMD = 0 | SSE3 = 0 | SSSE3 = 0 | VSX = 0 | MATMUL_INT8 = 0 | LLAMAFILE = 1 | Number of Backends: 4 Backend 0: MTL Backend 1: BLAS Backend 2: RPC Backend 3: CPU This model does not use SWA Automatic RoPE Scaling: Using model internal value. Threadpool set to 1 threads and 1 blasthreads... Attempting to apply Multimodal Projector: /Users//work/test-KoboldCpp/test-KoboldCpp/models/mmproj-SmolVLM-256M-Instruct-Q8_0.gguf Starting model warm up, please wait a moment... Loading Whisper Model: /Users//work/test-KoboldCpp/test-KoboldCpp/models/ggml-tiny.en.bin Whisper Load Complete. Loading Embeddings Model: /Users//work/test-KoboldCpp/test-KoboldCpp/models/all-MiniLM-L6-v2-Q8_0.gguf Embeddings Model Load Complete. Processing Prompt (0 / 159 tokens) Processing Prompt (18 / 159 tokens)encoding image slice... reserve_compute_meta: MTL0 compute buffer size = 21.00 MiB reserve_compute_meta: CPU compute buffer size = 17.39 MiB reserve_compute_meta: graph splits = 210, nodes = 396 warmup: flash attention is not supported by the current backend; falling back to CPU (performance will be degraded) warmup: flash attention is enabled warmup: ***************************************************************** warmup: WARNING: the CLIP graph uses unsupported operators by the backend warmup: the performance will be suboptimal warmup: list of unsupported ops (backend=MTL0): warmup: MUL_MAT: type = f32, ne = [1024 768 1 1] warmup: NORM: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: FLASH_ATTN_EXT: type = f32, ne = [64 12 1024 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: NORM: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [3072 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: NORM: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: FLASH_ATTN_EXT: type = f32, ne = [64 12 1024 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: NORM: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [3072 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: NORM: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: FLASH_ATTN_EXT: type = f32, ne = [64 12 1024 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: NORM: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [3072 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: NORM: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: FLASH_ATTN_EXT: type = f32, ne = [64 12 1024 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: NORM: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [3072 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: NORM: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: FLASH_ATTN_EXT: type = f32, ne = [64 12 1024 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: NORM: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [3072 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: NORM: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: FLASH_ATTN_EXT: type = f32, ne = [64 12 1024 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: NORM: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [3072 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: NORM: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: FLASH_ATTN_EXT: type = f32, ne = [64 12 1024 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: NORM: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [3072 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: NORM: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: FLASH_ATTN_EXT: type = f32, ne = [64 12 1024 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: NORM: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [3072 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: NORM: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: FLASH_ATTN_EXT: type = f32, ne = [64 12 1024 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: NORM: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [3072 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: NORM: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: FLASH_ATTN_EXT: type = f32, ne = [64 12 1024 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: NORM: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [3072 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: NORM: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: FLASH_ATTN_EXT: type = f32, ne = [64 12 1024 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: NORM: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [3072 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: NORM: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: FLASH_ATTN_EXT: type = f32, ne = [64 12 1024 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: NORM: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [3072 1024 1 1] warmup: MUL_MAT: type = f32, ne = [768 1024 1 1] warmup: NORM: type = f32, ne = [768 1024 1 1] warmup: MUL_MAT: type = f32, ne = [576 64 1 1] warmup: flash attention is enabled warmup: please report this on github as an issue warmup: ref: https://github.com/ggml-org/llama.cpp/pull/16837#issuecomment-3461676118 warmup: ***************************************************************** clip_encode: copying image 1/1 to input buffer (nx=512, ny=512) ggml_metal_library_compile_pipeline: compiling pipeline: base = 'kernel_im2col_f16', name = 'kernel_im2col_f16' ggml_metal_library_compile_pipeline: loaded kernel_im2col_f16 0x6000016b4000 | th_max = 1024 | th_width = 32 ggml_metal_library_compile_pipeline: compiling pipeline: base = 'kernel_cpy_f32_f32', name = 'kernel_cpy_f32_f32' ggml_metal_library_compile_pipeline: loaded kernel_cpy_f32_f32 0x6000016ad540 | th_max = 1024 | th_width = 32 ggml_metal_library_compile_pipeline: compiling pipeline: base = 'kernel_bin_fuse_f32_f32_f32_4', name = 'kernel_bin_fuse_f32_f32_f32_4_op=2_nf=1_rb=0_cb=0' ggml_metal_library_compile_pipeline: loaded kernel_bin_fuse_f32_f32_f32_4_op=2_nf=1_rb=0_cb=0 0x6000016b43c0 | th_max = 1024 | th_width = 32 ggml_metal_library_compile_pipeline: compiling pipeline: base = 'kernel_cpy_f32_f16', name = 'kernel_cpy_f32_f16' ggml_metal_library_compile_pipeline: loaded kernel_cpy_f32_f16 0x6000016ad5e0 | th_max = 1024 | th_width = 32 ggml_metal_library_compile_pipeline: compiling pipeline: base = 'kernel_unary_f32_f32_4', name = 'kernel_unary_f32_f32_4_op=103_cnt=0' ggml_metal_library_compile_pipeline: loaded kernel_unary_f32_f32_4_op=103_cnt=0 0x6000016b4460 | th_max = 1024 | th_width = 32 clip_encode: output embedding shape [576, 64, 1] image slice encoded in 2502 ms decoding image batch 1/1, n_tokens_batch = 64 image decoded (batch 1/1) in 209 ms encoding image slice... clip_encode: copying image 1/1 to input buffer (nx=512, ny=512) clip_encode: output embedding shape [576, 64, 1] image slice encoded in 2407 ms decoding image batch 1/1, n_tokens_batch = 64 image decoded (batch 1/1) in 209 ms Processing Media Embedding 1 (2 tokens) Processing Media Embedding 1 (64 tokens) Processing Media Embedding 1 (3 tokens) Processing Media Embedding 1 (64 tokens) Processing Media Embedding 1 (1 tokens) Processing Media End Separator (1 tokens) Processing Prompt (159 / 159 tokens) Generating (1 / 80 tokens) Generating (2 / 80 tokens) Generating (3 / 80 tokens) Generating (4 / 80 tokens) Generating (5 / 80 tokens) Generating (6 / 80 tokens) Generating (7 / 80 tokens) Generating (8 / 80 tokens) Generating (9 / 80 tokens) (EOS token triggered! ID:49279) [13:25:30] CtxLimit:168/4096, Init:0.01s, Processed:159 in 5.87s (27.10T/s), Generated:9/80 in 0.69s (13.08T/s), Total:6.57s