mirror of
https://github.com/ggerganov/llama.cpp.git
synced 2025-04-21 22:16:08 +00:00
e0dbec0bc6
* llama : refactor llama_context, llama_kv_cache, llm_build_context ggml-ci * graph : don't mutate the KV cache during defrag ggml-ci * context : reduce virtuals + remove test function ggml-ci * context : move interface implementation to source file + factory ggml-ci * graph : move KV cache build functions to llama_context impl ggml-ci * graph : remove model reference from build_pooling ggml-ci * graph : remove llama_model reference ggml-ci * kv_cache : provide rope factors ggml-ci * graph : rework inputs to use only unique_ptr, remove attn input abstraction ggml-ci * context : remove llama_context_i abstraction ggml-ci * context : clean-up ggml-ci * graph : clean-up ggml-ci * llama : remove redundant keywords (struct, enum) ggml-ci * model : adapt gemma3 ggml-ci * graph : restore same attention ops as on master ggml-ci * llama : remove TODO + fix indent ggml-ci
77 lines
1.7 KiB
C++
77 lines
1.7 KiB
C++
#pragma once
|
|
|
|
#include "llama.h"
|
|
|
|
#include "ggml-cpp.h"
|
|
|
|
#include <string>
|
|
#include <unordered_map>
|
|
#include <vector>
|
|
|
|
// TODO: pimpl
|
|
|
|
//
|
|
// llama_adapter_cvec
|
|
//
|
|
|
|
struct llama_adapter_cvec {
|
|
ggml_tensor * tensor_for(int il) const;
|
|
|
|
ggml_tensor * apply_to(ggml_context * ctx, ggml_tensor * cur, int il) const;
|
|
|
|
bool apply(
|
|
const llama_model & model,
|
|
const float * data,
|
|
size_t len,
|
|
int32_t n_embd,
|
|
int32_t il_start,
|
|
int32_t il_end);
|
|
|
|
private:
|
|
bool init(const llama_model & model);
|
|
|
|
int32_t layer_start = -1;
|
|
int32_t layer_end = -1;
|
|
|
|
std::vector<ggml_context_ptr> ctxs;
|
|
std::vector<ggml_backend_buffer_ptr> bufs;
|
|
|
|
std::vector<ggml_tensor *> tensors; // per layer
|
|
};
|
|
|
|
//
|
|
// llama_adapter_lora
|
|
//
|
|
|
|
struct llama_adapter_lora_weight {
|
|
ggml_tensor * a = nullptr;
|
|
ggml_tensor * b = nullptr;
|
|
|
|
// get actual scale based on rank and alpha
|
|
float get_scale(float alpha, float adapter_scale) const {
|
|
const float rank = (float) b->ne[0];
|
|
const float scale = alpha ? adapter_scale * alpha / rank : adapter_scale;
|
|
return scale;
|
|
}
|
|
|
|
llama_adapter_lora_weight() = default;
|
|
llama_adapter_lora_weight(ggml_tensor * a, ggml_tensor * b) : a(a), b(b) {}
|
|
};
|
|
|
|
struct llama_adapter_lora {
|
|
// map tensor name to lora_a_b
|
|
std::unordered_map<std::string, llama_adapter_lora_weight> ab_map;
|
|
|
|
std::vector<ggml_context_ptr> ctxs;
|
|
std::vector<ggml_backend_buffer_ptr> bufs;
|
|
|
|
float alpha;
|
|
|
|
llama_adapter_lora() = default;
|
|
~llama_adapter_lora() = default;
|
|
|
|
llama_adapter_lora_weight * get_weight(ggml_tensor * w);
|
|
};
|
|
|
|
using llama_adapter_loras = std::unordered_map<llama_adapter_lora *, float>;
|