Skip to content

Commit 6da5130

Browse files
tc-mbtc-mb
committed
support minicpmv2.6
1 parent 65f7455 commit 6da5130

File tree

4 files changed

+109
-26
lines changed

4 files changed

+109
-26
lines changed

‎examples/llava/clip.cpp

Lines changed: 77 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ static std::string format(const char * fmt, ...) {
7878
#define KEY_HAS_VIS_ENC "clip.has_vision_encoder"
7979
#define KEY_HAS_LLAVA_PROJ "clip.has_llava_projector"
8080
#define KEY_HAS_MINICPMV_PROJ "clip.has_minicpmv_projector"
81+
#define KEY_MINICPMV_VERSION "clip.minicpmv_version"
8182
#define KEY_USE_GELU "clip.use_gelu"
8283
#define KEY_N_EMBD "clip.%s.embedding_length"
8384
#define KEY_N_FF "clip.%s.feed_forward_length"
@@ -526,6 +527,7 @@ struct clip_ctx {
526527
bool has_vision_encoder = false;
527528
bool has_llava_projector = false;
528529
bool has_minicpmv_projector = false;
530+
int minicpmv_version = 2;
529531

530532
struct clip_vision_model vision_model;
531533
projector_type proj_type = PROJECTOR_TYPE_MLP;
@@ -616,7 +618,7 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32
616618
inp = ggml_add(ctx0, inp, model.patch_bias);
617619
}
618620
struct ggml_tensor * embeddings = inp;
619-
struct ggml_tensor * pos_embed;
621+
struct ggml_tensor * pos_embed = nullptr;
620622

621623
if(ctx->has_llava_projector){
622624
// concat class_embeddings and patch_embeddings
@@ -638,10 +640,15 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32
638640
embeddings =
639641
ggml_add(ctx0, embeddings, ggml_get_rows(ctx0, model.position_embeddings, positions));
640642

641-
if(ctx->has_minicpmv_projector){
643+
if (ctx->has_minicpmv_projector) {
642644
int pos_w = image_size_width/patch_size;
643645
int pos_h = image_size_height/patch_size;
644-
pos_embed = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, 4096, pos_w * pos_h, 1);
646+
if (ctx->minicpmv_version == 2) {
647+
pos_embed = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, 4096, pos_w * pos_h, 1);
648+
}
649+
else if (ctx->minicpmv_version == 3) {
650+
pos_embed = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, 3584, pos_w * pos_h, 1);
651+
}
645652
ggml_set_name(pos_embed, "pos_embed");
646653
ggml_set_input(pos_embed);
647654
}
@@ -931,8 +938,7 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32
931938
}
932939
}
933940
// minicpmv projector
934-
else if(ctx->has_minicpmv_projector)
935-
{
941+
else if (ctx->has_minicpmv_projector) {
936942
if (ctx->proj_type == PROJECTOR_TYPE_RESAMPLER) {
937943
struct ggml_tensor * q = model.mm_model_query;
938944
{ // layernorm
@@ -950,11 +956,20 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32
950956
}
951957

952958
{ // attention
953-
const int hidden_size = 4096;
959+
int hidden_size = 4096;
954960
const int d_head = 128;
955-
const int n_head = hidden_size/d_head;
956-
const int num_query = 96;
957-
961+
int n_head = hidden_size/d_head;
962+
int num_query = 96;
963+
if (ctx->minicpmv_version == 2) {
964+
hidden_size = 4096;
965+
n_head = hidden_size/d_head;
966+
num_query = 96;
967+
}
968+
else if (ctx->minicpmv_version == 3) {
969+
hidden_size = 3584;
970+
n_head = hidden_size/d_head;
971+
num_query = 64;
972+
}
958973
struct ggml_tensor * Q = ggml_add(ctx0, ggml_mul_mat(ctx0, model.mm_model_attn_q_w, q), model.mm_model_attn_q_b);
959974
Q = ggml_scale_inplace(ctx0, Q, 1.0f / sqrt((float)d_head));
960975
struct ggml_tensor * K = ggml_add(ctx0, ggml_mul_mat(ctx0, model.mm_model_attn_k_w, k), model.mm_model_attn_k_b);
@@ -1145,6 +1160,11 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {
11451160
new_clip->has_minicpmv_projector = gguf_get_val_bool(ctx, idx);
11461161
}
11471162

1163+
idx = gguf_find_key(ctx, KEY_MINICPMV_VERSION);
1164+
if (idx != -1) {
1165+
new_clip->minicpmv_version = gguf_get_val_i32(ctx, idx);
1166+
}
1167+
11481168
// GGML_ASSERT(new_clip->has_llava_projector); // see monatis/clip.cpp for image and/or text encoding for semantic search
11491169

11501170
GGML_ASSERT(new_clip->has_vision_encoder);
@@ -1908,7 +1928,14 @@ int clip_uhd_num_image_embeds_col(struct clip_ctx * ctx_clip){
19081928
bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, clip_image_f32_batch * res_imgs) {
19091929

19101930
if(clip_is_minicpmv(ctx)){
1911-
std::vector<std::vector<clip_image_u8 *>> imgs = uhd_slice_image(img);
1931+
int max_slice_nums = 9;
1932+
if (ctx->minicpmv_version == 2) {
1933+
max_slice_nums = 9;
1934+
}
1935+
else if (ctx->minicpmv_version == 3) {
1936+
max_slice_nums = 9;
1937+
}
1938+
std::vector<std::vector<clip_image_u8 *>> imgs = uhd_slice_image(img, max_slice_nums);
19121939
res_imgs->size = 0;
19131940
for (size_t i = 0; i < imgs.size(); ++i){
19141941
res_imgs->size += imgs[i].size();
@@ -2143,7 +2170,12 @@ int clip_n_patches(const struct clip_ctx * ctx) {
21432170
if (ctx->proj_type == PROJECTOR_TYPE_LDP || ctx->proj_type == PROJECTOR_TYPE_LDPV2) {
21442171
n_patches /= 4;
21452172
} else if (ctx->proj_type == PROJECTOR_TYPE_RESAMPLER) {
2146-
n_patches = 96;
2173+
if (ctx->minicpmv_version == 2) {
2174+
n_patches = 96;
2175+
}
2176+
else if (ctx->minicpmv_version == 3) {
2177+
n_patches = 64;
2178+
}
21472179
}
21482180

21492181
return n_patches;
@@ -2279,6 +2311,11 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
22792311
const int patch_size = hparams.patch_size;
22802312
const int num_patches = ((image_size_width / patch_size) * (image_size_height / patch_size));
22812313
const int num_positions = num_patches + (ctx->has_class_embedding ? 1 : 0);
2314+
if(ctx->load_image_size==nullptr){
2315+
ctx->load_image_size= clip_image_size_init();
2316+
}
2317+
const int pos_w = ctx->load_image_size->width/patch_size;
2318+
const int pos_h = ctx->load_image_size->height/patch_size;
22822319

22832320
{
22842321
struct ggml_tensor * inp_raw = ggml_graph_get_tensor(gf, "inp_raw");
@@ -2313,8 +2350,18 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
23132350
// -> https://huggingface.co/HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit/blob/d66538faeba44480d0bfaa42145eef26f9423199/modeling_siglip.py#L316
23142351
struct ggml_tensor * positions = ggml_graph_get_tensor(gf, "positions");
23152352
int* positions_data = (int*)malloc(ggml_nbytes(positions));
2316-
for (int i = 0; i < num_positions; i++) {
2317-
positions_data[i] = std::floor(70.0*i/num_positions);
2353+
int bucket_coords_h[70];
2354+
int bucket_coords_w[70];
2355+
for (int i = 0; i < pos_h; i++){
2356+
bucket_coords_h[i] = std::floor(70.0*i/pos_h);
2357+
}
2358+
for (int i = 0; i < pos_w; i++){
2359+
bucket_coords_w[i] = std::floor(70.0*i/pos_w);
2360+
}
2361+
for (int i = 0, id = 0; i < pos_h; i++){
2362+
for (int j = 0; j < pos_w; j++){
2363+
positions_data[id++] = bucket_coords_h[i]*70 + bucket_coords_w[j];
2364+
}
23182365
}
23192366
ggml_backend_tensor_set(positions, positions_data, 0, ggml_nbytes(positions));
23202367
free(positions_data);
@@ -2325,12 +2372,13 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
23252372
// -> https://huggingface.co/Qwen/Qwen-VL/tree/main
23262373
// -> https://huggingface.co/Qwen/Qwen-VL/blob/0547ed36a86561e2e42fecec8fd0c4f6953e33c4/visual.py#L23
23272374
struct ggml_tensor * pos_embed = ggml_graph_get_tensor(gf, "pos_embed");
2328-
if(ctx->load_image_size==nullptr){
2329-
ctx->load_image_size= clip_image_size_init();
2330-
}
2331-
int pos_w = ctx->load_image_size->width/patch_size;
2332-
int pos_h = ctx->load_image_size->height/patch_size;
23332375
int embed_dim = 4096;
2376+
if (ctx->minicpmv_version == 2) {
2377+
embed_dim = 4096;
2378+
}
2379+
else if (ctx->minicpmv_version == 3) {
2380+
embed_dim = 3584;
2381+
}
23342382
auto pos_embed_t = get_2d_sincos_pos_embed(embed_dim, std::make_pair(pos_w, pos_h));
23352383

23362384
float * pos_embed_data = (float *)malloc(ggml_nbytes(pos_embed));
@@ -2546,13 +2594,21 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
25462594
return ctx->vision_model.mm_3_b->ne[0];
25472595
}
25482596
if (ctx->proj_type == PROJECTOR_TYPE_RESAMPLER) {
2549-
return 4096;
2597+
if (ctx->minicpmv_version == 2) {
2598+
return 4096;
2599+
}
2600+
else if (ctx->minicpmv_version == 3) {
2601+
return 3584;
2602+
}
25502603
}
25512604

25522605
std::string proj_type = PROJECTOR_TYPE_NAMES[ctx->proj_type];
25532606
throw std::runtime_error(format("%s: don't support projector with: %s currently\n", __func__, proj_type.c_str()));
25542607
}
25552608

2556-
bool clip_is_minicpmv(const struct clip_ctx * ctx) {
2557-
return ctx->has_minicpmv_projector;
2609+
int clip_is_minicpmv(const struct clip_ctx * ctx) {
2610+
if (ctx->has_minicpmv_projector) {
2611+
return ctx->minicpmv_version;
2612+
}
2613+
return 0;
25582614
}

‎examples/llava/clip.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ CLIP_API bool clip_image_batch_encode(struct clip_ctx * ctx, int n_threads, cons
8686

8787
CLIP_API bool clip_model_quantize(const char * fname_inp, const char * fname_out, int itype);
8888

89-
CLIP_API bool clip_is_minicpmv(const struct clip_ctx * ctx);
89+
CLIP_API int clip_is_minicpmv(const struct clip_ctx * ctx);
9090

9191
#ifdef __cplusplus
9292
}

‎examples/llava/llava.cpp

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -256,7 +256,14 @@ static bool encode_image_with_clip(clip_ctx * ctx_clip, int n_threads, const cli
256256
load_image_size->width = img_res_v.data[i].nx;
257257
load_image_size->height = img_res_v.data[i].ny;
258258
clip_add_load_image_size(ctx_clip, load_image_size);
259-
const bool encoded = clip_image_encode(ctx_clip, n_threads, only_v2_5_reshape_by_patch(&img_res_v.data[i], patch_size), image_embd_v[i]);
259+
bool encoded = false;
260+
int has_minicpmv_projector = clip_is_minicpmv(ctx_clip);
261+
if (has_minicpmv_projector == 2) {
262+
encoded = clip_image_encode(ctx_clip, n_threads, only_v2_5_reshape_by_patch(&img_res_v.data[i], patch_size), image_embd_v[i]);
263+
}
264+
else if (has_minicpmv_projector == 3) {
265+
encoded = clip_image_encode(ctx_clip, n_threads, &img_res_v.data[i], image_embd_v[i]);
266+
}
260267
if (!encoded) {
261268
LOG_TEE("Unable to encode image - spatial_unpad - subimage %d of %d\n", (int) i+1, (int) img_res_v.size);
262269
return false;

‎examples/llava/minicpmv-cli.cpp

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,13 @@ static void process_image(struct llava_context * ctx_llava, struct llava_image_e
134134
std::string system_prompt;
135135
int idx = 0;
136136
int num_image_embeds = embeds->n_image_pos / clip_n_patches(ctx_llava->ctx_clip);
137-
system_prompt = "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n";
137+
int has_minicpmv_projector = clip_is_minicpmv(ctx_llava->ctx_clip);
138+
if (has_minicpmv_projector == 2) {
139+
system_prompt = "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n";
140+
}
141+
else if (has_minicpmv_projector == 3) {
142+
system_prompt = "<|im_start|>user\n";
143+
}
138144
LOG_TEE("%s: image token past: %d\n", __func__, n_past);
139145
eval_string(ctx_llava->ctx_llama, (system_prompt+"<image>").c_str(), params->n_batch, &n_past, false);
140146
process_eval_image_embed(ctx_llava, embeds, params->n_batch, &n_past, idx++);
@@ -210,10 +216,24 @@ static struct llava_context * minicpmv_init(gpt_params * params, const std::stri
210216

211217
static struct llama_sampling_context * llama_init(struct llava_context * ctx_llava, gpt_params * params, std::string prompt, int &n_past, bool is_first = false){
212218
std::string user_prompt = prompt;
213-
if (!is_first) user_prompt = "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n" + prompt;
219+
int has_minicpmv_projector = clip_is_minicpmv(ctx_llava->ctx_clip);
220+
if (!is_first) {
221+
if (has_minicpmv_projector == 2) {
222+
user_prompt = "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n" + prompt;
223+
}
224+
else if (has_minicpmv_projector == 3) {
225+
user_prompt = "<|im_start|>user\n" + prompt;
226+
}
227+
}
214228

215229
eval_string(ctx_llava->ctx_llama, user_prompt.c_str(), params->n_batch, &n_past, false);
216-
eval_string(ctx_llava->ctx_llama, "<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", params->n_batch, &n_past, false);
230+
if (has_minicpmv_projector == 2) {
231+
eval_string(ctx_llava->ctx_llama, "<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", params->n_batch, &n_past, false);
232+
}
233+
else if (has_minicpmv_projector == 3) {
234+
eval_string(ctx_llava->ctx_llama, "<|im_end|><|im_start|>assistant\n", params->n_batch, &n_past, false);
235+
}
236+
217237
// generate the response
218238

219239
LOG_TEE("\n");

0 commit comments

Comments
 (0)