| --- |
| library_name: transformers |
| base_model: |
| - zai-org/GLM-OCR |
| --- |
| |
| This tiny model is intended for debugging. It is randomly initialized using the configuration adapted from [zai-org/GLM-OCR](https://huggingface.co/zai-org/GLM-OCR). |
|
|
| | File path | Size | |
| |------|------| |
| | model.safetensors | 3.8MB | |
|
|
|
|
| ### Example usage: |
|
|
| ```python |
| import torch |
| from transformers import AutoModelForImageTextToText, AutoProcessor |
| |
| model_id = "tiny-random/glm-ocr" |
| model = AutoModelForImageTextToText.from_pretrained( |
| model_id, dtype=torch.bfloat16, device_map="cuda", |
| ) |
| processor = AutoProcessor.from_pretrained(model_id) |
| messages = [ |
| { |
| "role": "user", |
| "content": [ |
| { |
| "type": "image", |
| "image": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg", |
| }, |
| {"type": "text", "text": "Describe this image."}, |
| ], |
| } |
| ] |
| |
| # Preparation for inference |
| inputs = processor.apply_chat_template( |
| messages, |
| tokenize=True, |
| add_generation_prompt=True, |
| return_dict=True, |
| return_tensors="pt" |
| ).to(model.device) |
| inputs.pop("token_type_ids", None) |
| generated_ids = model.generate(**inputs, max_new_tokens=16) |
| output_text = processor.decode(generated_ids[0], skip_special_tokens=False) |
| print(output_text) |
| ``` |
|
|
| ### Codes to create this repo: |
|
|
| <details> |
| <summary>Click to expand</summary> |
|
|
| ```python |
| import json |
| from copy import deepcopy |
| from pathlib import Path |
| |
| import accelerate |
| import torch |
| import torch.nn as nn |
| from huggingface_hub import file_exists, hf_hub_download |
| from transformers import ( |
| AutoConfig, |
| AutoModelForCausalLM, |
| AutoProcessor, |
| GenerationConfig, |
| GlmOcrForConditionalGeneration, |
| set_seed, |
| ) |
| |
| source_model_id = "zai-org/GLM-OCR" |
| save_folder = "/tmp/tiny-random/glm-ocr" |
| |
| processor = AutoProcessor.from_pretrained( |
| source_model_id, trust_remote_code=True) |
| processor.save_pretrained(save_folder) |
| |
| with open(hf_hub_download(source_model_id, filename='config.json', repo_type='model'), 'r', encoding='utf-8') as f: |
| config_json: dict = json.load(f) |
| |
| config_json['text_config'].update({ |
| "head_dim": 32, |
| "hidden_size": 8, |
| "intermediate_size": 64, |
| "num_attention_heads": 8, |
| "num_hidden_layers": 2, |
| "num_key_value_heads": 4, |
| "rope_parameters": { |
| "rope_type": "default", |
| "mrope_section": [4, 4, 8], |
| "partial_rotary_factor": 1.0, |
| "rope_theta": 10000, |
| }, |
| }) |
| config_json['vision_config'].update({ |
| "hidden_size": 32, |
| "depth": 2, |
| "num_heads": 1, |
| "intermediate_size": 64, |
| "out_hidden_size": 8, |
| }) |
| with open(f"{save_folder}/config.json", "w", encoding='utf-8') as f: |
| json.dump(config_json, f, indent=2) |
| |
| config = AutoConfig.from_pretrained( |
| save_folder, |
| trust_remote_code=True, |
| ) |
| print(config) |
| torch.set_default_dtype(torch.bfloat16) |
| model = GlmOcrForConditionalGeneration(config) |
| torch.set_default_dtype(torch.float32) |
| |
| if file_exists(filename="generation_config.json", repo_id=source_model_id, repo_type='model'): |
| model.generation_config = GenerationConfig.from_pretrained( |
| source_model_id, trust_remote_code=True, |
| ) |
| model.generation_config.do_sample = True |
| print(model.generation_config) |
| |
| model = model.cpu() |
| set_seed(42) |
| n_params = sum(p.numel() for p in model.parameters()) |
| with torch.no_grad(): |
| for name, p in sorted(model.named_parameters()): |
| torch.nn.init.normal_(p, 0, 0.1) |
| print(name, p.shape, p.numel() / n_params * 100, '%') |
| # MTP |
| set_seed(42) |
| config = config.get_text_config() |
| model.model.language_model.layers.append(nn.ModuleDict(dict( |
| shared_head=nn.ModuleDict(dict( |
| norm=nn.RMSNorm(config.hidden_size), |
| head=deepcopy(model.model.language_model.embed_tokens), |
| )), |
| embed_tokens=deepcopy(model.model.language_model.embed_tokens), |
| eh_proj=nn.Linear(config.hidden_size * 2, |
| config.hidden_size, bias=False), |
| enorm=nn.RMSNorm(config.hidden_size), |
| hnorm=nn.RMSNorm(config.hidden_size), |
| input_layernorm=nn.RMSNorm(config.hidden_size), |
| post_mlp_layernorm=nn.RMSNorm(config.hidden_size), |
| post_attention_layernorm=nn.RMSNorm(config.hidden_size), |
| post_self_attn_layernorm=nn.RMSNorm(config.hidden_size), |
| self_attn=deepcopy(model.model.language_model.layers[1].self_attn), |
| mlp=deepcopy(model.model.language_model.layers[1].mlp), |
| ))) |
| # for i in range(1, len(model.model.language_model.layers)): |
| # model.model.language_model.layers[i].mlp.gate.e_score_correction_bias = torch.rand_like( |
| # model.model.language_model.layers[i].mlp.gate.e_score_correction_bias).float() |
| model.save_pretrained(save_folder) |
| print(model) |
| ``` |
|
|
| </details> |
|
|
| ### Printing the model: |
|
|
| <details><summary>Click to expand</summary> |
|
|
| ```text |
| GlmOcrForConditionalGeneration( |
| (model): GlmOcrModel( |
| (visual): GlmOcrVisionModel( |
| (patch_embed): GlmOcrVisionPatchEmbed( |
| (proj): Conv3d(3, 32, kernel_size=(2, 14, 14), stride=(2, 14, 14)) |
| ) |
| (rotary_pos_emb): GlmOcrVisionRotaryEmbedding() |
| (blocks): ModuleList( |
| (0-1): 2 x GlmOcrVisionBlock( |
| (norm1): GlmOcrRMSNorm((32,), eps=1e-05) |
| (norm2): GlmOcrRMSNorm((32,), eps=1e-05) |
| (attn): GlmOcrVisionAttention( |
| (qkv): Linear(in_features=32, out_features=96, bias=True) |
| (proj): Linear(in_features=32, out_features=32, bias=True) |
| (q_norm): GlmOcrRMSNorm((32,), eps=1e-05) |
| (k_norm): GlmOcrRMSNorm((32,), eps=1e-05) |
| ) |
| (mlp): GlmOcrVisionMlp( |
| (gate_proj): Linear(in_features=32, out_features=64, bias=True) |
| (up_proj): Linear(in_features=32, out_features=64, bias=True) |
| (down_proj): Linear(in_features=64, out_features=32, bias=True) |
| (act_fn): SiLUActivation() |
| ) |
| ) |
| ) |
| (merger): GlmOcrVisionPatchMerger( |
| (proj): Linear(in_features=8, out_features=8, bias=False) |
| (post_projection_norm): LayerNorm((8,), eps=1e-05, elementwise_affine=True) |
| (gate_proj): Linear(in_features=8, out_features=24, bias=False) |
| (up_proj): Linear(in_features=8, out_features=24, bias=False) |
| (down_proj): Linear(in_features=24, out_features=8, bias=False) |
| (act1): GELU(approximate='none') |
| (act_fn): SiLUActivation() |
| ) |
| (downsample): Conv2d(32, 8, kernel_size=(2, 2), stride=(2, 2)) |
| (post_layernorm): GlmOcrRMSNorm((32,), eps=1e-05) |
| ) |
| (language_model): GlmOcrTextModel( |
| (embed_tokens): Embedding(59392, 8, padding_idx=59246) |
| (layers): ModuleList( |
| (0-1): 2 x GlmOcrTextDecoderLayer( |
| (self_attn): GlmOcrTextAttention( |
| (q_proj): Linear(in_features=8, out_features=256, bias=False) |
| (k_proj): Linear(in_features=8, out_features=128, bias=False) |
| (v_proj): Linear(in_features=8, out_features=128, bias=False) |
| (o_proj): Linear(in_features=256, out_features=8, bias=False) |
| ) |
| (mlp): GlmOcrTextMLP( |
| (gate_up_proj): Linear(in_features=8, out_features=128, bias=False) |
| (down_proj): Linear(in_features=64, out_features=8, bias=False) |
| (activation_fn): SiLUActivation() |
| ) |
| (input_layernorm): GlmOcrRMSNorm((8,), eps=1e-05) |
| (post_attention_layernorm): GlmOcrRMSNorm((8,), eps=1e-05) |
| (post_self_attn_layernorm): GlmOcrRMSNorm((8,), eps=1e-05) |
| (post_mlp_layernorm): GlmOcrRMSNorm((8,), eps=1e-05) |
| ) |
| (2): ModuleDict( |
| (shared_head): ModuleDict( |
| (norm): RMSNorm((8,), eps=None, elementwise_affine=True) |
| (head): Embedding(59392, 8, padding_idx=59246) |
| ) |
| (embed_tokens): Embedding(59392, 8, padding_idx=59246) |
| (eh_proj): Linear(in_features=16, out_features=8, bias=False) |
| (enorm): RMSNorm((8,), eps=None, elementwise_affine=True) |
| (hnorm): RMSNorm((8,), eps=None, elementwise_affine=True) |
| (input_layernorm): RMSNorm((8,), eps=None, elementwise_affine=True) |
| (post_mlp_layernorm): RMSNorm((8,), eps=None, elementwise_affine=True) |
| (post_attention_layernorm): RMSNorm((8,), eps=None, elementwise_affine=True) |
| (post_self_attn_layernorm): RMSNorm((8,), eps=None, elementwise_affine=True) |
| (self_attn): GlmOcrTextAttention( |
| (q_proj): Linear(in_features=8, out_features=256, bias=False) |
| (k_proj): Linear(in_features=8, out_features=128, bias=False) |
| (v_proj): Linear(in_features=8, out_features=128, bias=False) |
| (o_proj): Linear(in_features=256, out_features=8, bias=False) |
| ) |
| (mlp): GlmOcrTextMLP( |
| (gate_up_proj): Linear(in_features=8, out_features=128, bias=False) |
| (down_proj): Linear(in_features=64, out_features=8, bias=False) |
| (activation_fn): SiLUActivation() |
| ) |
| ) |
| ) |
| (norm): GlmOcrRMSNorm((8,), eps=1e-05) |
| (rotary_emb): GlmOcrTextRotaryEmbedding() |
| ) |
| ) |
| (lm_head): Linear(in_features=8, out_features=59392, bias=False) |
| ) |
| ``` |
|
|
| </details> |
