| { |
| "schema_version": 1, |
| "peft_version": "0.19.2.dev0", |
| "baselines": { |
| "metamathqa": { |
| "experiment_name": "full-finetuning/llama-3.2-3B-lr_0.00001", |
| "test accuracy": 0.49507202426080366, |
| "forgetting": 0.5911064147949219, |
| "peak_memory_bytes": 38549848064, |
| "train_time_sec": 2154.846133010811, |
| "num_trainable_params": 3212749824, |
| "adapter_file_size_bytes": 6425499648, |
| "num_runs": 1 |
| }, |
| "image-gen": { |
| "experiment_name": "full-finetuning/flux2-klein-default", |
| "test dino_similarity": 0.5767437418301901, |
| "drift": null, |
| "peak_memory_bytes": 39873150976, |
| "train_time_sec": 292.64274606599975, |
| "num_trainable_params": 3875544576, |
| "adapter_file_size_bytes": 7751089152, |
| "num_runs": 1 |
| } |
| }, |
| "methods": { |
| "ADALORA": { |
| "config_class": "AdaLoraConfig", |
| "model_class": "AdaLoraModel", |
| "features": { |
| "category": { |
| "value": "adapter", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": { |
| "Linear": true, |
| "Embedding": false, |
| "Conv1d": false, |
| "Conv2d": true, |
| "Conv3d": false, |
| "LayerNorm": false, |
| "MultiheadAttention": false, |
| "Conv1D (transformers)": true, |
| "nn.Parameter": true |
| }, |
| "source": "probe", |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" |
| }, |
| "quantization_backends": { |
| "value": [ |
| "bnb_4bit", |
| "bnb_8bit", |
| "gptq" |
| ], |
| "source": "file_check" |
| }, |
| "multiple_adapters": { |
| "value": false, |
| "source": "probe", |
| "note": "ValueError: AdaLoraModel supports only 1 trainable adapter. When using multiple adapters, set inference_mode to True for all adapters except the one you want to train." |
| }, |
| "mixed_adapter_batches": { |
| "value": true, |
| "source": "introspection" |
| }, |
| "merging": { |
| "value": true, |
| "source": "probe" |
| }, |
| "peft_mixed_model": { |
| "value": true, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": false, |
| "source": "probe" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection", |
| "note": "explicitly disabled for this method" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": true |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [ |
| "alpha_pattern", |
| "layer_replication", |
| "rank_pattern", |
| "use_dora", |
| "use_rslora" |
| ], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": "https://openreview.net/forum?id=lq62uWRJjiY", |
| "source": "file_check", |
| "note": "from the model class docstring" |
| } |
| }, |
| "description": "AdaLoRA is a method for optimizing the number of trainable parameters to assign to weight matrices and layers, unlike LoRA, which distributes parameters evenly across all modules. More parameters are budgeted for important weight matrices and layers while less important ones receive fewer parameters.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/adalora", |
| "paper_url": "https://openreview.net/forum?id=lq62uWRJjiY", |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "adalora/llama-3.2-3B-rank32", |
| "test accuracy": 0.38362395754359363, |
| "forgetting": 0.0754537582397461, |
| "peak_memory_bytes": 22796042240, |
| "train_time_sec": 1097.5522341161268, |
| "num_trainable_params": 18353664, |
| "adapter_file_size_bytes": 35131056, |
| "num_runs": 1 |
| }, |
| "image-gen": { |
| "experiment_name": "adalora/flux2-klein-default", |
| "test dino_similarity": 0.6824255585670471, |
| "drift": 0.26671119034290314, |
| "peak_memory_bytes": 11630804992, |
| "train_time_sec": 718.568363055, |
| "num_trainable_params": 38341120, |
| "adapter_file_size_bytes": 68503160, |
| "num_runs": 1 |
| } |
| } |
| }, |
| "ADAMSS": { |
| "config_class": "AdamssConfig", |
| "model_class": "AdamssModel", |
| "features": { |
| "category": { |
| "value": "adapter", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": { |
| "Linear": true, |
| "Embedding": false, |
| "Conv1d": false, |
| "Conv2d": false, |
| "Conv3d": false, |
| "LayerNorm": false, |
| "MultiheadAttention": false, |
| "Conv1D (transformers)": false, |
| "nn.Parameter": false |
| }, |
| "source": "probe", |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" |
| }, |
| "quantization_backends": { |
| "value": [], |
| "source": "file_check" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "merging": { |
| "value": true, |
| "source": "probe" |
| }, |
| "peft_mixed_model": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": false, |
| "source": "probe" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": "https://openreview.net/forum?id=8ZdWmpYxT0", |
| "source": "file_check", |
| "note": "from the docs intro" |
| } |
| }, |
| "description": "AdaMSS (AdaMSS: Adaptive Multi-Subspace Approach for Parameter-Efficient Fine-Tuning) is a parameter-efficient fine-tuning method that decomposes weight matrices using SVD and clusters the decomposed space into multiple trainable subspaces. Each subspace learns independent low-rank updates while the original weights remain frozen.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/adamss", |
| "paper_url": "https://openreview.net/forum?id=8ZdWmpYxT0", |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "adamss/llama-3.2-3B-rank32", |
| "test accuracy": 0.3366186504927976, |
| "forgetting": 0.1917562484741211, |
| "peak_memory_bytes": 20323500032, |
| "train_time_sec": 4596.08915475203, |
| "num_trainable_params": 293888, |
| "adapter_file_size_bytes": 70459192, |
| "num_runs": 1 |
| } |
| } |
| }, |
| "ADAPTION_PROMPT": { |
| "config_class": "AdaptionPromptConfig", |
| "model_class": "AdaptionPromptModel", |
| "features": { |
| "category": { |
| "value": "other", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": "not_applicable", |
| "source": "introspection", |
| "note": "method does not wrap target layers" |
| }, |
| "quantization_backends": { |
| "value": "unknown", |
| "source": "introspection", |
| "note": "no known quantization signal" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection", |
| "note": "only supported by layer-wrapping adapter methods" |
| }, |
| "merging": { |
| "value": false, |
| "source": "introspection", |
| "note": "method does not implement merging" |
| }, |
| "peft_mixed_model": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": false, |
| "source": "introspection", |
| "note": "method does not wrap target layers" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": false, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": "https://huggingface.co/papers/2303.16199.", |
| "source": "file_check", |
| "note": "from the model class docstring" |
| } |
| }, |
| "description": "Llama-Adapter is a PEFT method specifically designed for turning Llama into an instruction-following model. The Llama model is frozen and only a set of adaptation prompts prefixed to the input instruction tokens are learned.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/llama_adapter", |
| "paper_url": "https://huggingface.co/papers/2303.16199.", |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "adaptionprompt/llama-3.2-3B-lr_0.0005", |
| "test accuracy": 0.21228203184230476, |
| "forgetting": -0.9543380737304688, |
| "peak_memory_bytes": 22445817856, |
| "train_time_sec": 1180.5937879550038, |
| "num_trainable_params": 8601628, |
| "adapter_file_size_bytes": 17210384, |
| "num_runs": 1 |
| } |
| } |
| }, |
| "BEFT": { |
| "config_class": "BeftConfig", |
| "model_class": "BeftModel", |
| "features": { |
| "category": { |
| "value": "adapter", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": { |
| "Linear": true, |
| "Embedding": false, |
| "Conv1d": false, |
| "Conv2d": false, |
| "Conv3d": false, |
| "LayerNorm": false, |
| "MultiheadAttention": false, |
| "Conv1D (transformers)": false, |
| "nn.Parameter": false |
| }, |
| "source": "probe", |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" |
| }, |
| "quantization_backends": { |
| "value": [], |
| "source": "file_check" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "merging": { |
| "value": true, |
| "source": "probe" |
| }, |
| "peft_mixed_model": { |
| "value": true, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": false, |
| "source": "probe" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": "https://arxiv.org/abs/2509.15974", |
| "source": "file_check", |
| "note": "from the docs intro" |
| } |
| }, |
| "description": "BEFT is a parameter efficient fine-tuning algorithm (PEFT) that only fine-tunes the added bias terms of value projections from pretrained transformer models. BEFT demonstrates that fine-tuning the added bias terms of value projections from pretrained transformers generally leads to a higher downstream performance in low-data regimes than fine-tuning the added bias terms of query/key projections.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/beft", |
| "paper_url": "https://arxiv.org/abs/2509.15974", |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "beft/llama-3.2-3B-target-v_proj", |
| "test accuracy": 0.3290371493555724, |
| "forgetting": -0.029061317443847656, |
| "peak_memory_bytes": 20174602240, |
| "train_time_sec": 895.6792339479998, |
| "num_trainable_params": 28672, |
| "adapter_file_size_bytes": 118192, |
| "num_runs": 1 |
| } |
| } |
| }, |
| "BOFT": { |
| "config_class": "BOFTConfig", |
| "model_class": "BOFTModel", |
| "features": { |
| "category": { |
| "value": "adapter", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": { |
| "Linear": true, |
| "Embedding": false, |
| "Conv1d": false, |
| "Conv2d": true, |
| "Conv3d": false, |
| "LayerNorm": false, |
| "MultiheadAttention": false, |
| "Conv1D (transformers)": false, |
| "nn.Parameter": false |
| }, |
| "source": "probe", |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" |
| }, |
| "quantization_backends": { |
| "value": [ |
| "aqlm", |
| "awq", |
| "bnb_4bit", |
| "bnb_8bit", |
| "eetq", |
| "gptq", |
| "hqq", |
| "inc", |
| "torchao" |
| ], |
| "source": "file_check" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "merging": { |
| "value": true, |
| "source": "probe" |
| }, |
| "peft_mixed_model": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": false, |
| "source": "probe" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": null, |
| "source": "file_check", |
| "note": "no unambiguous paper link in the docs intro or the config/model class docstrings" |
| } |
| }, |
| "description": "Orthogonal Butterfly (BOFT) is a generic method designed for finetuning foundation models. It improves the parameter efficiency of the finetuning paradigm -- Orthogonal Finetuning (OFT), by taking inspiration from Cooley-Tukey fast Fourier transform, showing favorable results across finetuning different foundation models, including large vision transformers, large language models and text-to-image diffusion models.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/boft", |
| "paper_url": null, |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "boft/llama-3.2-3B-default", |
| "test accuracy": 0.36239575435936316, |
| "forgetting": 0.3084230422973633, |
| "peak_memory_bytes": 24410849280, |
| "train_time_sec": 7190.453845723998, |
| "num_trainable_params": 802816, |
| "adapter_file_size_bytes": 3225360, |
| "num_runs": 1 |
| }, |
| "image-gen": { |
| "experiment_name": "boft/flux2-klein-default", |
| "test dino_similarity": 0.6449412306149801, |
| "drift": 0.2069137692451477, |
| "peak_memory_bytes": 17368612864, |
| "train_time_sec": 2524.314185989001, |
| "num_trainable_params": 2119680, |
| "adapter_file_size_bytes": 8500008, |
| "num_runs": 1 |
| } |
| } |
| }, |
| "C3A": { |
| "config_class": "C3AConfig", |
| "model_class": "C3AModel", |
| "features": { |
| "category": { |
| "value": "adapter", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": { |
| "Linear": true, |
| "Embedding": false, |
| "Conv1d": false, |
| "Conv2d": false, |
| "Conv3d": false, |
| "LayerNorm": false, |
| "MultiheadAttention": false, |
| "Conv1D (transformers)": false, |
| "nn.Parameter": false |
| }, |
| "source": "probe", |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" |
| }, |
| "quantization_backends": { |
| "value": [], |
| "source": "file_check" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "merging": { |
| "value": true, |
| "source": "probe" |
| }, |
| "peft_mixed_model": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": true, |
| "source": "probe" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": "https://huggingface.co/papers/2407.19342", |
| "source": "file_check", |
| "note": "from the docs intro" |
| } |
| }, |
| "description": "C3A is a parameter-efficient fine-tuning technique that leverages Circular Convolution to achieve high rank adaptation within reasonable resource limits.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/c3a", |
| "paper_url": "https://huggingface.co/papers/2407.19342", |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "c3a/llama-3.2-3B-default", |
| "test accuracy": 0.5026535253980288, |
| "forgetting": 0.4295167922973633, |
| "peak_memory_bytes": 22166896640, |
| "train_time_sec": 1153.291345822101, |
| "num_trainable_params": 5505024, |
| "adapter_file_size_bytes": 22027512, |
| "num_runs": 1 |
| }, |
| "image-gen": { |
| "experiment_name": "c3a/flux2-klein-default", |
| "test dino_similarity": 0.6366289258003235, |
| "drift": 0.3616682291030884, |
| "peak_memory_bytes": 11586764800, |
| "train_time_sec": 1141.61083238, |
| "num_trainable_params": 11427840, |
| "adapter_file_size_bytes": 45722624, |
| "num_runs": 1 |
| } |
| } |
| }, |
| "CARTRIDGE": { |
| "config_class": "CartridgeConfig", |
| "model_class": "CartridgeEncoder", |
| "features": { |
| "category": { |
| "value": "prompt_learning", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": "not_applicable", |
| "source": "introspection", |
| "note": "method does not wrap target layers" |
| }, |
| "quantization_backends": { |
| "value": "not_applicable", |
| "source": "introspection", |
| "note": "prompt learning does not wrap target layers and generally works regardless of quantization" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection", |
| "note": "only supported by layer-wrapping adapter methods" |
| }, |
| "merging": { |
| "value": false, |
| "source": "introspection", |
| "note": "virtual tokens cannot be merged into base weights" |
| }, |
| "peft_mixed_model": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": false, |
| "source": "introspection", |
| "note": "method does not wrap target layers" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": null, |
| "source": "file_check", |
| "note": "no unambiguous paper link in the docs intro or the config/model class docstrings" |
| } |
| }, |
| "description": "Cartridges are a prompt-learning method that stores a compressed long-context representation as a parameterized KV-cache", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/cartridges", |
| "paper_url": null, |
| "benchmarks": {} |
| }, |
| "CPT": { |
| "config_class": "CPTConfig", |
| "model_class": "CPTEmbedding", |
| "features": { |
| "category": { |
| "value": "prompt_learning", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": "not_applicable", |
| "source": "introspection", |
| "note": "method does not wrap target layers" |
| }, |
| "quantization_backends": { |
| "value": "not_applicable", |
| "source": "introspection", |
| "note": "prompt learning does not wrap target layers and generally works regardless of quantization" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection", |
| "note": "only supported by layer-wrapping adapter methods" |
| }, |
| "merging": { |
| "value": false, |
| "source": "introspection", |
| "note": "virtual tokens cannot be merged into base weights" |
| }, |
| "peft_mixed_model": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": false, |
| "source": "introspection", |
| "note": "method does not wrap target layers" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": "https://huggingface.co/papers/2410.17222", |
| "source": "file_check", |
| "note": "from the docs intro" |
| } |
| }, |
| "description": "CPT combines In-Context Learning (ICL), Prompt Tuning (PT), and adversarial optimization to improve few-shot learning by refining context embeddings. CPT updates the context tokens by optimizing both the context and the training examples, encapsulating them into a novel loss design that minimizes overfitting, enables more effective optimization, and drives significant improvements in classification tasks.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/cpt", |
| "paper_url": "https://huggingface.co/papers/2410.17222", |
| "benchmarks": {} |
| }, |
| "DELORA": { |
| "config_class": "DeloraConfig", |
| "model_class": "DeloraModel", |
| "features": { |
| "category": { |
| "value": "adapter", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": { |
| "Linear": true, |
| "Embedding": false, |
| "Conv1d": false, |
| "Conv2d": false, |
| "Conv3d": false, |
| "LayerNorm": false, |
| "MultiheadAttention": false, |
| "Conv1D (transformers)": false, |
| "nn.Parameter": false |
| }, |
| "source": "probe", |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" |
| }, |
| "quantization_backends": { |
| "value": [], |
| "source": "file_check" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "merging": { |
| "value": true, |
| "source": "probe" |
| }, |
| "peft_mixed_model": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": true, |
| "source": "probe" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [ |
| "rank_pattern" |
| ], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": "https://huggingface.co/papers/2503.18225", |
| "source": "file_check", |
| "note": "from the docs intro" |
| } |
| }, |
| "description": "DeLoRA is a parameter-efficient fine-tuning technique that implicitly maintains a Frobenius boundary with respect to the pretrained weights by normalizing and scaling learnable low-rank matrices. This effectively decouples the learning of directions (BA term) and magnitude (boundary term) of the weight updates, avoiding catastrophic shifts in the adapted weights and enhancing robustness to hyperparameter choices.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/delora", |
| "paper_url": "https://huggingface.co/papers/2503.18225", |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "delora/llama-3.2-3B-rank32", |
| "test accuracy": 0.5026535253980288, |
| "forgetting": 0.515233039855957, |
| "peak_memory_bytes": 22353543168, |
| "train_time_sec": 1058.4298028770136, |
| "num_trainable_params": 9175096, |
| "adapter_file_size_bytes": 37417520, |
| "num_runs": 1 |
| }, |
| "image-gen": { |
| "experiment_name": "delora/flux2-klein-default", |
| "test dino_similarity": 0.5392374992370605, |
| "drift": 0.46095240488648415, |
| "peak_memory_bytes": 10563354624, |
| "train_time_sec": 631.5103333749994, |
| "num_trainable_params": 38338640, |
| "adapter_file_size_bytes": 154627416, |
| "num_runs": 1 |
| } |
| } |
| }, |
| "FOURIERFT": { |
| "config_class": "FourierFTConfig", |
| "model_class": "FourierFTModel", |
| "features": { |
| "category": { |
| "value": "adapter", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": { |
| "Linear": true, |
| "Embedding": false, |
| "Conv1d": false, |
| "Conv2d": false, |
| "Conv3d": false, |
| "LayerNorm": false, |
| "MultiheadAttention": false, |
| "Conv1D (transformers)": true, |
| "nn.Parameter": false |
| }, |
| "source": "probe", |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" |
| }, |
| "quantization_backends": { |
| "value": [], |
| "source": "file_check" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "merging": { |
| "value": true, |
| "source": "probe" |
| }, |
| "peft_mixed_model": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": true, |
| "source": "probe" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": "https://huggingface.co/papers/2405.03003", |
| "source": "file_check", |
| "note": "from the docs intro" |
| } |
| }, |
| "description": "FourierFT is a parameter-efficient fine-tuning technique that leverages Discrete Fourier Transform to compress the model's tunable weights. This method outperforms LoRA in the GLUE benchmark and common ViT classification tasks using much less parameters.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/fourierft", |
| "paper_url": "https://huggingface.co/papers/2405.03003", |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "fourierft/llama-3.2-3B-n_frequency-5000", |
| "test accuracy": 0.11675511751326763, |
| "forgetting": 0.40204334259033203, |
| "peak_memory_bytes": 23674748928, |
| "train_time_sec": 1735.5117494451115, |
| "num_trainable_params": 280000, |
| "adapter_file_size_bytes": 1127472, |
| "num_runs": 2 |
| }, |
| "image-gen": { |
| "experiment_name": "fourierft/flux2-klein-default", |
| "test dino_similarity": 0.3470671971638997, |
| "drift": 0.020928651094436646, |
| "peak_memory_bytes": 12935233536, |
| "train_time_sec": 2988.2643262210013, |
| "num_trainable_params": 160000, |
| "adapter_file_size_bytes": 651152, |
| "num_runs": 1 |
| } |
| } |
| }, |
| "GRALORA": { |
| "config_class": "GraloraConfig", |
| "model_class": "GraloraModel", |
| "features": { |
| "category": { |
| "value": "adapter", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": { |
| "Linear": true, |
| "Embedding": false, |
| "Conv1d": false, |
| "Conv2d": false, |
| "Conv3d": false, |
| "LayerNorm": false, |
| "MultiheadAttention": false, |
| "Conv1D (transformers)": true, |
| "nn.Parameter": false |
| }, |
| "source": "probe", |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" |
| }, |
| "quantization_backends": { |
| "value": [], |
| "source": "file_check" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "merging": { |
| "value": true, |
| "source": "probe" |
| }, |
| "peft_mixed_model": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": true, |
| "source": "probe" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": "https://huggingface.co/papers/2505.20355", |
| "source": "file_check", |
| "note": "from the docs intro" |
| } |
| }, |
| "description": "**Granular Low-Rank Adaptation (GraLoRA)** is a PEFT method designed to enhance the **expressivity** of low-rank adaptation while improving **robustness to outlier** activations, based on insights from well-known issues in quantization.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/gralora", |
| "paper_url": "https://huggingface.co/papers/2505.20355", |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "gralora/llama-3.2-3B-rank32", |
| "test accuracy": 0.5011372251705838, |
| "forgetting": 0.49779701232910156, |
| "peak_memory_bytes": 22307405824, |
| "train_time_sec": 1078.0007609191234, |
| "num_trainable_params": 9175040, |
| "adapter_file_size_bytes": 36714960, |
| "num_runs": 1 |
| }, |
| "image-gen": { |
| "experiment_name": "gralora/flux2-klein-default", |
| "test dino_similarity": 0.7006240288416544, |
| "drift": 0.2746447026729584, |
| "peak_memory_bytes": 10729029632, |
| "train_time_sec": 727.7293240400013, |
| "num_trainable_params": 38338560, |
| "adapter_file_size_bytes": 153376720, |
| "num_runs": 1 |
| } |
| } |
| }, |
| "HIRA": { |
| "config_class": "HiraConfig", |
| "model_class": "HiraModel", |
| "features": { |
| "category": { |
| "value": "adapter", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": { |
| "Linear": true, |
| "Embedding": true, |
| "Conv1d": true, |
| "Conv2d": true, |
| "Conv3d": true, |
| "LayerNorm": false, |
| "MultiheadAttention": false, |
| "Conv1D (transformers)": true, |
| "nn.Parameter": false |
| }, |
| "source": "probe", |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" |
| }, |
| "quantization_backends": { |
| "value": [ |
| "bnb_4bit", |
| "bnb_8bit" |
| ], |
| "source": "file_check" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "merging": { |
| "value": true, |
| "source": "probe" |
| }, |
| "peft_mixed_model": { |
| "value": true, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": false, |
| "source": "probe" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [ |
| "rank_pattern" |
| ], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": null, |
| "source": "file_check", |
| "note": "no unambiguous paper link in the docs intro or the config/model class docstrings" |
| } |
| }, |
| "description": "High-Rank Adaptation (HiRA) is a PEFT method that extends the LoRA approach by applying an element-wise modulation on the original weight matrix. Instead of adding a low-rank update directly, HiRA computes:", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/hira", |
| "paper_url": null, |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "hira/llama-3.2-3B-rank32-lr4.5e-3", |
| "test accuracy": 0.5072024260803639, |
| "forgetting": 0.4813060760498047, |
| "peak_memory_bytes": 23599251456, |
| "train_time_sec": 1301.0097959969953, |
| "num_trainable_params": 9175040, |
| "adapter_file_size_bytes": 36714432, |
| "num_runs": 1 |
| } |
| } |
| }, |
| "HRA": { |
| "config_class": "HRAConfig", |
| "model_class": "HRAModel", |
| "features": { |
| "category": { |
| "value": "adapter", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": { |
| "Linear": true, |
| "Embedding": false, |
| "Conv1d": false, |
| "Conv2d": true, |
| "Conv3d": false, |
| "LayerNorm": false, |
| "MultiheadAttention": false, |
| "Conv1D (transformers)": false, |
| "nn.Parameter": false |
| }, |
| "source": "probe", |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" |
| }, |
| "quantization_backends": { |
| "value": [], |
| "source": "file_check" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "merging": { |
| "value": true, |
| "source": "probe" |
| }, |
| "peft_mixed_model": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": false, |
| "source": "probe" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": "https://huggingface.co/papers/2405.17484", |
| "source": "file_check", |
| "note": "from the docs intro" |
| } |
| }, |
| "description": "HRA is a simple but effective adapter-based fine-tuning method by leveraging Householder reflections. This method harnesses the advantages of both strategies, reducing parameters and computation costs while penalizing the loss of pre-training knowledge.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/hra", |
| "paper_url": "https://huggingface.co/papers/2405.17484", |
| "benchmarks": { |
| "image-gen": { |
| "experiment_name": "hra/flux2-klein-default", |
| "test dino_similarity": 0.6441149314244589, |
| "drift": 0.2597343921661377, |
| "peak_memory_bytes": 47359983616, |
| "train_time_sec": 9824.567679766005, |
| "num_trainable_params": 9830400, |
| "adapter_file_size_bytes": 39332240, |
| "num_runs": 1 |
| } |
| } |
| }, |
| "IA3": { |
| "config_class": "IA3Config", |
| "model_class": "IA3Model", |
| "features": { |
| "category": { |
| "value": "adapter", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": { |
| "Linear": true, |
| "Embedding": false, |
| "Conv1d": false, |
| "Conv2d": true, |
| "Conv3d": true, |
| "LayerNorm": false, |
| "MultiheadAttention": false, |
| "Conv1D (transformers)": true, |
| "nn.Parameter": false |
| }, |
| "source": "probe", |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" |
| }, |
| "quantization_backends": { |
| "value": [ |
| "bnb_4bit", |
| "bnb_8bit" |
| ], |
| "source": "file_check" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "merging": { |
| "value": true, |
| "source": "probe" |
| }, |
| "peft_mixed_model": { |
| "value": true, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": false, |
| "source": "probe" |
| }, |
| "add_weighted_adapter": { |
| "value": true, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": "https://huggingface.co/papers/2205.05638", |
| "source": "file_check", |
| "note": "from the model class docstring" |
| } |
| }, |
| "description": "Infused Adapter by Inhibiting and Amplifying Inner Activations, or IA3, is a method that adds three learned vectors to rescale the keys and values of the self-attention and encoder-decoder attention layers, and the intermediate activation of the position-wise feed-forward network.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/ia3", |
| "paper_url": "https://huggingface.co/papers/2205.05638", |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "ia3/llama-3.2-3B-lr_0.001", |
| "test accuracy": 0.41243366186504926, |
| "forgetting": 0.2050151824951172, |
| "peak_memory_bytes": 23240638464, |
| "train_time_sec": 969.7345388169051, |
| "num_trainable_params": 286720, |
| "adapter_file_size_bytes": 1157064, |
| "num_runs": 2 |
| }, |
| "image-gen": { |
| "experiment_name": "ia3/flux2-klein-default", |
| "test dino_similarity": 0.6476706067721049, |
| "drift": 0.22948315739631653, |
| "peak_memory_bytes": 9586081792, |
| "train_time_sec": 481.21032093000395, |
| "num_trainable_params": 307200, |
| "adapter_file_size_bytes": 1239104, |
| "num_runs": 1 |
| } |
| } |
| }, |
| "LILY": { |
| "config_class": "LilyConfig", |
| "model_class": "LilyModel", |
| "features": { |
| "category": { |
| "value": "adapter", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": { |
| "Linear": true, |
| "Embedding": false, |
| "Conv1d": false, |
| "Conv2d": false, |
| "Conv3d": false, |
| "LayerNorm": false, |
| "MultiheadAttention": false, |
| "Conv1D (transformers)": false, |
| "nn.Parameter": false |
| }, |
| "source": "probe", |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" |
| }, |
| "quantization_backends": { |
| "value": [], |
| "source": "file_check" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "merging": { |
| "value": false, |
| "source": "probe", |
| "note": "NotImplementedError: This method is not supported for Lily." |
| }, |
| "peft_mixed_model": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": false, |
| "source": "probe" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": "https://huggingface.co/papers/2407.09946", |
| "source": "file_check", |
| "note": "from the docs intro" |
| } |
| }, |
| "description": "Lily is a parameter-efficient fine-tuning technique that introduces cross-layer weight sharing for adapter matrices. Instead of learning an independent AB pair per layer as in LoRA, Lily uses **locally shared A adapters** (each A is shared across a block of `stride_A` consecutive layers) and **globally shared B experts** (a small pool of `num_B` B adapters is shared across all layers).", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/lily", |
| "paper_url": "https://huggingface.co/papers/2407.09946", |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "lily/llama-3.2-3B-rank140-mlp-a2-b2-s8.0", |
| "test accuracy": 0.5489006823351024, |
| "forgetting": -0.09576702117919922, |
| "peak_memory_bytes": 25620905984, |
| "train_time_sec": 1277.8012849430233, |
| "num_trainable_params": 9485280, |
| "adapter_file_size_bytes": 835056616, |
| "num_runs": 2 |
| }, |
| "image-gen": { |
| "experiment_name": "lily/flux2-klein-default", |
| "test dino_similarity": 0.648700992266337, |
| "drift": 0.2646396607160568, |
| "peak_memory_bytes": 10624172032, |
| "train_time_sec": 672.2888688070016, |
| "num_trainable_params": 14554112, |
| "adapter_file_size_bytes": 267441872, |
| "num_runs": 1 |
| } |
| } |
| }, |
| "LN_TUNING": { |
| "config_class": "LNTuningConfig", |
| "model_class": "LNTuningModel", |
| "features": { |
| "category": { |
| "value": "adapter", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": { |
| "Linear": true, |
| "Embedding": true, |
| "Conv1d": true, |
| "Conv2d": true, |
| "Conv3d": true, |
| "LayerNorm": true, |
| "MultiheadAttention": true, |
| "Conv1D (transformers)": true, |
| "nn.Parameter": false |
| }, |
| "source": "probe", |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" |
| }, |
| "quantization_backends": { |
| "value": [], |
| "source": "file_check" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "merging": { |
| "value": true, |
| "source": "probe" |
| }, |
| "peft_mixed_model": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": false, |
| "source": "probe" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": "https://huggingface.co/papers/2312.11420", |
| "source": "file_check", |
| "note": "from the docs intro" |
| } |
| }, |
| "description": "LayerNorm Tuning (LN Tuning) is a PEFT method that only fine-tunes the parameters of the LayerNorm layers in a model.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/layernorm_tuning", |
| "paper_url": "https://huggingface.co/papers/2312.11420", |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "ln_tuning/llama-3.2-3B-default", |
| "test accuracy": 0.0, |
| "forgetting": -0.032855987548828125, |
| "peak_memory_bytes": 21179138048, |
| "train_time_sec": 933.4248713181005, |
| "num_trainable_params": 175104, |
| "adapter_file_size_bytes": 358288, |
| "num_runs": 1 |
| }, |
| "image-gen": { |
| "experiment_name": "ln_tuning/flux2-klein-default", |
| "test dino_similarity": 0.34151384234428406, |
| "drift": 0.0, |
| "peak_memory_bytes": 9609150464, |
| "train_time_sec": 458.36126944199714, |
| "num_trainable_params": 6400, |
| "adapter_file_size_bytes": 19856, |
| "num_runs": 1 |
| } |
| } |
| }, |
| "LOHA": { |
| "config_class": "LoHaConfig", |
| "model_class": "LoHaModel", |
| "features": { |
| "category": { |
| "value": "adapter", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": { |
| "Linear": true, |
| "Embedding": false, |
| "Conv1d": true, |
| "Conv2d": true, |
| "Conv3d": false, |
| "LayerNorm": false, |
| "MultiheadAttention": false, |
| "Conv1D (transformers)": false, |
| "nn.Parameter": false |
| }, |
| "source": "probe", |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" |
| }, |
| "quantization_backends": { |
| "value": [], |
| "source": "file_check" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "merging": { |
| "value": true, |
| "source": "probe" |
| }, |
| "peft_mixed_model": { |
| "value": true, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": true, |
| "source": "probe" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [ |
| "alpha_pattern", |
| "rank_pattern" |
| ], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": "https://huggingface.co/papers/2108.06098", |
| "source": "file_check", |
| "note": "from the docs intro" |
| } |
| }, |
| "description": "Low-Rank Hadamard Product (LoHa), is similar to LoRA except it approximates the large weight matrix with more low-rank matrices and combines them with the Hadamard product. This method is even more parameter-efficient than LoRA and achieves comparable performance.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/loha", |
| "paper_url": "https://huggingface.co/papers/2108.06098", |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "loha/llama-3.2-3B-rank32", |
| "test accuracy": 0.4184988627748294, |
| "forgetting": 0.07485389709472656, |
| "peak_memory_bytes": 23907532800, |
| "train_time_sec": 1444.0684821169707, |
| "num_trainable_params": 18350080, |
| "adapter_file_size_bytes": 73429560, |
| "num_runs": 1 |
| }, |
| "image-gen": { |
| "experiment_name": "loha/flux2-klein-default", |
| "test dino_similarity": 0.6618489821751913, |
| "drift": 0.26028451323509216, |
| "peak_memory_bytes": 12691963904, |
| "train_time_sec": 2281.250607946, |
| "num_trainable_params": 76677120, |
| "adapter_file_size_bytes": 306753024, |
| "num_runs": 1 |
| } |
| } |
| }, |
| "LOKR": { |
| "config_class": "LoKrConfig", |
| "model_class": "LoKrModel", |
| "features": { |
| "category": { |
| "value": "adapter", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": { |
| "Linear": true, |
| "Embedding": false, |
| "Conv1d": true, |
| "Conv2d": true, |
| "Conv3d": false, |
| "LayerNorm": false, |
| "MultiheadAttention": false, |
| "Conv1D (transformers)": false, |
| "nn.Parameter": false |
| }, |
| "source": "probe", |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" |
| }, |
| "quantization_backends": { |
| "value": [], |
| "source": "file_check" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "merging": { |
| "value": true, |
| "source": "probe" |
| }, |
| "peft_mixed_model": { |
| "value": true, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": true, |
| "source": "probe" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [ |
| "alpha_pattern", |
| "rank_pattern" |
| ], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": null, |
| "source": "file_check", |
| "note": "no unambiguous paper link in the docs intro or the config/model class docstrings" |
| } |
| }, |
| "description": "Low-Rank Kronecker Product (LoKr), is a LoRA-variant method that approximates the large weight matrix with two low-rank matrices and combines them with the Kronecker product. LoKr also provides an optional third low-rank matrix to provide better control during fine-tuning.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/lokr", |
| "paper_url": null, |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "lokr/llama-3.2-3B-rank32", |
| "test accuracy": 0.379833206974981, |
| "forgetting": 0.3052024841308594, |
| "peak_memory_bytes": 23571988480, |
| "train_time_sec": 1317.615543427004, |
| "num_trainable_params": 279552, |
| "adapter_file_size_bytes": 1131984, |
| "num_runs": 1 |
| }, |
| "image-gen": { |
| "experiment_name": "lokr/flux2-klein-default", |
| "test dino_similarity": 0.5805431207021078, |
| "drift": 0.16636252403259277, |
| "peak_memory_bytes": 10856955904, |
| "train_time_sec": 2097.732945282998, |
| "num_trainable_params": 783360, |
| "adapter_file_size_bytes": 3160032, |
| "num_runs": 1 |
| } |
| } |
| }, |
| "LORA": { |
| "config_class": "LoraConfig", |
| "model_class": "LoraModel", |
| "features": { |
| "category": { |
| "value": "adapter", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": { |
| "Linear": true, |
| "Embedding": true, |
| "Conv1d": true, |
| "Conv2d": true, |
| "Conv3d": true, |
| "LayerNorm": false, |
| "MultiheadAttention": true, |
| "Conv1D (transformers)": true, |
| "nn.Parameter": true |
| }, |
| "source": "probe", |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" |
| }, |
| "quantization_backends": { |
| "value": [ |
| "aqlm", |
| "awq", |
| "bnb_4bit", |
| "bnb_8bit", |
| "eetq", |
| "gptq", |
| "hqq", |
| "inc", |
| "torchao" |
| ], |
| "source": "file_check" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": true, |
| "source": "introspection" |
| }, |
| "merging": { |
| "value": true, |
| "source": "probe" |
| }, |
| "peft_mixed_model": { |
| "value": true, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": true, |
| "source": "introspection", |
| "note": "already a LoRA adapter" |
| }, |
| "add_weighted_adapter": { |
| "value": true, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": true, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": true |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [ |
| "alpha_pattern", |
| "layer_replication", |
| "rank_pattern", |
| "use_dora", |
| "use_rslora" |
| ], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": "https://huggingface.co/papers/2309.15223", |
| "source": "file_check", |
| "note": "from the docs intro" |
| } |
| }, |
| "description": "Low-Rank Adaptation (LoRA) is a PEFT method that decomposes a large matrix into two smaller low-rank matrices in the attention layers. This drastically reduces the number of parameters that need to be fine-tuned.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/lora", |
| "paper_url": "https://huggingface.co/papers/2309.15223", |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "lora/llama-3.2-3B-rank64-rslora", |
| "test accuracy": 0.5322213798332069, |
| "forgetting": 0.6632366180419922, |
| "peak_memory_bytes": 22554869760, |
| "train_time_sec": 982.3692580849747, |
| "num_trainable_params": 18350080, |
| "adapter_file_size_bytes": 73415408, |
| "num_runs": 7 |
| }, |
| "image-gen": { |
| "experiment_name": "lora/flux2-klein-default", |
| "test dino_similarity": 0.697154720624288, |
| "drift": 0.26200442016124725, |
| "peak_memory_bytes": 10708058112, |
| "train_time_sec": 759.1968001689999, |
| "num_trainable_params": 38338560, |
| "adapter_file_size_bytes": 153377056, |
| "num_runs": 1 |
| } |
| } |
| }, |
| "MISS": { |
| "config_class": "MissConfig", |
| "model_class": "MissModel", |
| "features": { |
| "category": { |
| "value": "adapter", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": { |
| "Linear": true, |
| "Embedding": false, |
| "Conv1d": false, |
| "Conv2d": false, |
| "Conv3d": false, |
| "LayerNorm": false, |
| "MultiheadAttention": false, |
| "Conv1D (transformers)": false, |
| "nn.Parameter": false |
| }, |
| "source": "probe", |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" |
| }, |
| "quantization_backends": { |
| "value": [ |
| "aqlm", |
| "awq", |
| "bnb_4bit", |
| "bnb_8bit", |
| "eetq", |
| "gptq", |
| "hqq", |
| "inc", |
| "torchao" |
| ], |
| "source": "file_check" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "merging": { |
| "value": true, |
| "source": "probe" |
| }, |
| "peft_mixed_model": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": true, |
| "source": "probe" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": "https://arxiv.org/abs/2409.15371", |
| "source": "file_check", |
| "note": "from the docs intro" |
| } |
| }, |
| "description": "MiSS (Matrix Shard Sharing) is a PEFT method that achieves a good balance between model performance and computational efficiency. It requires only a single trainable matrix and introduces a shard-sharing mechanism distinct from LoRA.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/miss", |
| "paper_url": "https://arxiv.org/abs/2409.15371", |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "miss/llama-3.2-3B-bat", |
| "test accuracy": 0.5163002274450341, |
| "forgetting": 0.2831449508666992, |
| "peak_memory_bytes": 25209864192, |
| "train_time_sec": 1423.5636942129931, |
| "num_trainable_params": 7340032, |
| "adapter_file_size_bytes": 29367552, |
| "num_runs": 3 |
| }, |
| "image-gen": { |
| "experiment_name": "miss/flux2-klein-default", |
| "test dino_similarity": 0.6565384666124979, |
| "drift": 0.3087518662214279, |
| "peak_memory_bytes": 10561257472, |
| "train_time_sec": 626.3718940060062, |
| "num_trainable_params": 28508160, |
| "adapter_file_size_bytes": 114043792, |
| "num_runs": 1 |
| } |
| } |
| }, |
| "MULTITASK_PROMPT_TUNING": { |
| "config_class": "MultitaskPromptTuningConfig", |
| "model_class": "MultitaskPromptEmbedding", |
| "features": { |
| "category": { |
| "value": "prompt_learning", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": "not_applicable", |
| "source": "introspection", |
| "note": "method does not wrap target layers" |
| }, |
| "quantization_backends": { |
| "value": "not_applicable", |
| "source": "introspection", |
| "note": "prompt learning does not wrap target layers and generally works regardless of quantization" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection", |
| "note": "only supported by layer-wrapping adapter methods" |
| }, |
| "merging": { |
| "value": false, |
| "source": "introspection", |
| "note": "virtual tokens cannot be merged into base weights" |
| }, |
| "peft_mixed_model": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": false, |
| "source": "introspection", |
| "note": "method does not wrap target layers" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": "https://huggingface.co/papers/2303.02861", |
| "source": "file_check", |
| "note": "from the docs intro" |
| } |
| }, |
| "description": "Multitask prompt tuning decomposes the soft prompts of each task into a single learned transferable prompt instead of a separate prompt for each task. The single learned prompt can be adapted for each task by multiplicative low rank updates.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/multitask_prompt_tuning", |
| "paper_url": "https://huggingface.co/papers/2303.02861", |
| "benchmarks": {} |
| }, |
| "OFT": { |
| "config_class": "OFTConfig", |
| "model_class": "OFTModel", |
| "features": { |
| "category": { |
| "value": "adapter", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": { |
| "Linear": true, |
| "Embedding": true, |
| "Conv1d": false, |
| "Conv2d": true, |
| "Conv3d": false, |
| "LayerNorm": false, |
| "MultiheadAttention": false, |
| "Conv1D (transformers)": false, |
| "nn.Parameter": false |
| }, |
| "source": "probe", |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" |
| }, |
| "quantization_backends": { |
| "value": [ |
| "aqlm", |
| "awq", |
| "bnb_4bit", |
| "bnb_8bit", |
| "eetq", |
| "gptq", |
| "hqq", |
| "inc" |
| ], |
| "source": "file_check" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "merging": { |
| "value": true, |
| "source": "probe" |
| }, |
| "peft_mixed_model": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": false, |
| "source": "probe" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": "https://huggingface.co/papers/2306.07280", |
| "source": "file_check", |
| "note": "from the model class docstring" |
| } |
| }, |
| "description": "Orthogonal Finetuning (OFT) is a method developed for adapting text-to-image diffusion models. It works by reparameterizing the pretrained weight matrices with its orthogonal matrix to preserve information in the pretrained model.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/oft", |
| "paper_url": "https://huggingface.co/papers/2306.07280", |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "oft/llama-3.2-3B-rank32", |
| "test accuracy": 0.5034116755117514, |
| "forgetting": 0.5187549591064453, |
| "peak_memory_bytes": 22315794432, |
| "train_time_sec": 1118.0740923878038, |
| "num_trainable_params": 8171520, |
| "adapter_file_size_bytes": 32693568, |
| "num_runs": 1 |
| }, |
| "image-gen": { |
| "experiment_name": "oft/flux2-klein-default", |
| "test dino_similarity": 0.7077637513478597, |
| "drift": 0.26190416514873505, |
| "peak_memory_bytes": 9669967872, |
| "train_time_sec": 523.9673415769976, |
| "num_trainable_params": 4761600, |
| "adapter_file_size_bytes": 19057488, |
| "num_runs": 1 |
| } |
| } |
| }, |
| "OSF": { |
| "config_class": "OSFConfig", |
| "model_class": "OSFModel", |
| "features": { |
| "category": { |
| "value": "adapter", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": { |
| "Linear": true, |
| "Embedding": false, |
| "Conv1d": false, |
| "Conv2d": false, |
| "Conv3d": false, |
| "LayerNorm": false, |
| "MultiheadAttention": false, |
| "Conv1D (transformers)": false, |
| "nn.Parameter": false |
| }, |
| "source": "probe", |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" |
| }, |
| "quantization_backends": { |
| "value": [], |
| "source": "file_check" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "merging": { |
| "value": true, |
| "source": "probe" |
| }, |
| "peft_mixed_model": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": false, |
| "source": "probe" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [ |
| "rank_pattern" |
| ], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": "https://huggingface.co/papers/2504.07097", |
| "source": "file_check", |
| "note": "from the docs intro" |
| } |
| }, |
| "description": "Orthogonal Subspace Fine-tuning (OSF) is a PEFT method designed for continual learning that constrains parameter updates to be orthogonal to previously important directions. This approach enables full fine-tuning while preventing catastrophic forgetting without requiring additional parameters or storing previous gradients.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/osf", |
| "paper_url": "https://huggingface.co/papers/2504.07097", |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "osf/llama-3.2-3B-rank128", |
| "test accuracy": 0.44579226686884005, |
| "forgetting": 0.3507108688354492, |
| "peak_memory_bytes": 38600179712, |
| "train_time_sec": 2726.1649636220536, |
| "num_trainable_params": 194535936, |
| "adapter_file_size_bytes": 389155304, |
| "num_runs": 1 |
| } |
| } |
| }, |
| "PEANUT": { |
| "config_class": "PeanutConfig", |
| "model_class": "PeanutModel", |
| "features": { |
| "category": { |
| "value": "adapter", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": { |
| "Linear": true, |
| "Embedding": false, |
| "Conv1d": false, |
| "Conv2d": false, |
| "Conv3d": false, |
| "LayerNorm": false, |
| "MultiheadAttention": false, |
| "Conv1D (transformers)": false, |
| "nn.Parameter": false |
| }, |
| "source": "probe", |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" |
| }, |
| "quantization_backends": { |
| "value": [], |
| "source": "file_check" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "merging": { |
| "value": true, |
| "source": "probe" |
| }, |
| "peft_mixed_model": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": false, |
| "source": "probe" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": "https://arxiv.org/abs/2410.01870", |
| "source": "file_check", |
| "note": "from the docs intro" |
| } |
| }, |
| "description": "PEANuT is a parameter-efficient fine-tuning technique that introduces", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/peanut", |
| "paper_url": "https://arxiv.org/abs/2410.01870", |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "peanut/llama-3.2-3B-rank32-relu-depth0-s2.0", |
| "test accuracy": 0.4533737680060652, |
| "forgetting": 0.3534202575683594, |
| "peak_memory_bytes": 25216155648, |
| "train_time_sec": 1324.3875514060055, |
| "num_trainable_params": 7340032, |
| "adapter_file_size_bytes": 29375392, |
| "num_runs": 2 |
| }, |
| "image-gen": { |
| "experiment_name": "peanut/flux2-klein-default", |
| "test dino_similarity": 0.7129100759824117, |
| "drift": 0.31749118864536285, |
| "peak_memory_bytes": 12398362624, |
| "train_time_sec": 1981.0113854429874, |
| "num_trainable_params": 57016320, |
| "adapter_file_size_bytes": 228088552, |
| "num_runs": 1 |
| } |
| } |
| }, |
| "POLY": { |
| "config_class": "PolyConfig", |
| "model_class": "PolyModel", |
| "features": { |
| "category": { |
| "value": "adapter", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": { |
| "Linear": true, |
| "Embedding": false, |
| "Conv1d": false, |
| "Conv2d": false, |
| "Conv3d": false, |
| "LayerNorm": false, |
| "MultiheadAttention": false, |
| "Conv1D (transformers)": false, |
| "nn.Parameter": false |
| }, |
| "source": "probe", |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" |
| }, |
| "quantization_backends": { |
| "value": [], |
| "source": "file_check" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "merging": { |
| "value": false, |
| "source": "introspection", |
| "note": "no tuner layer class implements merge()" |
| }, |
| "peft_mixed_model": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": false, |
| "source": "probe" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": null, |
| "source": "file_check", |
| "note": "no unambiguous paper link in the docs intro or the config/model class docstrings" |
| } |
| }, |
| "description": "Polytropon is a multitask model with a number of different LoRA adapters in its \"inventory\". The model learns the correct combination of adapters from the inventory with a routing function to choose the best subset of modules for a specific task.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/poly", |
| "paper_url": null, |
| "benchmarks": {} |
| }, |
| "PREFIX_TUNING": { |
| "config_class": "PrefixTuningConfig", |
| "model_class": "PrefixEncoder", |
| "features": { |
| "category": { |
| "value": "prompt_learning", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": "not_applicable", |
| "source": "introspection", |
| "note": "method does not wrap target layers" |
| }, |
| "quantization_backends": { |
| "value": "not_applicable", |
| "source": "introspection", |
| "note": "prompt learning does not wrap target layers and generally works regardless of quantization" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection", |
| "note": "only supported by layer-wrapping adapter methods" |
| }, |
| "merging": { |
| "value": false, |
| "source": "introspection", |
| "note": "virtual tokens cannot be merged into base weights" |
| }, |
| "peft_mixed_model": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": false, |
| "source": "introspection", |
| "note": "method does not wrap target layers" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": null, |
| "source": "file_check", |
| "note": "no unambiguous paper link in the docs intro or the config/model class docstrings" |
| } |
| }, |
| "description": "Prefix tuning prefixes a series of task-specific vectors to the input sequence that can be learned while keeping the pretrained model frozen. The prefix parameters are inserted in all of the model layers.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/prefix_tuning", |
| "paper_url": null, |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "prefixtuning/llama-3.2-3B-tokens50-init_prefix-lr_0.005", |
| "test accuracy": 0.400303260045489, |
| "forgetting": 0.5316238403320312, |
| "peak_memory_bytes": 20598226944, |
| "train_time_sec": 876.9671407810383, |
| "num_trainable_params": 2867200, |
| "adapter_file_size_bytes": 11468928, |
| "num_runs": 3 |
| } |
| } |
| }, |
| "PROMPT_TUNING": { |
| "config_class": "PromptTuningConfig", |
| "model_class": "PromptEmbedding", |
| "features": { |
| "category": { |
| "value": "prompt_learning", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": "not_applicable", |
| "source": "introspection", |
| "note": "method does not wrap target layers" |
| }, |
| "quantization_backends": { |
| "value": "not_applicable", |
| "source": "introspection", |
| "note": "prompt learning does not wrap target layers and generally works regardless of quantization" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection", |
| "note": "only supported by layer-wrapping adapter methods" |
| }, |
| "merging": { |
| "value": false, |
| "source": "introspection", |
| "note": "virtual tokens cannot be merged into base weights" |
| }, |
| "peft_mixed_model": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": false, |
| "source": "introspection", |
| "note": "method does not wrap target layers" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": null, |
| "source": "file_check", |
| "note": "no unambiguous paper link in the docs intro or the config/model class docstrings" |
| } |
| }, |
| "description": "Prompt tuning adds task-specific prompts to the input, and these prompt parameters are updated independently of the pretrained model parameters which are frozen.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/prompt_tuning", |
| "paper_url": null, |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "prompt_tuning/llama-3.2-3B-sample_vocab-lr_0.001", |
| "test accuracy": 0.3555724033358605, |
| "forgetting": 2.631608009338379, |
| "peak_memory_bytes": 24423432192, |
| "train_time_sec": 1432.4597771912813, |
| "num_trainable_params": 614400, |
| "adapter_file_size_bytes": 2457728, |
| "num_runs": 3 |
| } |
| } |
| }, |
| "PSOFT": { |
| "config_class": "PsoftConfig", |
| "model_class": "PsoftModel", |
| "features": { |
| "category": { |
| "value": "adapter", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": { |
| "Linear": true, |
| "Embedding": false, |
| "Conv1d": false, |
| "Conv2d": false, |
| "Conv3d": false, |
| "LayerNorm": false, |
| "MultiheadAttention": false, |
| "Conv1D (transformers)": false, |
| "nn.Parameter": false |
| }, |
| "source": "probe", |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" |
| }, |
| "quantization_backends": { |
| "value": [], |
| "source": "file_check" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "merging": { |
| "value": true, |
| "source": "probe" |
| }, |
| "peft_mixed_model": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": true, |
| "source": "probe" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": "https://openreview.net/forum?id=FSHrinMArK.", |
| "source": "file_check", |
| "note": "from the config class docstring" |
| } |
| }, |
| "description": "PSOFT is an Orthogonal Fine-Tuning (OFT)-based parameter-efficient fine-tuning method that preserves the geometric relationships of pre-trained weight column vectors while achieving a balanced trade-off between performance and multi-dimensional efficiency, including parameter count, memory usage, and computational cost. By restricting orthogonal transformations to a low-rank principal subspace derived from pre-trained weights, PSOFT bridges the gap between LoRA and OFT, providing both theoretical guarantees and practical adaptability.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/psoft", |
| "paper_url": "https://openreview.net/forum?id=FSHrinMArK.", |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "psoft/llama-3.2-3B-default", |
| "test accuracy": 0.4488248673237301, |
| "forgetting": 0.5123701095581055, |
| "peak_memory_bytes": 21990735872, |
| "train_time_sec": 1505.385764200997, |
| "num_trainable_params": 1856512, |
| "adapter_file_size_bytes": 7448144, |
| "num_runs": 3 |
| }, |
| "image-gen": { |
| "experiment_name": "psoft/flux2-klein-default", |
| "test dino_similarity": 0.5984091758728027, |
| "drift": 0.15463966131210327, |
| "peak_memory_bytes": 10536091648, |
| "train_time_sec": 768.7374397489912, |
| "num_trainable_params": 44800, |
| "adapter_file_size_bytes": 211320, |
| "num_runs": 1 |
| } |
| } |
| }, |
| "PVERA": { |
| "config_class": "PveraConfig", |
| "model_class": "PveraModel", |
| "features": { |
| "category": { |
| "value": "adapter", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": { |
| "Linear": true, |
| "Embedding": false, |
| "Conv1d": false, |
| "Conv2d": false, |
| "Conv3d": false, |
| "LayerNorm": false, |
| "MultiheadAttention": false, |
| "Conv1D (transformers)": true, |
| "nn.Parameter": false |
| }, |
| "source": "probe", |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" |
| }, |
| "quantization_backends": { |
| "value": [ |
| "bnb_4bit", |
| "bnb_8bit" |
| ], |
| "source": "file_check" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "merging": { |
| "value": true, |
| "source": "probe" |
| }, |
| "peft_mixed_model": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": false, |
| "source": "probe" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": "https://huggingface.co/papers/2512.07703", |
| "source": "file_check", |
| "note": "from the docs intro" |
| } |
| }, |
| "description": "PVeRA is a parameter-efficient fine-tuning technique that is base on VeRA, in the family of the LoRA-based adapters. It keeps the very low parameter budget of VeRA, but increases the performance by learning a distribution of latent adaptations.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/pvera", |
| "paper_url": "https://huggingface.co/papers/2512.07703", |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "pvera/llama-3.2-3B-default", |
| "test accuracy": 0.3305534495830174, |
| "forgetting": 0.29680824279785156, |
| "peak_memory_bytes": 22152216576, |
| "train_time_sec": 1069.4061620029825, |
| "num_trainable_params": 143360, |
| "adapter_file_size_bytes": 10025160, |
| "num_runs": 4 |
| }, |
| "image-gen": { |
| "experiment_name": "pvera/flux2-klein-default", |
| "test dino_similarity": 0.6553937196731567, |
| "drift": 0.23013418912887573, |
| "peak_memory_bytes": 10567548928, |
| "train_time_sec": 729.4813956100115, |
| "num_trainable_params": 896000, |
| "adapter_file_size_bytes": 9504248, |
| "num_runs": 1 |
| } |
| } |
| }, |
| "P_TUNING": { |
| "config_class": "PromptEncoderConfig", |
| "model_class": "PromptEncoder", |
| "features": { |
| "category": { |
| "value": "prompt_learning", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": "not_applicable", |
| "source": "introspection", |
| "note": "method does not wrap target layers" |
| }, |
| "quantization_backends": { |
| "value": "not_applicable", |
| "source": "introspection", |
| "note": "prompt learning does not wrap target layers and generally works regardless of quantization" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection", |
| "note": "only supported by layer-wrapping adapter methods" |
| }, |
| "merging": { |
| "value": false, |
| "source": "introspection", |
| "note": "virtual tokens cannot be merged into base weights" |
| }, |
| "peft_mixed_model": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": false, |
| "source": "introspection", |
| "note": "method does not wrap target layers" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": null, |
| "source": "file_check", |
| "note": "no unambiguous paper link in the docs intro or the config/model class docstrings" |
| } |
| }, |
| "description": "P-tuning adds trainable prompt embeddings to the input that is optimized by a prompt encoder to find a better prompt, eliminating the need to manually design prompts. The prompt tokens can be added anywhere in the input sequence, and p-tuning also introduces anchor tokens for improving performance.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/p_tuning", |
| "paper_url": null, |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "ptuning/llama-3.2-3B-default", |
| "test accuracy": 0.3821076573161486, |
| "forgetting": 3.1548709869384766, |
| "peak_memory_bytes": 20950548480, |
| "train_time_sec": 959.7272375969915, |
| "num_trainable_params": 28382208, |
| "adapter_file_size_bytes": 245880, |
| "num_runs": 1 |
| } |
| } |
| }, |
| "RANDLORA": { |
| "config_class": "RandLoraConfig", |
| "model_class": "RandLoraModel", |
| "features": { |
| "category": { |
| "value": "adapter", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": { |
| "Linear": true, |
| "Embedding": false, |
| "Conv1d": false, |
| "Conv2d": false, |
| "Conv3d": false, |
| "LayerNorm": false, |
| "MultiheadAttention": false, |
| "Conv1D (transformers)": true, |
| "nn.Parameter": false |
| }, |
| "source": "probe", |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" |
| }, |
| "quantization_backends": { |
| "value": [ |
| "bnb_4bit", |
| "bnb_8bit" |
| ], |
| "source": "file_check" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "merging": { |
| "value": true, |
| "source": "probe" |
| }, |
| "peft_mixed_model": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": true, |
| "source": "probe" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": null, |
| "source": "file_check", |
| "note": "no unambiguous paper link in the docs intro or the config/model class docstrings" |
| } |
| }, |
| "description": "RandLora is a parameter-efficient fine-tuning technique that is similar to LoRA and VeRA but performs full rank updates to improve performance. RandLora can be particularly useful when adapting large model to hard tasks that require complex updates while preserving the parameter efficiency of LoRA.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/randlora", |
| "paper_url": null, |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "randlora/llama-3.2-3B-default", |
| "test accuracy": 0.49962092494313876, |
| "forgetting": 0.46999263763427734, |
| "peak_memory_bytes": 22791847936, |
| "train_time_sec": 1465.4105980930617, |
| "num_trainable_params": 9289728, |
| "adapter_file_size_bytes": 2211281240, |
| "num_runs": 1 |
| }, |
| "image-gen": { |
| "experiment_name": "randlora/flux2-klein-default", |
| "test dino_similarity": 0.6052322387695312, |
| "drift": 0.4385121874511242, |
| "peak_memory_bytes": 11456741376, |
| "train_time_sec": 2124.7402266980134, |
| "num_trainable_params": 23838720, |
| "adapter_file_size_bytes": 99999999, |
| "num_runs": 1 |
| } |
| } |
| }, |
| "ROAD": { |
| "config_class": "RoadConfig", |
| "model_class": "RoadModel", |
| "features": { |
| "category": { |
| "value": "adapter", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": { |
| "Linear": true, |
| "Embedding": false, |
| "Conv1d": false, |
| "Conv2d": false, |
| "Conv3d": false, |
| "LayerNorm": false, |
| "MultiheadAttention": false, |
| "Conv1D (transformers)": false, |
| "nn.Parameter": false |
| }, |
| "source": "probe", |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" |
| }, |
| "quantization_backends": { |
| "value": [ |
| "bnb_4bit", |
| "bnb_8bit" |
| ], |
| "source": "file_check" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": true, |
| "source": "introspection" |
| }, |
| "merging": { |
| "value": true, |
| "source": "probe" |
| }, |
| "peft_mixed_model": { |
| "value": true, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": false, |
| "source": "probe" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": "https://huggingface.co/papers/2409.00119", |
| "source": "file_check", |
| "note": "from the docs intro" |
| } |
| }, |
| "description": "RoAd is a parameter\u2011efficient fine\u2011tuning technique that adapts large language models by learning a small set of 2\u00d72 rotation matrices (and optional scaling factors) applied to pairs of hidden dimensions. RoAd achieves competitive or superior performance compared to other PEFT methods with under 0.1% trainable parameters.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/road", |
| "paper_url": "https://huggingface.co/papers/2409.00119", |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "road/llama-3.2-3B-lr_0.001", |
| "test accuracy": 0.3866565579984837, |
| "forgetting": 0.1559743881225586, |
| "peak_memory_bytes": 22806528000, |
| "train_time_sec": 986.1049144260469, |
| "num_trainable_params": 229376, |
| "adapter_file_size_bytes": 931480, |
| "num_runs": 1 |
| }, |
| "image-gen": { |
| "experiment_name": "road/flux2-klein-default", |
| "test dino_similarity": 0.6731533209482828, |
| "drift": 0.2640690803527832, |
| "peak_memory_bytes": 11345592320, |
| "train_time_sec": 1011.2359452090022, |
| "num_trainable_params": 890880, |
| "adapter_file_size_bytes": 3584880, |
| "num_runs": 1 |
| } |
| } |
| }, |
| "SHIRA": { |
| "config_class": "ShiraConfig", |
| "model_class": "ShiraModel", |
| "features": { |
| "category": { |
| "value": "adapter", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": { |
| "Linear": true, |
| "Embedding": false, |
| "Conv1d": false, |
| "Conv2d": false, |
| "Conv3d": false, |
| "LayerNorm": false, |
| "MultiheadAttention": false, |
| "Conv1D (transformers)": false, |
| "nn.Parameter": false |
| }, |
| "source": "probe", |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" |
| }, |
| "quantization_backends": { |
| "value": [], |
| "source": "file_check" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "merging": { |
| "value": true, |
| "source": "probe" |
| }, |
| "peft_mixed_model": { |
| "value": true, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": false, |
| "source": "probe" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": "https://huggingface.co/papers/2406.13175", |
| "source": "file_check", |
| "note": "from the docs intro" |
| } |
| }, |
| "description": "Sparse High Rank Adapters or SHiRA is an alternate type of adapter and has been found to have significant advantages over the low rank adapters. Specifically, SHiRA achieves better accuracy than LoRA for a variety of vision and language tasks.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/shira", |
| "paper_url": "https://huggingface.co/papers/2406.13175", |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "shira/llama-3.2-3B-lr_0.0003-random_seed_42", |
| "test accuracy": 0.5276724791508719, |
| "forgetting": 0.3599224090576172, |
| "peak_memory_bytes": 21678260224, |
| "train_time_sec": 1026.581762216112, |
| "num_trainable_params": 9175040, |
| "adapter_file_size_bytes": 110115520, |
| "num_runs": 1 |
| }, |
| "image-gen": { |
| "experiment_name": "shira/flux2-klein-default", |
| "test dino_similarity": 0.7041612068812052, |
| "drift": 0.2558731734752655, |
| "peak_memory_bytes": 11054088192, |
| "train_time_sec": 791.5365517199971, |
| "num_trainable_params": 38338560, |
| "adapter_file_size_bytes": 460085512, |
| "num_runs": 1 |
| } |
| } |
| }, |
| "TINYLORA": { |
| "config_class": "TinyLoraConfig", |
| "model_class": "TinyLoraModel", |
| "features": { |
| "category": { |
| "value": "adapter", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": { |
| "Linear": true, |
| "Embedding": true, |
| "Conv1d": false, |
| "Conv2d": false, |
| "Conv3d": false, |
| "LayerNorm": false, |
| "MultiheadAttention": false, |
| "Conv1D (transformers)": true, |
| "nn.Parameter": false |
| }, |
| "source": "probe", |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" |
| }, |
| "quantization_backends": { |
| "value": [], |
| "source": "file_check" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "merging": { |
| "value": true, |
| "source": "probe" |
| }, |
| "peft_mixed_model": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": true, |
| "source": "probe" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": "https://arxiv.org/abs/2602.04118", |
| "source": "file_check", |
| "note": "from the config class docstring" |
| } |
| }, |
| "description": "TinyLoRA is an extremely parameter-efficient fine-tuning technique that builds upon the LoRA-XS approach by using SVD decomposition of frozen weights and projecting a tiny trainable vector through fixed random tensors. When combined with reinforcement learning (RL) training methods like GRPO, TinyLoRA can achieve competitive performance with as few as 1-13 trainable parameters.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/tinylora", |
| "paper_url": "https://arxiv.org/abs/2602.04118", |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "tinylora/llama-3.2-3B-default", |
| "test accuracy": 0.009855951478392721, |
| "forgetting": -0.7508955001831055, |
| "peak_memory_bytes": 20126367744, |
| "train_time_sec": 1005.6939271469892, |
| "num_trainable_params": 3584, |
| "adapter_file_size_bytes": 2391912, |
| "num_runs": 2 |
| } |
| } |
| }, |
| "TRAINABLE_TOKENS": { |
| "config_class": "TrainableTokensConfig", |
| "model_class": "TrainableTokensModel", |
| "features": { |
| "category": { |
| "value": "adapter", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": { |
| "Linear": true, |
| "Embedding": true, |
| "Conv1d": false, |
| "Conv2d": false, |
| "Conv3d": false, |
| "LayerNorm": false, |
| "MultiheadAttention": false, |
| "Conv1D (transformers)": false, |
| "nn.Parameter": false |
| }, |
| "source": "probe", |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" |
| }, |
| "quantization_backends": { |
| "value": [], |
| "source": "file_check" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "merging": { |
| "value": true, |
| "source": "probe" |
| }, |
| "peft_mixed_model": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": false, |
| "source": "probe" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": false, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": null, |
| "source": "file_check", |
| "note": "no unambiguous paper link in the docs intro or the config/model class docstrings" |
| } |
| }, |
| "description": "The Trainable Tokens method provides a way to target specific token embeddings for fine-tuning without resorting to", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/trainable_tokens", |
| "paper_url": null, |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "trainable_tokens/llama-3.2-3B-sos+eos", |
| "test accuracy": 0.2721758908263836, |
| "forgetting": 0.20132732391357422, |
| "peak_memory_bytes": 20967325696, |
| "train_time_sec": 997.0611474359757, |
| "num_trainable_params": 6144, |
| "adapter_file_size_bytes": 49424, |
| "num_runs": 1 |
| } |
| } |
| }, |
| "VBLORA": { |
| "config_class": "VBLoRAConfig", |
| "model_class": "VBLoRAModel", |
| "features": { |
| "category": { |
| "value": "adapter", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": { |
| "Linear": true, |
| "Embedding": false, |
| "Conv1d": false, |
| "Conv2d": false, |
| "Conv3d": false, |
| "LayerNorm": false, |
| "MultiheadAttention": false, |
| "Conv1D (transformers)": true, |
| "nn.Parameter": false |
| }, |
| "source": "probe", |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" |
| }, |
| "quantization_backends": { |
| "value": [], |
| "source": "file_check" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "merging": { |
| "value": true, |
| "source": "probe" |
| }, |
| "peft_mixed_model": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": true, |
| "source": "probe" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": "https://huggingface.co/papers/2405.15179", |
| "source": "file_check", |
| "note": "from the config class docstring" |
| } |
| }, |
| "description": "## Overview", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/vblora", |
| "paper_url": "https://huggingface.co/papers/2405.15179", |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "vblora/llama-3.2-3B-default", |
| "test accuracy": 0.354814253222138, |
| "forgetting": -0.010824203491210938, |
| "peak_memory_bytes": 22168993792, |
| "train_time_sec": 1188.2950591499684, |
| "num_trainable_params": 1212416, |
| "adapter_file_size_bytes": 4864912, |
| "num_runs": 1 |
| }, |
| "image-gen": { |
| "experiment_name": "vblora/flux2-klein-default", |
| "test dino_similarity": 0.5697349011898041, |
| "drift": 0.1362656056880951, |
| "peak_memory_bytes": 10705960960, |
| "train_time_sec": 701.1592478580133, |
| "num_trainable_params": 38404096, |
| "adapter_file_size_bytes": 153639928, |
| "num_runs": 1 |
| } |
| } |
| }, |
| "VERA": { |
| "config_class": "VeraConfig", |
| "model_class": "VeraModel", |
| "features": { |
| "category": { |
| "value": "adapter", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": { |
| "Linear": true, |
| "Embedding": false, |
| "Conv1d": false, |
| "Conv2d": false, |
| "Conv3d": false, |
| "LayerNorm": false, |
| "MultiheadAttention": false, |
| "Conv1D (transformers)": true, |
| "nn.Parameter": false |
| }, |
| "source": "probe", |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" |
| }, |
| "quantization_backends": { |
| "value": [ |
| "aqlm", |
| "awq", |
| "bnb_4bit", |
| "bnb_8bit", |
| "eetq", |
| "gptq", |
| "hqq", |
| "inc", |
| "torchao" |
| ], |
| "source": "file_check" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "merging": { |
| "value": true, |
| "source": "probe" |
| }, |
| "peft_mixed_model": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": true, |
| "source": "probe" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": "https://huggingface.co/papers/2310.11454", |
| "source": "file_check", |
| "note": "from the docs intro" |
| } |
| }, |
| "description": "VeRA is a parameter-efficient fine-tuning technique that is similar to LoRA but requires even fewer extra parameters while promising similar or even better performance. As such, it is particularly useful when the parameter budget is very limited, e.g.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/vera", |
| "paper_url": "https://huggingface.co/papers/2310.11454", |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "vera/llama-3.2-3B-default", |
| "test accuracy": 0.37604245640636846, |
| "forgetting": 0.1016998291015625, |
| "peak_memory_bytes": 21617442816, |
| "train_time_sec": 1015.8542694449425, |
| "num_trainable_params": 129024, |
| "adapter_file_size_bytes": 6821968, |
| "num_runs": 1 |
| }, |
| "image-gen": { |
| "experiment_name": "vera/flux2-klein-default", |
| "test dino_similarity": 0.6626973748207092, |
| "drift": 0.23100578784942627, |
| "peak_memory_bytes": 10565451776, |
| "train_time_sec": 720.1334591520208, |
| "num_trainable_params": 893440, |
| "adapter_file_size_bytes": 8314200, |
| "num_runs": 1 |
| } |
| } |
| }, |
| "WAVEFT": { |
| "config_class": "WaveFTConfig", |
| "model_class": "WaveFTModel", |
| "features": { |
| "category": { |
| "value": "adapter", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": { |
| "Linear": true, |
| "Embedding": false, |
| "Conv1d": false, |
| "Conv2d": false, |
| "Conv3d": false, |
| "LayerNorm": false, |
| "MultiheadAttention": false, |
| "Conv1D (transformers)": true, |
| "nn.Parameter": false |
| }, |
| "source": "probe", |
| "note": "nn.Parameter support is based on the presence of the target_parameters config option" |
| }, |
| "quantization_backends": { |
| "value": [], |
| "source": "file_check" |
| }, |
| "multiple_adapters": { |
| "value": true, |
| "source": "probe" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "merging": { |
| "value": true, |
| "source": "probe" |
| }, |
| "peft_mixed_model": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": true, |
| "source": "probe" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": true, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": "https://huggingface.co/papers/2505.12532", |
| "source": "file_check", |
| "note": "from the docs intro" |
| } |
| }, |
| "description": "WaveFT is a novel parameter-efficient fine-tuning (PEFT) method that introduces sparse updates in the **wavelet domain** of residual matrices. Unlike LoRA, which is constrained by discrete low-rank choices, WaveFT enables fine-grained control over the number of trainable parameters by directly learning a sparse set of coefficients in the transformed space.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/waveft", |
| "paper_url": "https://huggingface.co/papers/2505.12532", |
| "benchmarks": { |
| "metamathqa": { |
| "experiment_name": "waveft/llama-3.2-3B-n_frequency-5000", |
| "test accuracy": 0.4200151630022744, |
| "forgetting": -0.13996505737304688, |
| "peak_memory_bytes": 24895291392, |
| "train_time_sec": 2067.484345029923, |
| "num_trainable_params": 280000, |
| "adapter_file_size_bytes": 1127304, |
| "num_runs": 1 |
| }, |
| "image-gen": { |
| "experiment_name": "waveft/flux2-klein-default", |
| "test dino_similarity": 0.6139253179232279, |
| "drift": 0.20272931456565857, |
| "peak_memory_bytes": 12985565184, |
| "train_time_sec": 2970.379196393995, |
| "num_trainable_params": 207360, |
| "adapter_file_size_bytes": 840360, |
| "num_runs": 1 |
| } |
| } |
| }, |
| "XLORA": { |
| "config_class": "XLoraConfig", |
| "model_class": "XLoraModel", |
| "features": { |
| "category": { |
| "value": "adapter", |
| "source": "introspection" |
| }, |
| "target_layer_types": { |
| "value": "unknown", |
| "source": "probe", |
| "note": "not probed: requires pre-trained LoRA adapter checkpoints to instantiate" |
| }, |
| "quantization_backends": { |
| "value": [], |
| "source": "file_check" |
| }, |
| "multiple_adapters": { |
| "value": "unknown", |
| "source": "probe", |
| "note": "not probed: requires pre-trained LoRA adapter checkpoints to instantiate" |
| }, |
| "mixed_adapter_batches": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "merging": { |
| "value": false, |
| "source": "introspection", |
| "note": "no tuner layer class implements merge()" |
| }, |
| "peft_mixed_model": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "lora_conversion": { |
| "value": false, |
| "source": "introspection", |
| "note": "based on presence of a supports_lora_conversion override; probing failed: not probed: requires pre-trained LoRA adapter checkpoints to instantiate" |
| }, |
| "add_weighted_adapter": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "hotswapping": { |
| "value": false, |
| "source": "introspection" |
| }, |
| "auxiliary_modules": { |
| "value": { |
| "modules_to_save": false, |
| "trainable_token_indices": false |
| }, |
| "source": "introspection" |
| }, |
| "extras": { |
| "value": [], |
| "source": "introspection" |
| }, |
| "paper_url": { |
| "value": "https://huggingface.co/papers/2402.07148", |
| "source": "file_check", |
| "note": "from the docs intro" |
| } |
| }, |
| "description": "Mixture of LoRA Experts (X-LoRA) is a PEFT method enabling sparse or dense mixture of LoRA experts based on a high granularity (token, layer, sequence) scalings matrix. This leverages frozen LoRA adapters and a frozen base model to drastically reduces the number of parameters that need to be fine-tuned.", |
| "docs_url": "https://huggingface.co/docs/peft/main/en/package_reference/xlora", |
| "paper_url": "https://huggingface.co/papers/2402.07148", |
| "benchmarks": {} |
| } |
| } |
| } |
|
|
