From ed0c962ff54e553c83dc0fbd46aec6f6674f1bba Mon Sep 17 00:00:00 2001
From: Sourab Mangrulkar <13534540+pacman100@users.noreply.github.com>
Date: Wed, 8 Feb 2023 12:59:29 +0530
Subject: [PATCH] fixes

---
 ...ccelerate_ds_zero3_cpu_offload_config.yaml | 22 +++++++++++++++++++
 src/peft/utils/__init__.py                    |  2 +-
 2 files changed, 23 insertions(+), 1 deletion(-)
 create mode 100644 examples/causal_language_modeling/accelerate_ds_zero3_cpu_offload_config.yaml

diff --git a/examples/causal_language_modeling/accelerate_ds_zero3_cpu_offload_config.yaml b/examples/causal_language_modeling/accelerate_ds_zero3_cpu_offload_config.yaml
new file mode 100644
index 0000000..a4a0bcf
--- /dev/null
+++ b/examples/causal_language_modeling/accelerate_ds_zero3_cpu_offload_config.yaml
@@ -0,0 +1,22 @@
+compute_environment: LOCAL_MACHINE
+deepspeed_config:
+  gradient_accumulation_steps: 1
+  gradient_clipping: 1.0
+  offload_optimizer_device: none
+  offload_param_device: none
+  zero3_init_flag: true
+  zero3_save_16bit_model: true
+  zero_stage: 3
+distributed_type: DEEPSPEED
+downcast_bf16: 'no'
+dynamo_backend: 'NO'
+fsdp_config: {}
+machine_rank: 0
+main_training_function: main
+megatron_lm_config: {}
+mixed_precision: 'no'
+num_machines: 1
+num_processes: 1
+rdzv_backend: static
+same_network: true
+use_cpu: false
\ No newline at end of file
diff --git a/src/peft/utils/__init__.py b/src/peft/utils/__init__.py
index b451fb4..55cdce7 100644
--- a/src/peft/utils/__init__.py
+++ b/src/peft/utils/__init__.py
@@ -26,4 +26,4 @@ from .other import (
     shift_tokens_right,
     transpose,
 )
-from .save_and_load import get_peft_model_state_dict, peft_model_load_and_dispatch, set_peft_model_state_dict
+from .save_and_load import get_peft_model_state_dict, set_peft_model_state_dict