在ChatGLM3-6B的微调过程中，遇到如下报错（The following error occurs during the fine-tuning of ChatGLM3-6B)ImportError: cannot import name 'log' from 'torch.distributed.elastic.agent.server.api'

[Scorponok31]

╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
│ /home/vivoadmin/miniconda3/envs/chatglm3_test20240726/chatglm3/ChatGLM3/finetune_demo/finetune_h │
│ f.py:537 in main │
│ │
│ 534 │ ) │
│ 535 │ │
536 │ if auto_resume_from_checkpoint.upper() == "" or auto_resume_from_checkpoint is None: │
│ ❱ 537 │ │ trainer.train() │
│ 538 │ else: │
│ 539 │ │ def do_rf_checkpoint(sn): │
│ 540 │ │ │ model.gradient_checkpointing_enable() │
│ │
│ /home/vivoadmin/miniconda3/envs/chatglm3_test20240726/lib/python3.12/site-packages/transformers/ │
│ trainer.py:1885 in train │
│ │
│ 1882 │ │ │ finally: │
│ 1883 │ │ │ │ hf_hub_utils.enable_progress_bars() │
│ 1884 │ │ else: │
│ ❱ 1885 │ │ │ return inner_training_loop( │
│ 1886 │ │ │ │ args=args, │
│ 1887 │ │ │ │ resume_from_checkpoint=resume_from_checkpoint, │
│ 1888 │ │ │ │ trial=trial, │
│ │
│ /home/vivoadmin/miniconda3/envs/chatglm3_test20240726/lib/python3.12/site-packages/transformers/ │
│ trainer.py:2022 in _inner_training_loop │
│ │
│ 2019 │ │ │ │
│ 2020 │ │ │ self.model.gradient_checkpointing_enable(gradient_checkpointing_kwargs=gradi │
│ 2021 │ │ │
│ ❱ 2022 │ │ model = self._wrap_model(self.model_wrapped) │
│ 2023 │ │ │
│ 2024 │ │ # as the model is wrapped, don't use accelerator.prepare │
│ 2025 │ │ # this is for unhandled cases such as │
│ │
│ /home/vivoadmin/miniconda3/envs/chatglm3_test20240726/lib/python3.12/site-packages/transformers/ │
│ trainer.py:1640 in _wrap_model │
│ │
│ 1637 │ │ │ return smp.DistributedModel(model, backward_passes_per_step=self.args.gradie │
│ 1638 │ │ │
│ 1639 │ │ # train/eval could be run multiple-times - if already wrapped, don't re-wrap it │
│ ❱ 1640 │ │ if self.accelerator.unwrap_model(model) is not model: │
│ 1641 │ │ │ return model │
│ 1642 │ │ │
│ 1643 │ │ # Mixed precision training with apex (torch < 1.6) │
│ │
│ /home/vivoadmin/miniconda3/envs/chatglm3_test20240726/lib/python3.12/site-packages/accelerate/ac │
│ celerator.py:2540 in unwrap_model │
│ │
│ 2537 │ │ MyModel │
│ 2538 │ │ ``` │
│ 2539 │ │ """ │
│ ❱ 2540 │ │ return extract_model_from_parallel(model, keep_fp32_wrapper) │
│ 2541 │ │
│ 2542 │ def wait_for_everyone(self): │
│ 2543 │ │ """ │
│ │
│ /home/vivoadmin/miniconda3/envs/chatglm3_test20240726/lib/python3.12/site-packages/accelerate/ut │
│ ils/other.py:80 in extract_model_from_parallel │
│ │
│ 77 │ │ model = model._orig_mod │
│ 78 │ │
│ 79 │ if is_deepspeed_available(): │
│ ❱ 80 │ │ from deepspeed import DeepSpeedEngine │
│ 81 │ │ │
│ 82 │ │ options += (DeepSpeedEngine,) │
│ 83 │
│ │
│ /home/vivoadmin/miniconda3/envs/chatglm3_test20240726/lib/python3.12/site-packages/deepspeed/i │
│ nit.py:22 in │
│ │
│ 19 │ HAS_TRITON = False │
│ 20 │
│ 21 from . import ops │
│ ❱ 22 from . import module_inject │
│ 23 │
│ 24 from .accelerator import get_accelerator │
│ 25 from .runtime.engine import DeepSpeedEngine, DeepSpeedOptimizerCallable, DeepSpeedSchedu │
│ │
│ /home/vivoadmin/miniconda3/envs/chatglm3_test20240726/lib/python3.12/site-packages/deepspeed/mod │
│ ule_inject/init.py:6 in │
│ │
│ 3 │
│ 4 # DeepSpeed Team │
│ 5 │
│ ❱ 6 from .replace_module import replace_transformer_layer, revert_transformer_layer, Replace │
│ 7 from .module_quantize import quantize_transformer_layer │
│ 8 from .replace_policy import HFBertLayerPolicy │
│ 9 from .layers import LinearAllreduce, LinearLayer, EmbeddingLayer, Normalize │
│ │
│ /home/vivoadmin/miniconda3/envs/chatglm3_test20240726/lib/python3.12/site-packages/deepspeed/mod │
│ ule_inject/replace_module.py:607 in │
│ │
│ 604 │ return replaced_module │
│ 605 │
│ 606 │
│ ❱ 607 from ..pipe import PipelineModule │
│ 608 │
│ 609 import re │
│ 610 │
│ │
│ /home/vivoadmin/miniconda3/envs/chatglm3_test20240726/lib/python3.12/site-packages/deepspeed/pip │
│ e/init.py:6 in │
│ │
│ 3 │
│ 4 # DeepSpeed Team │
│ 5 │
│ ❱ 6 from ..runtime.pipe import PipelineModule, LayerSpec, TiedLayerSpec │
│ 7 │
│ │
│ /home/vivoadmin/miniconda3/envs/chatglm3_test20240726/lib/python3.12/site-packages/deepspeed/run │
│ time/pipe/init.py:6 in │
│ │
│ 3 │
│ 4 # DeepSpeed Team │
│ 5 │
│ ❱ 6 from .module import PipelineModule, LayerSpec, TiedLayerSpec │
│ 7 from .topology import ProcessTopology │
│ 8 │
│ │
│ /home/vivoadmin/miniconda3/envs/chatglm3_test20240726/lib/python3.12/site-packages/deepspeed/run │
│ time/pipe/module.py:19 in │
│ │
│ 16 │
│ 17 from deepspeed.utils import logger │
│ 18 from .. import utils as ds_utils │
│ ❱ 19 from ..activation_checkpointing import checkpointing │
│ 20 from .topology import PipeDataParallelTopology, PipelineParallelGrid │
│ 21 from deepspeed.runtime.state_dict_factory import SDLoaderFactory │
│ 22 from deepspeed.accelerator import get_accelerator │
│ │
│ /home/vivoadmin/miniconda3/envs/chatglm3_test20240726/lib/python3.12/site-packages/deepspeed/run │
│ time/activation_checkpointing/checkpointing.py:26 in │
│ │
│ 23 import mmap │
│ 24 from torch import _C │
│ 25 │
│ ❱ 26 from deepspeed.runtime.config import DeepSpeedConfig │
│ 27 from deepspeed.utils import logger │
│ 28 from deepspeed.runtime.utils import copy_to_device, move_to_device, see_memory_usage, bw │
│ 29 from deepspeed.utils.timer import SynchronizedWallClockTimer as Timers, FORWARD_GLOBAL_T │
│ │
│ /home/vivoadmin/miniconda3/envs/chatglm3_test20240726/lib/python3.12/site-packages/deepspeed/run │
│ time/config.py:41 in │
│ │
│ 38 from ..git_version_info import version as version │
│ 39 from ..utils import logger │
│ 40 │
│ ❱ 41 from ..elasticity import ( │
│ 42 │ elasticity_enabled, │
│ 43 │ compute_elastic_config, │
│ 44 │ ensure_immutable_elastic_config, │
│ │
│ /home/vivoadmin/miniconda3/envs/chatglm3_test20240726/lib/python3.12/site-packages/deepspeed/ela │
│ sticity/init.py:10 in │
│ │
│ 7 from .utils import is_torch_elastic_compatible │
│ 8 from .constants import ENABLED, ENABLED_DEFAULT, ELASTICITY │
│ 9 if is_torch_elastic_compatible(): │
│ ❱ 10 │ from .elastic_agent import DSElasticAgent │
│ 11 │
│ │
│ /home/vivoadmin/miniconda3/envs/chatglm3_test20240726/lib/python3.12/site-packages/deepspeed/ela │
│ sticity/elastic_agent.py:9 in │
│ │
│ 6 from torch.distributed.elastic.agent.server.local_elastic_agent import LocalElasticAgent │
│ 7 from typing import Any, Dict, Optional, Tuple │
│ 8 from datetime import datetime │
│ ❱ 9 from torch.distributed.elastic.agent.server.api import log, _get_socket_with_port │
│ 10 from torch.distributed.elastic.metrics import put_metric │
│ 11 from torch.distributed.elastic.agent.server.api import ( │
│ 12 │ RunResult, │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
ImportError: cannot import name 'log' from 'torch.distributed.elastic.agent.server.api'

ComplelteError.txt

我分别用lora或者p-tunningV2微调，都会出现该报错，我使用的是官方demo进行的微调，单卡或者多卡都会出现这个问题，请各位帮忙检查这个错误到底如何修正？Linux服务器，显卡为2张L40S
( fine tuning of ChatGLM3-6B, I encountered the following errors when I used lora or p-tunningV2 for fine tuning respectively. I used the official demo for fine tuning, and this problem would occur on either single card or multi-card. Please help to check how to correct this error. Linux server with two L40S graphics cards.)