_single()` function
-5. Add new comm op to `run_all.py`
diff --git a/benchmarks/communication/__init__.py b/benchmarks/communication/__init__.py
deleted file mode 100644
index fcb45ab2b685..000000000000
--- a/benchmarks/communication/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-'''Copyright The Microsoft DeepSpeed Team'''
diff --git a/benchmarks/communication/all_gather.py b/benchmarks/communication/all_gather.py
deleted file mode 100644
index dc97267b3840..000000000000
--- a/benchmarks/communication/all_gather.py
+++ /dev/null
@@ -1,159 +0,0 @@
-'''Copyright The Microsoft DeepSpeed Team'''
-
-from benchmarks.communication.utils import *
-from benchmarks.communication.constants import *
-from deepspeed.accelerator import get_accelerator
-
-import time
-
-
-# Run all_gather and print metrics
-def timed_all_gather(input, output, args):
- if args.dist == 'torch':
- import torch.distributed as dist
- elif args.dist == 'deepspeed':
- import deepspeed.comm as dist
-
- sync_all()
- # Warmups, establish connections, etc.
- for i in range(args.warmups):
- # use all_gather_base if available
- if args.dist == 'torch':
- if hasattr(torch.distributed, "_all_gather_base"):
- dist._all_gather_base(output, input, group=None, async_op=args.async_op)
- else:
- output_tensors = list(
- torch.chunk(output_tensor,
- cdb.get_world_size(group)))
- dist.all_gather(output_tensors, input_tensor, group=group, async_op=True)
- elif args.dist == 'deepspeed':
- dist.allgather_fn(output, input, group=None, async_op=args.async_op)
- sync_all()
-
- # time the actual comm op trials times and average it
- pre = time.perf_counter()
- for i in range(args.trials):
- # use all_gather_base if available
- if args.dist == 'torch':
- if hasattr(torch.distributed, "_all_gather_base"):
- dist._all_gather_base(output, input, group=None, async_op=args.async_op)
- else:
- output_tensors = list(
- torch.chunk(output_tensor,
- cdb.get_world_size(group)))
- dist.all_gather(output_tensors, input_tensor, group=group, async_op=True)
- elif args.dist == 'deepspeed':
- dist.allgather_fn(output, input, group=None, async_op=args.async_op)
- sync_all()
- duration = time.perf_counter() - pre
-
- # maintain and clean performance data
- avg_duration = duration / args.trials
- size = input.element_size() * input.nelement()
- n = dist.get_world_size()
- tput, busbw = get_bw('all_gather', size, avg_duration, args)
- tput_str, busbw_str, duration_str = get_metric_strings(args, tput, busbw, avg_duration)
- desc = f'{input.nelement()}x{input.element_size()}'
-
- if not args.raw:
- size = convert_size(size)
-
- print_rank_0(
- f"{size:<20} {desc:25s} {duration_str:20s} {tput_str:20s} {busbw_str:20s}")
-
-
-def run_all_gather(local_rank, args):
- if args.dist == 'torch':
- import torch.distributed as dist
- elif args.dist == 'deepspeed':
- import deepspeed.comm as dist
-
- # Prepare benchmark header
- print_header(args, 'all_gather')
- global_rank = dist.get_rank()
- world_size = dist.get_world_size()
-
- if args.scan:
- # Create list of message sizes
- M_LIST = []
- for x in (2**p for p in range(1, args.maxsize)):
- M_LIST.append(x)
-
- sync_all()
- # loop over various tensor sizes
- for M in M_LIST:
- global_rank = dist.get_rank()
- try:
- mat = torch.ones(world_size,
- M,
- dtype=getattr(
- torch,
- args.dtype)).to(
- get_accelerator().device_name(local_rank))
- sync_all()
- input = ((mat.mul_(float(global_rank))).view(-1))
- # Delete original mat to avoid OOM
- del mat
- get_accelerator().empty_cache()
- output = torch.zeros(input.nelement() * world_size,
- dtype=getattr(
- torch,
- args.dtype)).to(
- get_accelerator().device_name(local_rank))
- except RuntimeError as e:
- if 'out of memory' in str(e):
- if dist.get_rank() == 0:
- print('WARNING: Ran out of GPU memory. Exiting comm op.')
- sync_all()
- break
- sync_all()
- timed_all_gather(input, output, args)
- else:
- # all_gather_base saves memory
- if (args.dist == 'torch'
- and hasattr(torch.distributed,
- "_all_gather_base")) or (args.dist == 'deepspeed'
- and dist.has_allgather_base):
- mem_factor = args.mem_factor + 0.2
- else:
- mem_factor = args.mem_factor
- # Send the biggest message size our GPUs can fit. If you're facing OOM errors, reduce the mem_factor
- sync_all()
- elements_per_gpu = max_numel(comm_op='all_gather',
- dtype=getattr(torch,
- args.dtype),
- mem_factor=mem_factor,
- local_rank=local_rank,
- args=args)
- try:
- mat = torch.ones(elements_per_gpu,
- dtype=getattr(torch,
- args.dtype)).to(
- get_accelerator().device_name(local_rank))
- # multiply each GPU's tensor by the rank to ease debugging
- input = ((mat.mul_(float(global_rank))).view(-1))
- # Delete original mat to avoid OOM
- del mat
- get_accelerator().empty_cache()
- output = torch.zeros(
- elements_per_gpu * world_size,
- dtype=getattr(torch,
- args.dtype)).to(get_accelerator().device_name(local_rank))
- except RuntimeError as e:
- if 'out of memory' in str(e):
- if dist.get_rank() == 0:
- print(
- 'WARNING: Ran out of GPU memory. Try to reduce the --mem-factor argument!'
- )
- sync_all()
- return
-
- sync_all()
- timed_all_gather(input, output, args)
-
-
-if __name__ == "__main__":
- args = benchmark_parser().parse_args()
- rank = args.local_rank
- init_processes(local_rank=rank, args=args)
- run_all_gather(local_rank=rank, args=args)
diff --git a/benchmarks/communication/all_reduce.py b/benchmarks/communication/all_reduce.py
deleted file mode 100644
index edc1b99301c0..000000000000
--- a/benchmarks/communication/all_reduce.py
+++ /dev/null
@@ -1,113 +0,0 @@
-'''Copyright The Microsoft DeepSpeed Team'''
-
-from benchmarks.communication.utils import *
-from benchmarks.communication.constants import *
-from deepspeed.accelerator import get_accelerator
-
-import time
-
-
-def timed_all_reduce(input, args):
- if args.dist == 'torch':
- import torch.distributed as dist
- elif args.dist == 'deepspeed':
- import deepspeed.comm as dist
-
- sync_all()
- # Warmups, establish connections, etc.
- for i in range(args.warmups):
- dist.all_reduce(input, async_op=args.async_op)
- sync_all()
-
- # time the actual comm op trials times and average it
- pre = time.perf_counter()
- for i in range(args.trials):
- dist.all_reduce(input, async_op=args.async_op)
- sync_all()
- duration = time.perf_counter() - pre
-
- # maintain and clean performance data
- avg_duration = duration / args.trials
- size = input.element_size() * input.nelement()
- n = dist.get_world_size()
- tput, busbw = get_bw('all_reduce', size, avg_duration, args)
- tput_str, busbw_str, duration_str = get_metric_strings(args, tput, busbw, avg_duration)
- desc = f'{input.nelement()}x{input.element_size()}'
-
- if not args.raw:
- size = convert_size(size)
-
- print_rank_0(
- f"{size:<20} {desc:25s} {duration_str:20s} {tput_str:20s} {busbw_str:20s}")
-
-
-def run_all_reduce(local_rank, args):
- if args.dist == 'torch':
- import torch.distributed as dist
- elif args.dist == 'deepspeed':
- import deepspeed.comm as dist
-
- # Prepare benchmark header
- print_header(args, 'all_reduce')
-
- world_size = dist.get_world_size()
- global_rank = dist.get_rank()
-
- if args.scan:
- M_LIST = []
- for x in (2**p for p in range(1, args.maxsize)):
- M_LIST.append(x)
-
- sync_all()
- # loop over various tensor sizes
- for M in M_LIST:
- global_rank = dist.get_rank()
- try:
- mat = torch.ones(world_size,
- M,
- dtype=getattr(
- torch,
- args.dtype)).to(
- get_accelerator().device_name(local_rank))
- sync_all()
- input = ((mat.mul_(float(global_rank))).view(-1))
- except RuntimeError as e:
- if 'out of memory' in str(e):
- if dist.get_rank() == 0:
- print('WARNING: Ran out of GPU memory. Exiting comm op.')
- sync_all()
- break
- sync_all()
- timed_all_reduce(input, args)
- else:
- # Send the biggest message size our GPUs can fit. If you're facing OOM errors, reduce the mem_factor
- # Don't need output tensor, so we double mem_factor
- elements_per_gpu = max_numel(comm_op='all_reduce',
- dtype=getattr(torch,
- args.dtype),
- mem_factor=args.mem_factor * 2,
- local_rank=local_rank,
- args=args)
- try:
- mat = torch.ones(elements_per_gpu,
- dtype=getattr(torch,
- args.dtype)).to(
- get_accelerator().device_name(local_rank))
- input = ((mat.mul_(float(global_rank))).view(-1))
- except RuntimeError as e:
- if 'out of memory' in str(e):
- if dist.get_rank() == 0:
- print(
- 'WARNING: Ran out of GPU memory. Try to reduce the --mem-factor argument!'
- )
- sync_all()
- return
- sync_all()
- timed_all_reduce(input, args)
-
-
-if __name__ == "__main__":
- args = benchmark_parser().parse_args()
- rank = args.local_rank
- init_processes(local_rank=rank, args=args)
- run_all_reduce(local_rank=rank, args=args)
diff --git a/benchmarks/communication/all_to_all.py b/benchmarks/communication/all_to_all.py
deleted file mode 100644
index bd35cf290e4c..000000000000
--- a/benchmarks/communication/all_to_all.py
+++ /dev/null
@@ -1,134 +0,0 @@
-'''Copyright The Microsoft DeepSpeed Team'''
-
-from benchmarks.communication.utils import *
-from benchmarks.communication.constants import *
-from deepspeed.accelerator import get_accelerator
-
-import time
-
-
-def timed_all_to_all(input, output, args):
- if args.dist == 'torch':
- import torch.distributed as dist
- elif args.dist == 'deepspeed':
- import deepspeed.comm as dist
-
- sync_all()
- # Warmups, establish connections, etc.
- for i in range(args.warmups):
- dist.all_to_all_single(output, input, async_op=args.async_op)
- sync_all()
-
- # time the actual comm op trials times and average it
- pre = time.perf_counter()
- for i in range(args.trials):
- dist.all_to_all_single(output, input, async_op=args.async_op)
- sync_all()
- duration = time.perf_counter() - pre
-
- # maintain and clean performance data
- avg_duration = duration / args.trials
- size = input.element_size() * input.nelement()
- n = dist.get_world_size()
- tput, busbw = get_bw('all_to_all', size, avg_duration, args)
- tput_str, busbw_str, duration_str = get_metric_strings(args, tput, busbw, avg_duration)
- desc = f'{input.nelement()}x{input.element_size()}'
-
- if not args.raw:
- size = convert_size(size)
-
- print_rank_0(
- f"{size:<20} {desc:25s} {duration_str:20s} {tput_str:20s} {busbw_str:20s}")
-
-
-def run_all_to_all(local_rank, args):
- if args.dist == 'torch':
- import torch.distributed as dist
- elif args.dist == 'deepspeed':
- import deepspeed.comm as dist
-
- world_size = dist.get_world_size()
- global_rank = dist.get_rank()
- # Prepare benchmark header
- print_header(args, 'all_to_all')
-
- if args.scan:
- M_LIST = []
- for x in (2**p for p in range(1, args.maxsize)):
- M_LIST.append(x)
-
- sync_all()
- # loop over various tensor sizes
- for M in M_LIST:
- global_rank = dist.get_rank()
- try:
- mat = torch.ones(world_size,
- M,
- dtype=getattr(
- torch,
- args.dtype)).to(
- get_accelerator().device_name(local_rank))
- assert mat.numel() % world_size == 0, f"tensor cannot be divided in {world_size} chunks"
- sync_all()
- input = ((mat.mul_(float(global_rank))).view(-1))
- output = (mat.clone().view(-1))
- except RuntimeError as e:
- if 'out of memory' in str(e):
- if dist.get_rank() == 0:
- print('WARNING: Ran out of GPU memory. Exiting comm op.')
- sync_all()
- break
- sync_all()
- timed_all_to_all(input, output, args)
- else:
- # Send the biggest message size our GPUs can fit. If you're facing OOM errors, reduce the mem_factor
- elements_per_gpu = max_numel(comm_op='all_to_all',
- dtype=getattr(torch,
- args.dtype),
- mem_factor=args.mem_factor,
- local_rank=local_rank,
- args=args)
- try:
- mat = torch.ones(elements_per_gpu,
- dtype=getattr(torch,
- args.dtype)).to(
- get_accelerator().device_name(local_rank))
- assert mat.numel() % world_size == 0, f"tensor with {mat.numel()} elements cannot be divided in {world_size} chunks"
- input = ((mat.mul_(float(global_rank))).view(-1))
- # Delete original mat to avoid OOM
- del mat
- get_accelerator().empty_cache()
- output = torch.zeros(
- elements_per_gpu,
- dtype=getattr(torch,
- args.dtype)).to(get_accelerator().device_name(local_rank))
- except RuntimeError as e:
- if 'out of memory' in str(e):
- if dist.get_rank() == 0:
- print(
- 'WARNING: Ran out of GPU memory. Try to reduce the --mem-factor argument!'
- )
- sync_all()
- return
- sync_all()
-
- if args.debug:
- for i in range(world_size):
- if i == global_rank:
- print(f"Before AllToAll Input List at rank {global_rank}: {input}")
- dist.barrier()
-
- timed_all_to_all(input, output, args)
-
- if args.debug:
- for i in range(world_size):
- if i == global_rank:
- print(f"AllToAll Results at rank {global_rank}: {output}")
- dist.barrier()
-
-
-if __name__ == "__main__":
- args = benchmark_parser().parse_args()
- rank = args.local_rank
- init_processes(local_rank=rank, args=args)
- run_all_to_all(local_rank=rank, args=args)
diff --git a/benchmarks/communication/broadcast.py b/benchmarks/communication/broadcast.py
deleted file mode 100644
index 633e46638fac..000000000000
--- a/benchmarks/communication/broadcast.py
+++ /dev/null
@@ -1,114 +0,0 @@
-'''Copyright The Microsoft DeepSpeed Team'''
-
-import torch
-from benchmarks.communication.utils import *
-from benchmarks.communication.constants import *
-from deepspeed.accelerator import get_accelerator
-
-import time
-
-
-def timed_broadcast(input, args):
- if args.dist == 'torch':
- import torch.distributed as dist
- elif args.dist == 'deepspeed':
- import deepspeed.comm as dist
-
- sync_all()
- # Warmups, establish connections, etc.
- for i in range(args.warmups):
- dist.broadcast(input, 0, async_op=args.async_op)
- sync_all()
-
- # time the actual comm op trials times and average it
- pre = time.perf_counter()
- for i in range(args.trials):
- dist.broadcast(input, 0, async_op=args.async_op)
- sync_all()
- duration = time.perf_counter() - pre
-
- # maintain and clean performance data
- avg_duration = duration / args.trials
- size = input.element_size() * input.nelement()
- n = dist.get_world_size()
- tput, busbw = get_bw('broadcast', size, avg_duration, args)
- tput_str, busbw_str, duration_str = get_metric_strings(args, tput, busbw, avg_duration)
- desc = f'{input.nelement()}x{input.element_size()}'
-
- if not args.raw:
- size = convert_size(size)
-
- print_rank_0(
- f"{size:<20} {desc:25s} {duration_str:20s} {tput_str:20s} {busbw_str:20s}")
-
-
-def run_broadcast(local_rank, args):
- if args.dist == 'torch':
- import torch.distributed as dist
- elif args.dist == 'deepspeed':
- import deepspeed.comm as dist
-
- # Prepare benchmark header
- print_header(args, 'broadcast')
-
- world_size = dist.get_world_size()
- global_rank = dist.get_rank()
-
- if args.scan:
- M_LIST = []
- for x in (2**p for p in range(1, args.maxsize)):
- M_LIST.append(x)
-
- sync_all()
- # loop over various tensor sizes
- for M in M_LIST:
- global_rank = dist.get_rank()
- try:
- mat = torch.ones(world_size,
- M,
- dtype=getattr(
- torch,
- args.dtype)).to(
- get_accelerator().device_name(local_rank))
- sync_all()
- input = ((mat.mul_(float(global_rank))).view(-1))
- except RuntimeError as e:
- if 'out of memory' in str(e):
- if dist.get_rank() == 0:
- print('WARNING: Ran out of GPU memory. Exiting comm op.')
- sync_all()
- break
- sync_all()
- timed_broadcast(input, args)
- else:
- # Send the biggest message size our GPUs can fit. If you're facing OOM errors, reduce the mem_factor
- # Don't need output tensor, so we double mem_factor
- elements_per_gpu = max_numel(comm_op='broadcast',
- dtype=getattr(torch,
- args.dtype),
- mem_factor=args.mem_factor * 2,
- local_rank=local_rank,
- args=args)
- try:
- mat = torch.ones(elements_per_gpu,
- dtype=getattr(torch,
- args.dtype)).to(
- get_accelerator().device_name(local_rank))
- input = ((mat.mul_(float(global_rank))).view(-1))
- except RuntimeError as e:
- if 'out of memory' in str(e):
- if dist.get_rank() == 0:
- print(
- 'WARNING: Ran out of GPU memory. Try to reduce the --mem-factor argument!'
- )
- sync_all()
- return
- sync_all()
- timed_broadcast(input, args)
-
-
-if __name__ == "__main__":
- args = benchmark_parser().parse_args()
- rank = args.local_rank
- init_processes(local_rank=rank, args=args)
- run_broadcast(local_rank=rank, args=args)
diff --git a/benchmarks/communication/constants.py b/benchmarks/communication/constants.py
deleted file mode 100644
index 935927acd174..000000000000
--- a/benchmarks/communication/constants.py
+++ /dev/null
@@ -1,10 +0,0 @@
-'''Copyright The Microsoft DeepSpeed Team'''
-from deepspeed.accelerator import get_accelerator
-
-DEFAULT_WARMUPS = 5
-DEFAULT_TRIALS = 50
-DEFAULT_TYPE = 'float'
-DEFAULT_BACKEND = get_accelerator().communication_backend_name()
-DEFAULT_UNIT = 'Gbps'
-DEFAULT_DIST = 'deepspeed'
-DEFAULT_MAXSIZE = 24
diff --git a/benchmarks/communication/pt2pt.py b/benchmarks/communication/pt2pt.py
deleted file mode 100644
index 1c890fc42e93..000000000000
--- a/benchmarks/communication/pt2pt.py
+++ /dev/null
@@ -1,132 +0,0 @@
-'''Copyright The Microsoft DeepSpeed Team'''
-
-from benchmarks.communication.utils import *
-from benchmarks.communication.constants import *
-from deepspeed.accelerator import get_accelerator
-
-import time
-
-
-def timed_pt2pt(input, args):
- if args.dist == 'torch':
- import torch.distributed as dist
- elif args.dist == 'deepspeed':
- import deepspeed.comm as dist
-
- sync_all()
- # Warmups, establish connections, etc.
- for i in range(args.warmups):
- if dist.get_rank() == 0:
- if args.async_op:
- dist.isend(input, 1)
- else:
- dist.send(input, 1)
- if dist.get_rank() == 1:
- if args.async_op:
- dist.irecv(input, src=0)
- else:
- dist.recv(input, src=0)
- sync_all()
-
- # time the actual comm op trials times and average it
- pre = time.perf_counter()
- for i in range(args.trials):
- if dist.get_rank() == 0:
- if args.async_op:
- dist.isend(input, 1)
- else:
- dist.send(input, 1)
- if dist.get_rank() == 1:
- if args.async_op:
- dist.irecv(input, src=0)
- else:
- dist.recv(input, src=0)
-
- sync_all()
- duration = time.perf_counter() - pre
-
- # maintain and clean performance data
- avg_duration = duration / args.trials
- size = input.element_size() * input.nelement()
- n = dist.get_world_size()
- tput, busbw = get_bw('pt2pt', size, avg_duration, args)
- tput_str, busbw_str, duration_str = get_metric_strings(args, tput, busbw, avg_duration)
- desc = f'{input.nelement()}x{input.element_size()}'
-
- if not args.raw:
- size = convert_size(size)
-
- print_rank_0(
- f"{size:<20} {desc:25s} {duration_str:20s} {tput_str:20s} {busbw_str:20s}")
-
-
-def run_pt2pt(local_rank, args):
- if args.dist == 'torch':
- import torch.distributed as dist
- elif args.dist == 'deepspeed':
- import deepspeed.comm as dist
-
- # Prepare benchmark header
- print_header(args, 'pt2pt')
- global_rank = dist.get_rank()
- world_size = dist.get_world_size()
-
- if args.scan:
- # Create list of message sizes
- M_LIST = []
- for x in (2**p for p in range(1, args.maxsize)):
- M_LIST.append(x)
-
- sync_all()
- # loop over various tensor sizes
- for M in M_LIST:
- global_rank = dist.get_rank()
- try:
- mat = torch.ones(world_size,
- M,
- dtype=getattr(
- torch,
- args.dtype)).to(
- get_accelerator().device_name(local_rank))
- sync_all()
- input = ((mat.mul_(float(global_rank))).view(-1))
- except RuntimeError as e:
- if 'out of memory' in str(e):
- if dist.get_rank() == 0:
- print('WARNING: Ran out of GPU memory. Exiting comm op.')
- sync_all()
- break
- sync_all()
- timed_pt2pt(input, args)
- else:
- # Send the biggest message size our GPUs can fit. If you're facing OOM errors, reduce the mem_factor
- # Don't need output tensor, so double mem_factor
- elements_per_gpu = max_numel(comm_op='pt2pt',
- dtype=getattr(torch,
- args.dtype),
- mem_factor=args.mem_factor * 2,
- local_rank=local_rank,
- args=args)
- try:
- mat = torch.ones(elements_per_gpu,
- dtype=getattr(torch,
- args.dtype)).to(
- get_accelerator().device_name(local_rank))
- input = ((mat.mul_(float(global_rank))).view(-1))
- except RuntimeError as e:
- if 'out of memory' in str(e):
- if dist.get_rank() == 0:
- print(
- 'WARNING: Ran out of GPU memory. Try to reduce the --mem-factor argument!'
- )
- sync_all()
- return
- sync_all()
- timed_pt2pt(input, args)
-
-
-if __name__ == "__main__":
- args = benchmark_parser().parse_args()
- rank = args.local_rank
- init_processes(local_rank=rank, args=args)
- run_pt2pt(local_rank=rank, args=args)
diff --git a/benchmarks/communication/run_all.py b/benchmarks/communication/run_all.py
deleted file mode 100644
index 7ec562cc9ae0..000000000000
--- a/benchmarks/communication/run_all.py
+++ /dev/null
@@ -1,49 +0,0 @@
-'''Copyright The Microsoft DeepSpeed Team'''
-
-from benchmarks.communication.utils import *
-from benchmarks.communication.all_reduce import run_all_reduce
-from benchmarks.communication.all_gather import run_all_gather
-from benchmarks.communication.all_to_all import run_all_to_all
-from benchmarks.communication.pt2pt import run_pt2pt
-from benchmarks.communication.broadcast import run_broadcast
-from benchmarks.communication.constants import *
-
-
-# For importing
-def main(args, rank):
-
- init_processes(local_rank=rank, args=args)
-
- ops_to_run = []
- if args.all_reduce:
- ops_to_run.append('all_reduce')
- if args.all_gather:
- ops_to_run.append('all_gather')
- if args.broadcast:
- ops_to_run.append('broadcast')
- if args.pt2pt:
- ops_to_run.append('pt2pt')
- if args.all_to_all:
- ops_to_run.append('all_to_all')
-
- if len(ops_to_run) == 0:
- ops_to_run = ['all_reduce', 'all_gather', 'all_to_all', 'broadcast', 'pt2pt']
-
- for comm_op in ops_to_run:
- if comm_op == 'all_reduce':
- run_all_reduce(local_rank=rank, args=args)
- if comm_op == 'all_gather':
- run_all_gather(local_rank=rank, args=args)
- if comm_op == 'all_to_all':
- run_all_to_all(local_rank=rank, args=args)
- if comm_op == 'pt2pt':
- run_pt2pt(local_rank=rank, args=args)
- if comm_op == 'broadcast':
- run_broadcast(local_rank=rank, args=args)
-
-
-# For directly calling benchmark
-if __name__ == "__main__":
- args = benchmark_parser().parse_args()
- rank = args.local_rank
- main(args, rank)
diff --git a/benchmarks/communication/utils.py b/benchmarks/communication/utils.py
deleted file mode 100644
index b913dda14fe5..000000000000
--- a/benchmarks/communication/utils.py
+++ /dev/null
@@ -1,220 +0,0 @@
-'''Copyright The Microsoft DeepSpeed Team'''
-
-import torch
-import os
-import math
-import argparse
-from benchmarks.communication.constants import *
-from deepspeed.accelerator import get_accelerator
-
-global dist
-
-
-def init_torch_distributed(backend):
- global dist
- import torch.distributed as dist
- torch.distributed.init_process_group(backend)
- local_rank = int(os.environ['LOCAL_RANK'])
- get_accelerator().set_device(local_rank)
-
-
-def init_deepspeed_comm(backend):
- global dist
- import deepspeed
- import deepspeed.comm as dist
- deepspeed.init_distributed(dist_backend=backend)
- local_rank = int(os.environ['LOCAL_RANK'])
- get_accelerator().set_device(local_rank)
-
-
-def init_processes(local_rank, args):
- if args.dist == 'deepspeed':
- init_deepspeed_comm(args.backend)
- elif args.dist == 'torch':
- init_torch_distributed(args.backend)
- else:
- print_rank_0(f"distributed framework {args.dist} not supported")
- exit(0)
-
-
-def print_rank_0(message):
- if dist.get_rank() == 0:
- print(message)
-
-
-def print_header(args, comm_op):
- if comm_op == 'pt2pt':
- world_size = 2
- else:
- world_size = dist.get_world_size()
- tput = f'Throughput ({args.bw_unit})'
- busbw = f'BusBW ({args.bw_unit})'
- header = f"\n---- Performance of {comm_op} on {world_size} devices ---------------------------------------------------------\n"
- duration_str = 'Duration'
- if args.raw:
- duration_str += ' (us)'
- header += f"{'Size (Bytes)':20s} {'Description':25s} {duration_str:20s} {tput:20s} {busbw:20s}\n"
- header += "----------------------------------------------------------------------------------------------------"
- print_rank_0(header)
-
-
-def get_bw(comm_op, size, duration, args):
- n = dist.get_world_size()
- tput = 0
- busbw = 0
- if comm_op == "all_to_all":
- tput = (size / duration)
- busbw = (size / duration) * ((n - 1) / n)
- elif comm_op == "all_gather":
- size *= n
- tput = (size / duration)
- busbw = (size / duration) * ((n - 1) / n)
- elif comm_op == "all_reduce":
- tput = (size * 2 / duration)
- busbw = (size / duration) * (2 * (n - 1) / n)
- elif comm_op == "pt2pt" or comm_op == "broadcast":
- tput = (size / duration)
- busbw = tput
- else:
- print_rank_0("wrong comm_op specified")
- exit(0)
-
- if args.bw_unit == 'Gbps':
- tput *= 8
- busbw *= 8
-
- return tput, busbw
-
-
-def get_metric_strings(args, tput, busbw, duration):
- duration_ms = duration * 1e3
- duration_us = duration * 1e6
- tput = f'{tput / 1e9:.3f}'
- busbw = f'{busbw /1e9:.3f}'
-
- if duration_us < 1e3 or args.raw:
- duration = f'{duration_us:.3f}'
- if not args.raw:
- duration += ' us'
- else:
- duration = f'{duration_ms:.3f} ms'
- return tput, busbw, duration
-
-
-def sync_all():
- get_accelerator().synchronize()
- dist.barrier()
-
-
-def max_numel(comm_op, dtype, mem_factor, local_rank, args):
- dtype_size = _element_size(dtype)
- max_memory_per_gpu = get_accelerator().total_memory(local_rank) * mem_factor
- if comm_op == 'all_reduce' or comm_op == 'pt2pt' or comm_op == 'broadcast':
- elements_per_gpu = int(max_memory_per_gpu // dtype_size)
- elif comm_op == 'all_gather':
- # all_gather performance is lower for non-powers of two, and the output buffer size scales with world size
- # Therefore, divide by world size and round down to nearest power of 2
- elements_per_gpu = int(max_memory_per_gpu // dtype_size // dist.get_world_size())
- elements_per_gpu = int(pow(2, int(math.log(elements_per_gpu, 2))))
- elif comm_op == 'all_to_all':
- # Number of elements must be divisible by world_size
- # all_to_all performance is lower for non-powers of two. Round down like all_gather.
- elements_per_gpu = int(max_memory_per_gpu // dtype_size)
- elements_per_gpu = int(dist.get_world_size() *
- round(elements_per_gpu / dist.get_world_size()))
- elements_per_gpu = int(pow(2, int(math.log(elements_per_gpu, 2))))
- else:
- print(f"This communication operation: {comm_op} is not supported yet")
- exit(0)
- return elements_per_gpu
-
-
-# Helper function to pretty-print message sizes
-def convert_size(size_bytes):
- if size_bytes == 0:
- return "0B"
- size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
- i = int(math.floor(math.log(size_bytes, 1024)))
- p = math.pow(1024, i)
- s = round(size_bytes / p, 2)
- return "%s %s" % (s, size_name[i])
-
-
-# Copied from torch. Need to add the func here for old torch compatibility.
-def _element_size(dtype):
- """
- Returns the element size for a dtype, in bytes
- """
- if not isinstance(dtype, torch.dtype):
- raise RuntimeError(f'expected torch.dtype, but got {type(dtype)}')
-
- if dtype.is_complex:
- return torch.finfo(dtype).bits >> 2
- elif dtype.is_floating_point:
- return torch.finfo(dtype).bits >> 3
- elif dtype == torch.bool:
- # NOTE: torch.bool is not supported in torch.iinfo()
- return 1
- else:
- return torch.iinfo(dtype).bits >> 3
-
-
-def benchmark_parser():
- parser = argparse.ArgumentParser()
- parser.add_argument("--local_rank", type=int)
- parser.add_argument("--trials",
- type=int,
- default=DEFAULT_TRIALS,
- help='Number of timed iterations')
- parser.add_argument("--warmups",
- type=int,
- default=DEFAULT_WARMUPS,
- help='Number of warmup (non-timed) iterations')
- parser.add_argument("--maxsize",
- type=int,
- default=24,
- help='Max message size as a power of 2')
- parser.add_argument("--async-op",
- action="store_true",
- help='Enables non-blocking communication')
- parser.add_argument("--bw-unit",
- type=str,
- default=DEFAULT_UNIT,
- choices=['Gbps',
- 'GBps'])
- parser.add_argument("--backend",
- type=str,
- default=DEFAULT_BACKEND,
- choices=['nccl',
- 'ccl'],
- help='Communication library to use')
- parser.add_argument("--dist",
- type=str,
- default=DEFAULT_DIST,
- choices=['deepspeed',
- 'torch'],
- help='Distributed DL framework to use')
- parser.add_argument("--scan",
- action="store_true",
- help='Enables scanning all message sizes')
- parser.add_argument("--raw",
- action="store_true",
- help='Print the message size and latency without units')
- parser.add_argument("--all-reduce", action="store_true", help='Run all_reduce')
- parser.add_argument("--all-gather", action="store_true", help='Run all_gather')
- parser.add_argument("--all-to-all", action="store_true", help='Run all_to_all')
- parser.add_argument("--pt2pt", action="store_true", help='Run pt2pt')
- parser.add_argument("--broadcast", action="store_true", help='Run broadcast')
- parser.add_argument("--dtype",
- type=str,
- default=DEFAULT_TYPE,
- help='PyTorch tensor dtype')
- parser.add_argument(
- "--mem-factor",
- type=float,
- default=.4,
- help='Proportion of max available GPU memory to use for single-size evals')
- parser.add_argument("--debug",
- action="store_true",
- help='Enables all_to_all debug prints')
- return parser
diff --git a/benchmarks/inference/bert-bench.py b/benchmarks/inference/bert-bench.py
deleted file mode 100644
index 9d586d033cd7..000000000000
--- a/benchmarks/inference/bert-bench.py
+++ /dev/null
@@ -1,92 +0,0 @@
-'''Copyright The Microsoft DeepSpeed Team'''
-
-import torch
-import time
-import deepspeed
-import argparse
-from transformers import pipeline
-from deepspeed.accelerator import get_accelerator
-
-parser = argparse.ArgumentParser()
-parser.add_argument("--model", "-m", type=str, help="hf model name")
-parser.add_argument("--deepspeed", action="store_true", help="use deepspeed inference")
-parser.add_argument("--dtype", type=str, default="fp16", help="fp16 or fp32")
-parser.add_argument("--max-tokens", type=int, default=50, help="max new tokens")
-parser.add_argument("--local_rank", type=int, default=0, help="local rank")
-parser.add_argument("--trials", type=int, default=30, help="number of trials")
-parser.add_argument("--kernel-inject", action="store_true", help="inject kernels on")
-parser.add_argument("--graphs", action="store_true", help="CUDA Graphs on")
-args = parser.parse_args()
-
-
-def print_latency(latency_set, title, warmup=3):
- # trim warmup queries
- latency_set = latency_set[warmup:]
- count = len(latency_set)
- if count > 0:
- latency_set.sort()
- n50 = (count - 1) * 0.5 + 1
- n90 = (count - 1) * 0.9 + 1
- n95 = (count - 1) * 0.95 + 1
- n99 = (count - 1) * 0.99 + 1
- n999 = (count - 1) * 0.999 + 1
-
- avg = sum(latency_set) / count
- p50 = latency_set[int(n50) - 1]
- p90 = latency_set[int(n90) - 1]
- p95 = latency_set[int(n95) - 1]
- p99 = latency_set[int(n99) - 1]
- p999 = latency_set[int(n999) - 1]
-
- print(f"====== latency stats {title} ======")
- print("\tAvg Latency: {0:8.2f} ms".format(avg * 1000))
- print("\tP50 Latency: {0:8.2f} ms".format(p50 * 1000))
- print("\tP90 Latency: {0:8.2f} ms".format(p90 * 1000))
- print("\tP95 Latency: {0:8.2f} ms".format(p95 * 1000))
- print("\tP99 Latency: {0:8.2f} ms".format(p99 * 1000))
- print("\t999 Latency: {0:8.2f} ms".format(p999 * 1000))
-
-
-deepspeed.init_distributed()
-
-print(args.model, args.max_tokens, args.dtype)
-
-if args.dtype.lower() == "fp16":
- dtype = torch.float16
-else:
- dtype = torch.float32
-
-pipe = pipeline("fill-mask", model=args.model, framework="pt", device=args.local_rank)
-
-if dtype == torch.half:
- pipe.model.half()
-
-mask = pipe.tokenizer.mask_token
-
-br = pipe(f"Hello I'm a {mask} model")
-if args.deepspeed:
- pipe.model = deepspeed.init_inference(pipe.model,
- dtype=dtype,
- mp_size=1,
- replace_with_kernel_inject=args.kernel_inject,
- enable_cuda_graph=args.graphs)
- pipe.model.profile_model_time()
-
-responses = []
-times = []
-mtimes = []
-for i in range(args.trials):
- get_accelerator().synchronize()
- start = time.time()
- r = pipe(f"Hello I'm a {mask} model")
- get_accelerator().synchronize()
- end = time.time()
- responses.append(r)
- times.append((end - start))
- mtimes += pipe.model.model_times()
- #print(f"{pipe.model.model_times()=}")
-
-print_latency(times, "e2e latency")
-print_latency(mtimes, "model latency")
-
-print(responses[0:3])
diff --git a/benchmarks/inference/collect_results.py b/benchmarks/inference/collect_results.py
deleted file mode 100644
index 0e51033114db..000000000000
--- a/benchmarks/inference/collect_results.py
+++ /dev/null
@@ -1,147 +0,0 @@
-'''Copyright The Microsoft DeepSpeed Team'''
-
-import os
-import re
-import argparse
-import pandas as pd
-
-parser = argparse.ArgumentParser()
-parser.add_argument(
- "--results-dir",
- "-r",
- type=str,
- default="./results",
- help="directory containing sweep results",
-)
-parser.add_argument("--version",
- "-v",
- type=int,
- default=0,
- help="version to be collected")
-parser.add_argument("--gen-text-n",
- "-n",
- type=int,
- default=1,
- help="expected number of generated text")
-parser.add_argument("--output",
- "-o",
- type=str,
- default="./results.csv",
- help="output file")
-args = parser.parse_args()
-
-
-def get_branch(file_path):
- match = re.match(r".*\/(.*)\.log", file_path)
- if match is None:
- return False
- else:
- return match.groups()[0]
-
-
-def get_benchmark_params(root_dir, file_path):
- match = re.match(
- rf"{root_dir}\/(.+?)_(fp\d+)_(true|false)_(true|false)_(\d+)gpus_v(\d+)\/",
- file_path,
- )
- if match is None:
- return False
- else:
- model, dtype, graphs, kernel, gpus, version = match.groups()
- bool_dict = {"true": True, "false": False}
- return {
- "model": model,
- "dtype": dtype,
- "graphs": bool_dict[graphs.lower()],
- "kernel": bool_dict[kernel.lower()],
- "gpus": int(gpus),
- "version": int(version),
- }
-
-
-def get_perf_data(file_content):
- matches = re.findall(r"\s+(.+?)\sLatency:\s+(\d+\.\d+)\sms", file_content)
- if matches is []:
- return False
- else:
- return {f"latency-{key}": float(val) for key, val in matches}
-
-
-def get_generated_text(file_content, gen_text_n):
- file_content = file_content.replace("\n", " ")
- file_content = file_content.replace("\t", " ")
- matches = re.findall(r"RESPONSE\s(\d+):\s+[-]{30}\s+(.+?)\s+[-]{30}", file_content)
- if len(matches) != gen_text_n:
- return False
- else:
- return {f"generated-text-{key}": val for key, val in matches}
-
-
-def get_error(file_content):
- matches = re.findall(r"Error:\s+(.+?)\n", file_content)
- if matches is []:
- return False
- else:
- return {f"error": val for val in matches}
-
-
-if __name__ == "__main__":
- # List to collect data from all benchmarks
- benchmarks_data = []
-
- # Walk through directory of results from sweep.sh
- for root, dirs, files in os.walk(args.results_dir):
- # Because of how some models are named, the dir structure for results can vary, e.g.:
- # "EleutherAI/gpt-neo_*/baseline.log" versus "gpt2_*/baseline.log"
- if dirs:
- continue
-
- # Get data from baseline and each tested branch
- for name in files:
- file_path = os.path.join(root, name)
-
- branch = get_branch(file_path)
- if not branch:
- print(f"WARNING: Could not detect branch for file {file_path}, skipping")
- continue
-
- params = get_benchmark_params(args.results_dir, file_path)
- if not params:
- print(
- f"WARNING: Could not detect benchmark settings for file {file_path}, skipping"
- )
- continue
-
- # Verify that the version matches that which we want to collect
- if params["version"] != args.version:
- continue
-
- with open(file_path, "r") as f:
- file_content = f.read()
-
- perf_data = get_perf_data(file_content)
- if not perf_data:
- print(
- f"WARNING: Could not detect benchmark performance data for file {file_path}"
- )
-
- generated_text = get_generated_text(file_content, args.gen_text_n)
- if not generated_text:
- print(f"WARNING: Could not detect generated text for file {file_path}")
-
- error = get_error(file_content)
- if error:
- print(f"Error found in {file_path}, collecting error info...")
- benchmarks_data.append({"branch": branch, **params, **error})
- continue
-
- benchmarks_data.append({
- "branch": branch,
- **params,
- **perf_data,
- **generated_text
- })
-
- # Convert to a DataFrame and save
- benchmarks_df = pd.DataFrame(benchmarks_data)
- benchmarks_df.to_csv(args.output)
diff --git a/benchmarks/inference/gpt-bench.py b/benchmarks/inference/gpt-bench.py
deleted file mode 100644
index 29578b30cf1f..000000000000
--- a/benchmarks/inference/gpt-bench.py
+++ /dev/null
@@ -1,124 +0,0 @@
-'''Copyright The Microsoft DeepSpeed Team'''
-
-import os
-import torch
-import time
-import deepspeed
-import argparse
-from transformers import pipeline
-from deepspeed.accelerator import get_accelerator
-
-parser = argparse.ArgumentParser()
-parser.add_argument("--model", "-m", type=str, help="hf model name")
-parser.add_argument("--deepspeed", action="store_true", help="use deepspeed inference")
-parser.add_argument("--dtype",
- type=str,
- default="fp16",
- choices=["fp16",
- "fp32",
- "int8"],
- help="int8, fp16, or fp32")
-parser.add_argument("--graphs", action="store_true", help="CUDA Graphs on")
-parser.add_argument("--kernel-inject", action="store_true", help="inject kernels on")
-parser.add_argument("--max-tokens", type=int, default=50, help="max new tokens")
-parser.add_argument("--local_rank",
- type=int,
- default=int(os.getenv("LOCAL_RANK",
- "0")),
- help="local rank")
-parser.add_argument("--world_size",
- type=int,
- default=int(os.getenv("WORLD_SIZE",
- "1")),
- help="world size")
-parser.add_argument("--trials", type=int, default=30, help="number of trials")
-args = parser.parse_args()
-
-
-def print_latency(latency_set, title, warmup=3):
- # trim warmup queries
- latency_set = list(latency_set)
- latency_set = latency_set[warmup:]
- count = len(latency_set)
- if count > 0:
- latency_set.sort()
- n50 = (count - 1) * 0.5 + 1
- n90 = (count - 1) * 0.9 + 1
- n95 = (count - 1) * 0.95 + 1
- n99 = (count - 1) * 0.99 + 1
- n999 = (count - 1) * 0.999 + 1
-
- avg = sum(latency_set) / count
- p50 = latency_set[int(n50) - 1]
- p90 = latency_set[int(n90) - 1]
- p95 = latency_set[int(n95) - 1]
- p99 = latency_set[int(n99) - 1]
- p999 = latency_set[int(n999) - 1]
-
- print(f"====== latency stats {title} ======")
- print("\tAvg Latency: {0:8.2f} ms".format(avg * 1000))
- print("\tP50 Latency: {0:8.2f} ms".format(p50 * 1000))
- print("\tP90 Latency: {0:8.2f} ms".format(p90 * 1000))
- print("\tP95 Latency: {0:8.2f} ms".format(p95 * 1000))
- print("\tP99 Latency: {0:8.2f} ms".format(p99 * 1000))
- print("\t999 Latency: {0:8.2f} ms".format(p999 * 1000))
-
-
-deepspeed.init_distributed()
-
-if args.local_rank == 0:
- print("BENCHMARK SETTINGS:")
- print(f"\tMODEL: {args.model}")
- print(f"\tMAX_TOKENS: {args.max_tokens}")
- print(f"\tDTYPE: {args.dtype}")
- print(f"\tCUDA_GRAPHS: {args.graphs}")
- print(f"\tKERNEL_INJECT: {args.kernel_inject}")
-
-if args.dtype == "int8":
- dtype = torch.int8
-elif args.dtype == "fp16":
- dtype = torch.float16
-else:
- dtype = torch.float32
-
-pipe = pipeline("text-generation",
- model=args.model,
- framework="pt",
- device=args.local_rank)
-
-if dtype == torch.float16:
- pipe.model.half()
-
-if args.deepspeed:
- pipe.model = deepspeed.init_inference(
- pipe.model,
- dtype=dtype,
- mp_size=args.world_size,
- replace_with_kernel_inject=args.kernel_inject,
- enable_cuda_graph=args.graphs,
- )
- pipe.model.profile_model_time()
-
-responses = []
-times = []
-mtimes = []
-for i in range(args.trials):
- get_accelerator().synchronize()
- start = time.time()
- r = pipe("DeepSpeed is", do_sample=False, max_new_tokens=args.max_tokens)
- get_accelerator().synchronize()
- end = time.time()
- responses.append(r)
- times.append(end - start) # / (args.max_tokens - 3))
- mtimes.append(sum(pipe.model.model_times()))
-
-if args.local_rank == 0:
- print_latency(times, "(e2e) latency")
- print_latency(mtimes, "(model-only) latency")
- print_latency(map(lambda t: t / (args.max_tokens - 3),
- times),
- "(e2e) per token latency")
- print(f"RESPONSE 0:")
- print("-" * 30)
- print(responses[0][0]["generated_text"])
- print("-" * 30)
diff --git a/benchmarks/inference/requirements.txt b/benchmarks/inference/requirements.txt
deleted file mode 100644
index 00899dd5f485..000000000000
--- a/benchmarks/inference/requirements.txt
+++ /dev/null
@@ -1 +0,0 @@
-transformers>=4.21.3
diff --git a/benchmarks/inference/run_model.sh b/benchmarks/inference/run_model.sh
deleted file mode 100644
index 8e5fe3ac0133..000000000000
--- a/benchmarks/inference/run_model.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-set -x
-
-model=$1
-branch1=$2
-branch2=$3
-dtype=$4
-graphs=$5
-kernel=$6
-gpus=$7
-
-version=0
-log_path=results/${model}_${dtype}_${graphs}_${kernel}_${gpus}gpus_v${version}
-mkdir -p ${log_path}
-
-params="--dtype $dtype "
-if [[ "$graphs" == "true" ]]; then
- params+="--graphs "
-fi
-if [[ "$kernel" == "true" ]]; then
- params+="--kernel "
-fi
-
-echo "baseline $log_path"
-deepspeed --num_gpus 1 gpt-bench.py -m "${model}" $params &> ${log_path}/baseline.log
-
-cd ../../
-git checkout ${branch1}
-cd -
-echo "ds ${branch1} $log_path"
-deepspeed --num_gpus $gpus gpt-bench.py --deepspeed -m "${model}" $params &> ${log_path}/ds-${branch1}.log
-
-cd ../../
-git checkout ${branch2}
-cd -
-echo "ds ${branch2} $log_path"
-deepspeed --num_gpus $gpus gpt-bench.py --deepspeed -m "${model}" $params&> ${log_path}/ds-${branch2}.log
diff --git a/benchmarks/inference/sweep.sh b/benchmarks/inference/sweep.sh
deleted file mode 100644
index aabcb0bfdbd8..000000000000
--- a/benchmarks/inference/sweep.sh
+++ /dev/null
@@ -1,41 +0,0 @@
-set -x
-
-export TRANSFORMERS_CACHE=/tmp/hf-cache
-
-branch1=$1
-branch2=$2
-
-gptneo_models="EleutherAI/gpt-neo-2.7B EleutherAI/gpt-neo-1.3B EleutherAI/gpt-neo-125M"
-gpt2_models="gpt2 gpt2-large gpt2-xl"
-gptj_models="EleutherAI/gpt-j-6B"
-opt_models="facebook/opt-125m facebook/opt-1.3b facebook/opt-2.7b facebook/opt-6.7b facebook/opt-13b"
-bloom_models="bigscience/bloom-560m bigscience/bloom-1b7 bigscience/bloom-3b bigscience/bloom-7b1"
-
-for gpus in `echo "1 2 4 8"`; do
- for dtype in `echo "fp16 fp32"`; do
- for graphs in `echo "true false"`; do
- for kernel in `echo "true false"`; do
- params="$dtype $graphs $kernel $gpus"
- for m in `echo "$gptneo_models"`; do
- bash run_model.sh $m $branch1 $branch2 $params
- done
-
- for m in `echo "$gpt2_models"`; do
- bash run_model.sh $m $branch1 $branch2 $params
- done
-
- for m in `echo "$gptj_models"`; do
- bash run_model.sh $m $branch1 $branch2 $params
- done
-
- for m in `echo "$opt_models"`; do
- bash run_model.sh $m $branch1 $branch2 $params
- done
-
- for m in `echo "$bloom_models"`; do
- bash run_model.sh $m $branch1 $branch2 $params
- done
- done
- done
- done
-done
diff --git a/bin/ds_elastic b/bin/ds_elastic
index c9987d4565da..1c78aea88894 100755
--- a/bin/ds_elastic
+++ b/bin/ds_elastic
@@ -9,11 +9,7 @@ from deepspeed.elasticity import compute_elastic_config
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('-c', '--config', type=str, help="DeepSpeed config json")
- parser.add_argument('-w',
- '--world-size',
- type=int,
- default=0,
- help="Intended/current world size")
+ parser.add_argument('-w', '--world-size', type=int, default=0, help="Intended/current world size")
args = parser.parse_args()
ds_config = json.load(open(args.config, 'r'))
@@ -26,7 +22,9 @@ if __name__ == '__main__':
print(json.dumps(elastic_config, indent=4, sort_keys=True))
if args.world_size > 0:
- final_batch_size, valid_gpus, micro_batch_size = compute_elastic_config(ds_config=ds_config, target_deepspeed_version=ds_version, world_size=args.world_size)
+ final_batch_size, valid_gpus, micro_batch_size = compute_elastic_config(ds_config=ds_config,
+ target_deepspeed_version=ds_version,
+ world_size=args.world_size)
print('------------------------------------------')
print(f"Calculated results for world size {args.world_size}:")
print('------------------------------------------')
diff --git a/blogs/README.md b/blogs/README.md
new file mode 100644
index 000000000000..c5c72fd4efb0
--- /dev/null
+++ b/blogs/README.md
@@ -0,0 +1 @@
+All DeepSpeed blogs are linked here:
diff --git a/blogs/assets/images/Throughputs-OPT.png b/blogs/assets/images/Throughputs-OPT.png
new file mode 100755
index 000000000000..9b968319f759
Binary files /dev/null and b/blogs/assets/images/Throughputs-OPT.png differ
diff --git a/blogs/assets/images/ds-chat-overview.png b/blogs/assets/images/ds-chat-overview.png
new file mode 100755
index 000000000000..479ca4bb2ad7
Binary files /dev/null and b/blogs/assets/images/ds-chat-overview.png differ
diff --git a/blogs/assets/images/figure3.png b/blogs/assets/images/figure3.png
new file mode 100755
index 000000000000..44b8e67bd6b9
Binary files /dev/null and b/blogs/assets/images/figure3.png differ
diff --git a/blogs/assets/images/figure4.png b/blogs/assets/images/figure4.png
new file mode 100755
index 000000000000..dca56637049c
Binary files /dev/null and b/blogs/assets/images/figure4.png differ
diff --git a/blogs/assets/images/figure5.png b/blogs/assets/images/figure5.png
new file mode 100755
index 000000000000..6282c0d19ed1
Binary files /dev/null and b/blogs/assets/images/figure5.png differ
diff --git a/blogs/assets/images/figure6.png b/blogs/assets/images/figure6.png
new file mode 100755
index 000000000000..8e60773b5709
Binary files /dev/null and b/blogs/assets/images/figure6.png differ
diff --git a/blogs/assets/images/figure7.png b/blogs/assets/images/figure7.png
new file mode 100755
index 000000000000..9b400b11efb0
Binary files /dev/null and b/blogs/assets/images/figure7.png differ
diff --git a/blogs/assets/images/hero-figure-black-ja.png b/blogs/assets/images/hero-figure-black-ja.png
new file mode 100644
index 000000000000..5c6cf05d3e7a
Binary files /dev/null and b/blogs/assets/images/hero-figure-black-ja.png differ
diff --git a/blogs/assets/images/hero-figure-black.png b/blogs/assets/images/hero-figure-black.png
new file mode 100755
index 000000000000..6a05f4b27bb9
Binary files /dev/null and b/blogs/assets/images/hero-figure-black.png differ
diff --git a/blogs/assets/images/hybrid-engine.png b/blogs/assets/images/hybrid-engine.png
new file mode 100755
index 000000000000..5e501108a5b7
Binary files /dev/null and b/blogs/assets/images/hybrid-engine.png differ
diff --git a/blogs/assets/images/mascot.png b/blogs/assets/images/mascot.png
new file mode 100755
index 000000000000..e9f7a354fc85
Binary files /dev/null and b/blogs/assets/images/mascot.png differ
diff --git a/blogs/assets/images/triton-bert-base-latency.png b/blogs/assets/images/triton-bert-base-latency.png
new file mode 100644
index 000000000000..4f733f4d1afe
Binary files /dev/null and b/blogs/assets/images/triton-bert-base-latency.png differ
diff --git a/blogs/assets/images/triton-bert-large-latency.png b/blogs/assets/images/triton-bert-large-latency.png
new file mode 100644
index 000000000000..d82dc0ccac51
Binary files /dev/null and b/blogs/assets/images/triton-bert-large-latency.png differ
diff --git a/blogs/comm-opt/README.md b/blogs/comm-opt/README.md
new file mode 100644
index 000000000000..4767c4342816
--- /dev/null
+++ b/blogs/comm-opt/README.md
@@ -0,0 +1,82 @@
+
+
+# Communication Optimizations for Large-Scale Training
+
+
+
+
+## Table of Contents
+1. [Introduction](#introduction)
+2. [Gradient AllReduce Optimization for ZeRO stages 1 and 2](#ar-opt)
+3. [Optimizing Parameter All-Gather for ZeRO2 Training](#ag-opt)
+4. [Optimizing AlltoAll for Sequence-Parallel Training](#sp-opt)
+
+
+## 1. Introduction
+Training LLMs on large datasets can be extremely costly both in terms of hardware resources and time. An important step to minimize such costs is to carefully combine an appropriate number of resources together with a scalable library that guarantees training completion within a time limit. In this post, we discuss a key aspect of the scalability features of DeepSpeed, the communication optimization. Communication collectives (e.g., all-reduce, all-gather, etc.) are critical pieces of many popular DeepSpeed technologies (e.g., ZeRO, MoE, AutoTP, etc.), and in the following sections we discuss our new optimizations of some of these collectives. These optimizations are available in DeepSpeed versions >= 0.x.x.
+
+## 2. Gradient AllReduce Optimization for ZeRO stages 1 and 2
+
+Before diving into this optimization, let's take a step back and show some of the case studies that demonstrate the need.
+
+AllReduce operation is an important part of the training process. In ZeRO, we handle this in buckets, which can be configured to get good communication throughput. As the number of GPUs increases, we encounter smaller-partition AllReduces. In this case, the current bucketing scheme cannot help with the communication overhead. This mostly becomes an issue when training smaller-scale models (like Llama-7B) with large number of GPUs.
+
+For instance, when training a dense-7B architecture with Zero stages 1 or 2, we encounter a 1 and 2 second increase for the AllReduce time by increasing from 256 to 512 and 1024 A100 GPUs. This issue mostly arises from the fact that, the gradient-averaging happens with smaller partitions (#parameters / #GPUs) per-GPU rank. This issue gets more serious when training MoE architectures (3 - 12 second) for which the expert's parameters can be farther away due to the current parallelism layout of data and expert parallelism.
+
+In this section, we introduce two main optimization techniques for alleviating these communication bottleneck.
+
+First, Multi-rank bucketing for the same process group: for this optimization, we simply pack all data that requires to be reduced from different ranks into one big flattened tensor and call AllReduce instead of reduce operations. After the reduction, we scatter the right portion of data to the corresponding ranks.
+
+Second, add new layout for the expert-data parallelism: the default parallelism layout for MoE architecture (as shown in Fig 1) is planned in a way that the experts are placed first on E parallel GPUs and replicated D times (data-parallel). With this layout, we encounter slower AllReduce as data-parallel ranks are placed farther away especially when we have cross-rank communication. We call this layout E + D.
+
+
+
+
+ *Fig 1: Different MoE parallel layout. left) E + D, which places the GPUs in EP dimension first before adding DP, right) D + E, that replicates each expert by DP size, before constructing EP. We get faster AllReduce for the second layout while increasing the AlltoAll time. It potentially results in faster e2e training time, as the communication volume for AllReduce (total parameter size) is normally much more than AlltoAll (MLP activation memory).*
+
+
+| | GPUs | AllReduce time | Iteration time |
+|----------|:------:|:------:|:------:|
+baseline (dense) | 1024| 1.2 | 5.4
+optimized (dense) | 1024| 0.36 | 4.5
+baseline (MoE) | 1024 | 11.5 | 16.1
+optimized (MoE) | 1024 | 0.45 | 5.1
+
+Table 1. AllReduce saving observed for both dense and MoE architectures.
+
+
+
+## 3. Optimizing Parameter All-Gather for ZeRO2 Training
+
+The same as with AllReduce, all-gather takes longer as we have more partitions. As the parameters are stored in a flattened buffer for ZeRO stage-2, we can simply have a one call to all-gather the parameters into this tensor.
+
+When all-gathering the updated parameters at Zero-Stage2, the bucketing scheme uses several narrow operations and creates a list of tensors with the bucket size from each partition. We needed this scheme to align with the `all_gather` operation from PyTorch.
+However, by adding the support for the `all_gather_into_tensor`, operation that has been added to the newer versions of PyTorch, we can simply have a kernel call to do the full-parameter all-gather. With this optimization, we see about 2x reduction in the step time for large-scale training.
+
+## 4. Optimizing AlltoAll for Sequence-Parallel Training
+
+For this part of the optimization, we add some fusion for the communication that is required for the DeepSpeed-Ulysses to provide a more scalable approach for when we increase the SP from 2 to 8 (for this study, we consider A100-DGX hardware, which has 8 GPUs per-node and by increasing the parallelism more than 8, we encounter performance-hit by the cross-node communication).
+
+These fusions are done at two levels:
+1. Fuse the sequence AlltoAll for q,k, and v: we Scatter the heads using the mixed tensor rather than splitting them beforehand. For this part, we need to get some more information from the modeling side (such as the number of q and kv heads), to split the heads before calling AlltoAll. We have added some new changes on the Megatron-DeepSpeed repo that incorporate these changes for the sequence-parallelism.
+2. Fuse the AlltoAll tensors and call the PyTorch's AlltoAll-sinlge API: we reshape the tensors for the scatter dimension and use a single tensor for AlltoAll which alleviates the overhead of using a list of tensors which requires a contiguous call for each element of the list.
+
+By adding these optimizations, we see about 10 to 15% speedup compared to the previous design, and obtain good scalability across different SP-degree and context-lengths. In the following table, we show the improvement achieved by using SP, when doubling the GPU-count and increasing the SP-degree. We obtain over 80% of efficiency when increasing from 256 to 512 GPUs using SP-2. Furthermore, by increasing the sequence-length and SP, while keeping the processed tokens similar, we achieve over 75% of efficiency for 2x more resources. On the other hand, if we can double the number of tokens (shown on the last row of table 2), we can improve the performance to 1.81x.
+
+
+
+| GPUs | bsz | seq | Tokens (M) | SP | Sample (4K)-per-second | Speedup (x) |
+|----------|:------:|:------:|:------:|:------:|:------:|:------:|
+256 | 256| 8192 |2|1 | 60.71 |1
+512 | 256| 8192 |2|2 | 111.18 | 1.83
+512 | 128| 16384 |2|4 | 108.81 | 1.79
+512 | 64 |32768 |2|8 | 106.54 | 1.75
+512 | 64 |65536 |4|8 | 110.05 | 1.81
+
+Table 2. Sequence-Parallelism scalability using DeepSpeed-Ulysses.
+
+
diff --git a/blogs/comm-opt/assets/images/e+d.png b/blogs/comm-opt/assets/images/e+d.png
new file mode 100644
index 000000000000..72ad0f583857
Binary files /dev/null and b/blogs/comm-opt/assets/images/e+d.png differ
diff --git a/blogs/comm-opt/assets/images/sp+fp.png b/blogs/comm-opt/assets/images/sp+fp.png
new file mode 100644
index 000000000000..0b2940418f7a
Binary files /dev/null and b/blogs/comm-opt/assets/images/sp+fp.png differ
diff --git a/blogs/comm-opt/assets/images/sp-conv.png b/blogs/comm-opt/assets/images/sp-conv.png
new file mode 100644
index 000000000000..e1e36b4436a0
Binary files /dev/null and b/blogs/comm-opt/assets/images/sp-conv.png differ
diff --git a/blogs/deepspeed-chat/README.md b/blogs/deepspeed-chat/README.md
new file mode 100644
index 000000000000..43501652b98e
--- /dev/null
+++ b/blogs/deepspeed-chat/README.md
@@ -0,0 +1,332 @@
+
+
+# DeepSpeed Chat: Easy, Fast and Affordable RLHF Training of ChatGPT-like Models at All Scales
+
+
+
+
+
+
+
+
+
+| GPUs | OPT-6.7B | OPT-13B | OPT-30B | OPT-66B |
+|-------------|:--------:|:--------------:|:-------------:|:-----------:|
+| 8x A100-40GB | 5.7 hours | 10.8 hours | 1.85 days | NA |
+| 8x A100-80GB | 4.1 hours ($132) | 9 hours ($290) | 18 hours ($580) | 2.1 days ($1620) |
+
+*Table 1. Single-Node 8x A100: Training Time and Corresponding Approximate Cost on Azure.*
+
+
+
+
+***Excellent Scalability***: DeepSpeed-HE supports models with hundreds of billions of parameters and can achieve excellent scalability on multi-node multi-GPU systems. As a result, even a 13B model can be trained in 1.25 hours and a massive 175B model can be trained with DeepSpeed-HE in under a day.
+
+
+
+
+| GPUs | OPT-13B | OPT-30B | OPT-66B | OPT-175B |
+|---------------|:-----------------:|:---------------:|:-------------:|:-------------:|
+| 64x A100-80G | 1.25 hours ($320) | 4 hours ($1024) | 7.5 hours ($1920) | 20 hours ($5120)|
+
+*Table 2. Multi-Node 64x A100-80GB: Training Time and Corresponding Approximate Cost on Azure.*
+
+
+> ***Very Important Details***: The numbers in both tables above are for Step 3 of the training and are based on actual measured training throughput on DeepSpeed-RLHF curated dataset and training recipe which trains for one epoch on a total of 135M tokens. We have in total 67.5M query tokens (131.9k queries with sequence length 256) and 67.5M generated tokens (131.9k answers with sequence length 256), and a maximum global batch size per step of 0.5M tokens (1024 query-answer pairs). We urge readers to pay attention to these specifications before making any cost and e2e time comparisons with DeepSpeed-RLHF. See our [benchmark settings](https://github.com/microsoft/DeepSpeedExamples/tree/master/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/BenckmarkSetting.md) page for more details.
+
+
+***Democratizing RLHF Training***: With just a single GPU, DeepSpeed-HE supports training models with over 13 billion parameters, enabling data scientists without access to multi-GPU systems to create not just toy RLHF models but large and powerful ones that can be used in real-world scenarios.
+
+
+
+| | V100 32G | A6000 48G | A100 40G | A100 80G |
+|------------|:---------:|:----------:|:--------:|:---------:|
+| Model Size | OPT-2.7B | OPT-6.7B | OPT-6.7B | OPT-13B |
+
+Table 3. Max Model Size Supported by DeepSpeed-HE on a Single GPU
+
+
+
+Next, we dive deeper into the three capabilities of DeepSpeed-Chat introduced above.
+
+# 2. Easy-to-use ChatGPT Training and Inference Experience
+
+We start with the easy-to-use experience by showing how you can train OPT-13B and then OPT-66B models with DeepSpeed-RLHF system. If you are short on time, you can even train an OPT-1.3B model on a single consumer-grade GPU in just two hours. We also demonstrate how you can use our DeepSpeed-chat RLHF API to develop your own custom pipelines.
+
+## Training your first ChatGPT-Style model is so easy with DeepSpeed-Chat’s RLHF examples
+
+*a) One single script completes all three stages of RLHF training and generates your first ChatGPT model!*
+
+We use an example of pretrained OPT-13B as the actor model and OPT-350M as the reward model in the following single script to generate a final 13B ChatGPT-style model:
+
+```python
+pip install deepspeed>=0.9.0
+
+git clone https://github.com/microsoft/DeepSpeedExamples.git
+cd DeepSpeedExamples/applications/DeepSpeed-Chat/
+pip install -r requirements.txt
+
+python train.py --actor-model facebook/opt-13b --reward-model facebook/opt-350m --deployment-type single_node
+```
+
+In about half a day, your 13B model would be fully trained with its checkpoints ready. The following table demonstrates a breakdown of the training time for each of the three steps:
+
+
+
+| Model Sizes | Step 1 | Step 2 | Step 3 | Total |
+|--------------------------------- |:------:|:------:|:------:|:------:|
+| Actor: OPT-13B, Reward: OPT-350M | 2.5hr | 0.25hr | 10.8hr | 13.6hr |
+
+Table 4. E2E time breakdown for training a 13 billion parameter ChatGPT model via DeepSpeed-Chat on a single DGX node with 8 NVIDIA A100-40G GPUs.
+
+
+
+*b) Test your final model with DeepSpeed-Chat’s Inference API*
+
+Now it is time to plug-in and test your final model trained by DeepSpeed-Chat! You can play with not only Q&A sessions but also conversation-style interactions.
+
+```
+Human: Do you know Microsoft?
+Assistant: Yes, I’m a big fan of Microsoft. What about them?
+Human: Can you explain it to a 6-year old child? I wonder how I should describe it
+Assistant: Sure, I can try. Microsoft is a company that makes computers, and they make a program called “Windows” which
+ is the operating system that runs on the computer. It’s like the “brain” of the computer. It’s where all the
+ programs and files are stored. It’s also where you’ll find the “start” button, which is where you’ll start to
+ use the computer.
+```
+
+## Want to try different model sizes and configurations? You got it!
+We understand users often like to play with different model sizes and configurations to meet their training time, resources, and quality requirements. With DeepSpeed-Chat, users can easily do that. For example, if you want to train a larger and higher-quality model on your GPU cluster for your research or business, you can simply use the same script with your desired model size e.g., 66B and GPU counts e.g., 64 GPUs:
+
+```python
+python train.py --actor-model facebook/opt-66b --reward-model facebook/opt-350m --deployment-type multi_node
+```
+
+Within 9 hours, you can have your 66 billion parameters ChatGPT model ready to be served in your favorite front-end GUI:
+
+
+
+
+| Model Sizes | Step 1 | Step 2 | Step 3 | Total |
+|---------------------------------- |:-------:|:------:|:------:|:------:|
+| Actor: OPT-66B, Reward: OPT-350M | 82 mins | 5 mins | 7.5hr | 9hr |
+
+Table 5. E2E time breakdown for training a 66 billion parameter ChatGPT model via DeepSpeed-Chat on 8 DGX nodes with 8 NVIDIA A100-80G GPUs/node.
+
+
+
+If you only have around 1-2 hours for coffee or lunch break, you can also try to train a small/toy model with DeepSpeed-Chat. For example, we prepared a training example for a 1.3B model with a single dataset to test our framework on your consumer-grade GPUs. The best part is that you will have your model checkpoint ready to play with when you are back from your lunch break!
+
+```python
+python train.py --actor-model facebook/opt-1.3b --reward-model facebook/opt-350m --deployment-type single_gpu
+```
+
+
+
+| Model Sizes | Step 1 | Step 2 | Step 3 | Total |
+|--------------------------------- |:---------:|:--------:|:------:|:------:|
+| Actor: OPT-1.3B, Reward: OPT-350M | 2900 secs | 670 secs | 1.2hr | 2.2hr |
+
+*Table 6. E2E time breakdown for training a 1.3 billion parameter ChatGPT model via DeepSpeed-Chat on a single commodity NVIDIA A6000 GPU with 48GB memory.*
+
+
+
+## Customizing your own RLHF training pipeline using DeepSpeed-Chat’s RLHF APIs
+DeepSpeed-Chat allows users to build their very own RLHF training pipeline using our flexible APIs shown below, which users can use to reconstruct their own RLHF training strategy. This enables a general interface and backend for creating a wide range of RLHF algorithms for research exploration.
+
+```python
+
+engine = DeepSpeedRLHFEngine(
+ actor_model_name_or_path=args.actor_model_name_or_path,
+ critic_model_name_or_path=args.critic_model_name_or_path,
+ tokenizer=tokenizer,
+ num_total_iters=num_total_iters,
+ args=args)
+
+trainer = DeepSpeedPPOTrainer(engine=engine, args=args)
+
+for prompt_batch in prompt_train_dataloader:
+ out = trainer.generate_experience(prompt_batch)
+ actor_loss, critic_loss = trainer.train_rlhf(out)
+```
+
+# 3. Full-fledged RLHF Training Pipeline
+
+To provide a seamless training experience, we follow InstructGPT and include a full-fledged end-to-end training pipeline in DeepSpeed-Chat as shown in Figure 1.
+
+
+
+
+
+*Figure 1: The illustration of DeepSpeed-Chat’s RLHF training pipeline with optional features.*
+
+
+
+
+
+*Figure 2. DeepSpeed Hybrid Engine design for accelerating the most time-consuming portion of a RLHF pipeline.*
+
+
+
+
+
+*Figure 3. Step 3 throughput comparison against two other system frameworks for accelerating RLHF \
+training on a single NVIDIA A100-40G commodity GPU. No icons represent OOM scenarios.*
+
+
+
+
+
+*Figure 4. End-to-end training throughput comparison for step 3 of the training pipeline (the most time \
+consuming portion) with different model sizes on a single DGX node equipped with 8 NVIDIA A100-40G GPUs.\
+No icons represent OOM scenarios.*
+
+
+
+
+
+*Figure 5. Superior generation phase acceleration from DeepSpeed Chat’s Hybrid Engine: A time/sequence breakdown for training OPT-1.3B actor model + OPT-350M reward model on a single DGX node with 8 A100-40G GPUs.*
+
+
+
+
+
+*Figure 6. RLHF Generation, training, and effective throughput with DeepSpeed-HE for different model sizes, at the GPU count that maximizes efficiency.*
+
+
+
+
+
+*Figure 7. Scalability for training 13B (left) and 66B (right) actor model+350M reward model on an increasing number of DGX nodes with 8 A100-40/80G GPUs*
+
+
+
+# DeepSpeed Chat: 一键式RLHF训练，让你的类ChatGPT千亿大模型提速省钱15倍
+
+
+
+
+
+
+
+
+
+| GPUs | OPT-6.7B | OPT-13B | OPT-30B | OPT-66B |
+|-------------|:--------:|:--------------:|:-------------:|:-----------:|
+| 8x A100-40GB | 5.7 hours | 10.8 hours | 1.85 days | NA |
+| 8x A100-80GB | 4.1 hours ($132) | 9 hours ($290) | 18 hours ($580) | 2.1 days ($1620) |
+
+*表 1. 单节点 8x A100：训练时长及预估的 Azure 费用。*
+
+
+
+***卓越的扩展性***：DeepSpeed-HE 能够支持训练拥有数千亿参数的模型，并在多节点多 GPU 系统上展现出卓越的扩展性。因此，即使是一个拥有 130 亿参数的模型，也只需 1.25 小时就能完成训练。而对于庞大的 拥有1750 亿参数的模型，使用 DeepSpeed-HE 进行训练也只需不到一天的时间。
+
+
+
+| GPUs | OPT-13B | OPT-30B | OPT-66B | OPT-175B |
+|---------------|:-----------------:|:---------------:|:-------------:|:-------------:|
+| 64x A100-80G | 1.25 hours ($320) | 4 hours ($1024) | 7.5 hours ($1920) | 20 hours ($5120)|
+
+*表 2. 多节点 64x A100-80GB：训练时长及预估的 Azure 费用。*
+
+
+> ***非常重要的细节***: 上述两个表格（即表一和表二）中的数据均针对 RLHF 训练的第 3 步，基于实际数据集和 DeepSpeed-RLHF 训练吞吐量的测试。该训练在总共 1.35 亿（135M）个字符（token）上进行一个时期（epoch）的训练。我们总共有 6750 万个查询（query）字符（131.9k 个 query，每个序列长度为 256）和 6750 万个生成/回答字符（131.9k 个答案，每个序列长度为 256），每步的最大全局字符批量大小约为 500 万个字符（1024 个查询-答案对）。在与 DeepSpeed-RLHF 进行任何成本和端到端时间比较之前，我们建议读者注意这些设定。想要了解更多详细信息，请参阅我们的页面 [benchmark setting](https://github.com/microsoft/DeepSpeedExamples-internal/blob/staging-deepspeed-chat-v2/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/BenckmarkSetting.md)。
+
+***实现 RLHF 训练的普及化***：仅凭单个 GPU，DeepSpeed-HE 就能支持训练超过 130 亿参数的模型。这使得那些无法使用多 GPU 系统的数据科学家和研究者不仅能够轻松创建轻量级的 RLHF 模型，还能创建大型且功能强大的模型，以应对不同的使用场景。
+
+
+
+| | V100 32G | A6000 48G | A100 40G | A100 80G |
+|------------|:---------:|:----------:|:--------:|:---------:|
+| Model Size | OPT-2.7B | OPT-6.7B | OPT-6.7B | OPT-13B |
+
+*表 3. DeepSpeed-HE 在不同的 GPU 单卡上支持的最大模型。*
+
+
+
+接下来，我们将深入了解上面介绍的 DeepSpeed-Chat 的三个功能。
+
+# 2. 简洁高效且经济的 ChatGPT 训练与推理体验
+
+接下来，我们向你展示如何使用一行命令即可利用 DeepSpeed-RLHF 系统训练 OPT-13B 和 OPT-66B 模型。即使时间有限，你也可以在大约两小时内在单个消费级 GPU 上训练一个 OPT-1.3B 模型。此外，我们还演示了如何利用我们的 DeepSpeed-chat RLHF API 来开发你自己的模型：
+
+## 使用 DeepSpeed-Chat 的 RLHF 示例轻松训练你的第一个 类ChatGPT 模型
+
+*a) 仅需一个脚本，完成 RLHF 训练的全部三个阶段，生成你的第一个 ChatGPT 模型！*
+
+以下示例展示了如何使用一个脚本，以预训练的 OPT-13B 作为 actor 模型，OPT-350M 作为 reward 模型，生成一个最终的 130 亿参数的 ChatGPT 类型的模型：
+```
+pip install deepspeed>=0.9.0
+
+git clone https://github.com/microsoft/DeepSpeedExamples.git
+cd DeepSpeedExamples/applications/DeepSpeed-Chat/
+pip install -r requirements.txt
+
+python train.py --actor-model facebook/opt-13b --reward-model facebook/opt-350m --deployment-type single_node
+```
+
+
+大约半天时间，你的 拥有130 亿参数的模型即可完成训练。下表展示了整个过程在三个步骤中所需的时间：
+
+
+
+| Model Sizes | Step 1 | Step 2 | Step 3 | Total |
+|--------------------------------- |:------:|:------:|:------:|:------:|
+| Actor: OPT-13B, Reward: OPT-350M | 2.5hr | 0.25hr | 10.8hr | 13.6hr |
+
+*表 4. 在单个DGX节点上（8x A100-40G），针对不同的RLHF步骤， 使用DeepSpeed-Chat训练OPT-13b所需的时间。*
+
+
+
+*b) 使用 DeepSpeed-Chat 的推理 API 测试训练好的模型*
+
+将训练好的模型导入提供的简易 API，你不仅可以与模型进行问答互动，还可以进行对话式交流。
+
+```
+Human: Do you know Microsoft?
+Assistant: Yes, I’m a big fan of Microsoft. What about them?
+Human: Can you explain it to a 6-year old child? I wonder how I should describe it
+Assistant: Sure, I can try. Microsoft is a company that makes computers, and they make a program called “Windows” which
+ is the operating system that runs on the computer. It’s like the “brain” of the computer. It’s where all the
+ programs and files are stored. It’s also where you’ll find the “start” button, which is where you’ll start to
+ use the computer.
+```
+
+## 想尝试不同的模型大小和配置？轻松实现！
+我们了解到用户通常喜欢尝试不同的模型大小和配置，以满足他们不同的训练时间、资源和质量的需求。借助 DeepSpeed-Chat，你可以轻松实现这些目标。例如，如果你想在 GPU 集群上训练一个更大、更高质量的模型，用于你的研究或业务，你可以使用相同的脚本，只需输入你期望的模型大小（例如 660 亿参数）和 GPU 数量（例如 64 个 GPU）：
+
+```python
+python train.py --actor-model facebook/opt-66b --reward-model facebook/opt-350m --deployment-type multi_node
+```
+
+在 接下来的9 小时内，你将拥有一个 660 亿参数的 ChatGPT 模型，并可以在你喜欢的前端 GUI 中使用：
+
+
+
+| Model Sizes | Step 1 | Step 2 | Step 3 | Total |
+|---------------------------------- |:-------:|:------:|:------:|:------:|
+| Actor: OPT-66B, Reward: OPT-350M | 82 mins | 5 mins | 7.5hr | 9hr |
+
+*表 5. 在8个DGX节点上（64x A100-80G），针对不同的RLHF步骤， 使用DeepSpeed-Chat训练OPT-66b所需的时间。*
+
+
+
+如果你只有大约 1-2 小时的咖啡或午餐休息时间，你也可以尝试使用 DeepSpeed-Chat 训练一个小型模型。例如，我们为单个数据集准备了一个 13 亿参数模型的训练示例，以便在你的消费级 GPU 上测试我们的框架。当你从午餐休息回来时，你的模型已经准备好供你使用！
+
+```python
+python train.py --actor-model facebook/opt-1.3b --reward-model facebook/opt-350m --deployment-type single_gpu
+```
+
+
+
+| Model Sizes | Step 1 | Step 2 | Step 3 | Total |
+|--------------------------------- |:---------:|:--------:|:------:|:------:|
+| Actor: OPT-1.3B, Reward: OPT-350M | 2900 secs | 670 secs | 1.2hr | 2.2hr |
+
+*表 6. 在单个消费级A6000-48G上，针对不同的RLHF步骤， 使用DeepSpeed-Chat训练OPT-1.3b所需的时间。*
+
+
+
+## 利用 DeepSpeed-Chat 的 RLHF API 自定义你自己的 RLHF 训练流程
+DeepSpeed-Chat 允许用户使用我们灵活的 API（如下所示）构建自己的 RLHF 训练流程，用户可以使用这些 API 重建自己的 RLHF 训练策略。我们希望这些功能可以为研究探索中创建各种 RLHF 算法提供通用接口和后端。
+```python
+
+engine = DeepSpeedRLHFEngine(
+ actor_model_name_or_path=args.actor_model_name_or_path,
+ critic_model_name_or_path=args.critic_model_name_or_path,
+ tokenizer=tokenizer,
+ num_total_iters=num_total_iters,
+ args=args)
+
+trainer = DeepSpeedPPOTrainer(engine=engine, args=args)
+
+for prompt_batch in prompt_train_dataloader:
+ out = trainer.generate_experience(prompt_batch)
+ actor_loss, critic_loss = trainer.train_rlhf(out)
+```
+
+# 3. 完整的 RLHF 训练流程概述
+
+为了实现无缝的训练体验，我们遵循 InstructGPT 论文的方法，并在 DeepSpeed-Chat 中整合了一个端到端的训练流程，如图 1 所示。
+
+
+
+
+
+*图 1: DeepSpeed-Chat 的 RLHF 训练流程图示，包含了一些可选择的功能。*
+
+
+
+
+
+*Figure 2. 设计图解：DeepSpeed Hybrid Engine，用于加速 RLHF 流程中最耗时的部分。*
+
+
+
+
+
+*图 3. 在单个 NVIDIA A100-40G GPU 上，将 RLHF 训练的吞吐量与另外两个系统框架在步骤 3 进行比较。没有图标表示 OOM（内存不足）的情况*
+
+
+
+
+
+*图 4. 在单个 DGX 节点上，使用 8 个 NVIDIA A100-40G GPU，对训练流程第 3 步（耗时最长的部分）的不同模型大小进行端到端训练吞吐量比较。没有图标表示 OOM（内存不足）的情况。*
+
+
+
+
+
+*图 5. DeepSpeed Chat 的混合引擎在生成阶段的优越加速：在单个 DGX 节点上使用 8 个 A100-40G GPU 训练 OPT-1.3B actor 模型 + OPT-350M reward 模型的时间/序列分解。*
+
+
+
+
+
+*图 6. 在最大效率的情况下，DeepSpeed-HE 针对不同模型大小的RLHF生成、训练和有效吞吐量。*
+
+
+
+
+
+*图 7. 在不同数量的DGX (A100-40/80G GPU) 节点上，进行13B（左）和66B（右）actor 模型 和 350M reward 模型的可扩展性训练。*
+
+
+
+# DeepSpeed-Chat: Llama/Llama-2 system support, efficiency boost, and training stability improvements
+
+
+
+
+
+
+
+
+ *Figure 1: Up to 7.1x faster Llama-2 generation with DS-Chat Hybrid Engine*
+
+
+
+
+
+ *Figure 2: [Left] 4x faster Llama-2-7B generation with DS-Chat Hybrid Engine (16 V100 GPUs) [Right] 2.1x faster Llama-2-13B generation with DS-Chat Hybrid Engine on 32 V100 GPUS vs. DS-Chat without Hybrid Engine on 16 V100 GPUs.*
+
+
+
+
+ *Figure 3: We achieve 3.3x increased throughput for RLHF training of Llama-2-70B on 128 V100 GPUs using Mixed Precision ZeRO++ vs. ZeRO-3. We obsvered 2x improved throughout for the same experiment on 64 V100 GPUs.*
+
+
+
+
+ *Figure 4: ZeRO-Offload enables us to train Llama-2-7B with 16x fewer GPUs. 16 V100 GPUs are required for training Llama-2-7B with DS-Chat ZeRO-3. Enabling LoRA allows for the number of GPUs to be reduced to 4 while enabling ZeRO-Offload reduces the number of needed GPUs to 1. The HuggingFace Baseline does not run due to memory limitations.*
+
+
+
+
+ *Figure 5: Reward scores for all supported DeepSpeed-Chat configurations with ZeRO-Offload enabled. Run with 16 V100 GPUs, [AdamG012/chat-opt-1.3b-sft-deepspeed](https://huggingface.co/AdamG012/chat-opt-1.3b-sft-deepspeed) actor model, [AdamG012/chat-opt-350m-reward-deepspeed](https://huggingface.co/AdamG012/chat-opt-350m-reward-deepspeed) critic model, DS commit: [f036f00c](https://github.com/microsoft/DeepSpeed/tree/f036f00c3763694e539a9070a98130e2667e49bd), DSE commit: [81a8521f](https://github.com/microsoft/DeepSpeedExamples/tree/81a8521f05e2761eed34fcf65f19873df9f74403).*
+
+
+
+
+ *Figure 6: Step 3 Reward Scores for all supported DeepSpeed-Chat configurations. Run with 16 V100 GPUs, [AdamG012/chat-opt-1.3b-sft-deepspeed](https://huggingface.co/AdamG012/chat-opt-1.3b-sft-deepspeed) actor model, [AdamG012/chat-opt-350m-reward-deepspeed](https://huggingface.co/AdamG012/chat-opt-350m-reward-deepspeed) critic model, DS commit: [f036f00c](https://github.com/microsoft/DeepSpeed/tree/f036f00c3763694e539a9070a98130e2667e49bd), DSE commit: [81a8521f](https://github.com/microsoft/DeepSpeedExamples/tree/81a8521f05e2761eed34fcf65f19873df9f74403).*
+
+
+
+
+ *Figure 7: Before the fix, the sweep across all ZeRO-2 cases without a separate LoRA learning rate shows training instability when LoRA is used.*
+
+
+
+
+ *Figure 8: After creating a separate LoRA learning rate, the sweep across all ZeRO-2 cases shows proper convergence.*
+
+
+DeepSpeedExamples/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning$ bash training_scripts/opt/single_node/sweep/run_step3_sweep.sh
+
+
+The training log for each combination of features will be stored in a folder with the name formatted as `z${z}_he_${he}_offload_${offload}_lora_${lora}`
+
+
+Related PRs:
+
+- [DS Chat Characterization Scripts (Step 1 and 3)](https://github.com/microsoft/DeepSpeedExamples/pull/638)
+- [Add step 2 sweep script, clean up scripts](https://github.com/microsoft/DeepSpeedExamples/pull/664)
+- [Update script location and docs for all 3 steps](https://github.com/microsoft/DeepSpeedExamples/pull/681)
+
+## Instrumentation
+
+To gain better insight into DeepSpeed-Chat training, new [instrumentation features](https://github.com/microsoft/DeepSpeedExamples/pull/624) were added across all three steps of DeepSpeed-Chat and can be enabled via arguments to each step's `main.py`.
+
+| Argument | Description | Step(s) |
+| --- | --- | --- |
+| --print_loss | Print loss during each step | 1 |
+| --enable_tensorboard | Enable TensorBoard logging at the model Runtime Engine level | 1,2,3 |
+| | Enable TensorBoard logging at the Training Pipeline level | 3 |
+| --tensorboard_path | Path to write TensorBoard log | 1,2,3 |
+| --print_answers | Print actor model prompt and answers during training across all ranks | 3 |
+
+
+### TensorBoard
+TensorBoard logging can be enabled in each of the three training steps, with some slight nuances in Step 3. To start, for each training step, the `enable_tensorboard` argument can be used to enable a TensorBoard monitor at the Runtime Engine level ([see documentation](https://www.deepspeed.ai/docs/config-json/#monitoring-module-tensorboard-wandb-csv)) and is reflected in the corresponding model training configuration:
+```python
+"tensorboard": {
+ "enabled": enable_tensorboard,
+ "output_path": f"{tb_path}/ds_tensorboard_logs/",
+ "job_name": f"{tb_name}_tensorboard"
+}
+```
+
+- **Step 3**:
+ Due to Step 3 initializing both an Actor and a Critic model, _each_ of the models will have their own corresponding TensorBoard monitor at the Runtime Engine level. Beyond that, Step 3 training also contains a Pipeline-level TensorBoard monitor a level above the model runtime engines, which captures the `reward`, `actor_loss`, `actor_loss_sum`, `critic_loss`, and `critic_loss_sum`.
+
+## Testing
+
+As part of the DeepSpeed team's commitment to maintaining the DeepSpeed-Chat training framework, continuous integration [PyTest](https://github.com/microsoft/DeepSpeedExamples/blob/master/applications/DeepSpeed-Chat/training/tests/test_training.py) testing has been added for Step 3 RLHF training in a new [GitHub Actions workflow](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-ds-chat.yml).
+
+| Description | Status |
+| ----------- | ------ |
+| Integrations | [](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-ds-chat.yml) |
+
+ The workflow is run on a **nightly** basis across a **16-case** test matrix (see table below), and uses the **facebook/opt-125m** model for both the actor and critic.
+
+| Parameter | Values |
+| --- | --- |
+| ZeRO Stage | 2, 3 |
+| Hybrid Engine | True, False |
+| ZeRO-Offload | True, False |
+| LoRA | True, False |
+
+Each configuration (16 total) runs through a limited number of Step 3 non-overflow training steps (i.e. steps where neither actor nor critic overflow) and saves the actor/critic models. Assertions are used to check if the training pipeline executed correctly and if the actor and critic models were saved properly.
+
+# 6. Try Out DeepSpeed-Chat
+We are very excited to share this DeepSpeed-Chat feature and stability release.
+
+* To get started, please visit our GitHub page for DeepSpeed-Chat: [GitHub Landing Page](https://github.com/microsoft/DeepSpeedExamples/tree/master/applications/DeepSpeed-Chat)
+
+* We will continue to improve DeepSpeed-Chat with your feedback and support. Our [roadmap](https://github.com/microsoft/DeepSpeedExamples/tree/master/applications/DeepSpeed-Chat/README.md#-deepspeed-chats-roadmap-) shows currently supported features as well as ones that are planned for future.
+
+DeepSpeed-Chat is part of the bigger DeepSpeed ecosystem comprising of a multitude of Deep Learning systems and modeling technologies. To learn more,
+
+* Please visit our [website](https://www.deepspeed.ai/) for detailed blog posts, tutorials, and helpful documentation.
+* You can also follow us on our [English Twitter](https://twitter.com/MSFTDeepSpeed), [Japanese Twitter](https://twitter.com/MSFTDeepSpeedJP), and [Chinese Zhihu](https://www.zhihu.com/people/deepspeed) for latest news on DeepSpeed.
+
+DeepSpeed welcomes your contributions! We encourage you to report issues, contribute PRs, and join discussions on the [DeepSpeed GitHub](https://github.com/microsoft/DeepSpeed/) page. Please see our [contributing guide](https://github.com/microsoft/DeepSpeed/blob/master/CONTRIBUTING.md) for more details. We are open to collaborations with universities, research labs, companies, such as those working together on deep learning research, applying DeepSpeed to empower real-world AI models and applications, and so on. For such requests (and other requests unsuitable for GitHub), please directly email to deepspeed-info@microsoft.com.
+
+* "Star" our [DeepSpeed GitHub](https://github.com/microsoft/DeepSpeed/) and [DeepSpeedExamples GitHub](https://github.com/microsoft/DeepSpeedExamples/) repositories if you like our work!
diff --git a/blogs/deepspeed-chat/japanese/README.md b/blogs/deepspeed-chat/japanese/README.md
new file mode 100644
index 000000000000..e7aa62721417
--- /dev/null
+++ b/blogs/deepspeed-chat/japanese/README.md
@@ -0,0 +1,345 @@
+
+
+# DeepSpeed Chat: ChatGPTライクなモデルを簡単・高速・低コストに、あらゆるスケールで学習
+
+
+
+
+
+
+
+
+
+| GPUs | OPT-6.7B | OPT-13B | OPT-30B | OPT-66B |
+| ------- | :----------------------------------------------------------: | :------------------------------: | :-----: | :-----: |
+| 8x A100-40GB | 5.7 時間 | 10.8 時間 | 1.85 日 | NA |
+| 8x A100-80GB | 4.1 時間 ($132) | 9 時間 ($290) | 18 時間 ($580) | 2.1 日（$1620） |
+
+*表1. ノード1台（8x A100）を用いた場合の訓練時間とAzureでの概算実行コスト*
+
+
+
+
+***高スケーラビリティ***: DeepSpeed-HEは、数千億のパラメータを持つモデルをサポートし、複数ノード・複数GPUのシステムで、優れたスケーラビリティを実現することができます。その結果、13Bのモデルであれば1.25時間で学習でき、175Bの巨大モデルでも、1日以内に学習できます。
+
+
+
+| GPUs | OPT-13B | OPT-30B | OPT-66B | OPT-like-175B |
+| ------------ | :-------------------------------: | :---------------------------------: | :-----: | :-----------: |
+| 64x A100-80G | 1.25 時間 ($320) | 4 時間 ($1024) | 7.5 時間 ($1920) | 20 時間 ($5120) | | |
+
+*表2. 複数ノード（64x A100-80GB）を用いた場合の訓練時間とAzureでの概算実行コスト*
+
+
+> ***注意事項***: 上記の2つの表の数値は、訓練のステージ3のものです。DeepSpeed-RLHFが用いるデータセットと訓練の設定において、合計1.35億トークンを1エポックで訓練した際のスループットの実測値に基づいています。合計6750万のクエリートークン（配列長256の13万件のクエリー）と6750万の生成トークン（配列長256の13万件の回答）があり、ステップごとの最大グローバルバッチサイズは 50万 トークン（クエリーと回答それぞれ1024件）です。DeepSpeedRLHFを用いた場合のコストおよび実行時間の比較にあたっては、これらの詳細をよくご確認ください。さらに詳細な情報は[ベンチマーク設定](https://github.com/microsoft/DeepSpeedExamples/blob/staging-deepspeed-chat-v2/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/BenckmarkSetting.md)を参照ください。
+
+***RLHFを誰もが利用できるように***: DeepSpeed-HEは、1台のGPUのみで130億以上のパラメーターを持つモデルの訓練を実行できます。複数のGPUを備えた高価な計算設備を持たないデータサイエンティストも、小規模なトイモデルではなく、実際のシナリオで使用できる大規模で強力なRLHFモデルを作成できます。
+
+
+
+| | V100 32G | A6000 48G | A100 40G | A100 80G |
+| ---------- | :------: | :-------: | :------: | :------: |
+| モデルサイズ| OPT-2.7B | OPT-6.7B | OPT-6.7B | OPT-13B |
+
+表3. DeepSpeed-HEを用いて1台のGPUで訓練できるモデルサイズ
+
+
+
+以降では、上記で紹介したDeepSpeed-Chatの3つの機能を深く掘り下げて紹介していきます。
+
+# 2. ChatGPTの訓練と推論を手軽に実行
+
+初めに、DeepSpeed-RLHFを用いた訓練の容易さを示す例として、OPT-13BとOPT-66Bのモデルを訓練できることを示します。訓練を実行できる時間が短い場合でも、コンシューマーグレードのGPU1つで、OPT-1.3Bモデルをわずか2時間で訓練することも可能です。DeepSpeed-ChatのAPIを用いて、カスタムされたパイプラインをどのように作るかも紹介していきます。
+
+## DeepSpeed-Chatの例を用いたChatGPTスタイルのモデルの訓練
+
+*a) スクリプトを一つ実行するだけで、RLHF訓練の全3ステージが実行され、ChatGPTライクなモデルを生成されるのを見てみましょう！*
+
+ここではアクターモデルとして事前学習済みOPT-13B、報酬モデルとしてOPT-350MをRLHFで訓練する例を示します。
+以下のスクリプトを実行すると、最終的に130億パラメータのChatGPTライクなモデルが生成されます。
+
+```python
+pip install deepspeed>=0.9.0
+
+git clone https://github.com/microsoft/DeepSpeedExamples.git
+cd DeepSpeedExamples/applications/DeepSpeed-Chat/
+pip install -r requirements.txt
+
+python train.py --actor-model facebook/opt-13b --reward-model facebook/opt-350m --deployment-type single_node
+```
+
+8台のA100を備えたDGXサーバを使う場合、130億パラメータのモデルを半日で訓練できます。以下は各ステップに要する時間の内訳です。
+
+
+
+| モデルサイズ| Step 1 | Step 2 | Step 3 | Total |
+| -------------------------------- | :----: | :----: | :----: | :----: |
+| アクターモデル: OPT-13B, 報酬モデル: OPT-350M | 2.5時間 | 0.25時間 | 10.8時間 | 13.6時間 |
+
+表4. 8台のA100を備えたDGXサーバ1台でのend-to-end処理時間の内訳
+
+
+
+*b) 学習したモデルをDeepSpeed Chatの推論APIで試してみましょう！*
+
+学習させたモデルを組み込んで、試してみましょう。質問応答だけでなく、会話形式のやりとりも楽しめます。
+
+```
+Human: Do you know Microsoft?
+Assistant: Yes, I’m a big fan of Microsoft. What about them?
+Human: Can you explain it to a 6-year old child? I wonder how I should describe it
+Assistant: Sure, I can try. Microsoft is a company that makes computers, and they make a program called “Windows” which
+ is the operating system that runs on the computer. It’s like the “brain” of the computer. It’s where all the
+ programs and files are stored. It’s also where you’ll find the “start” button, which is where you’ll start to
+ use the computer.
+```
+
+## モデルサイズ・設定の変更
+
+訓練の時間、計算機資源、および品質の要件を満たすために、さまざまなモデルのサイズや構成を試す必要があることがあります。DeepSpeed-Chatを使用すれば、簡単にそれが可能です。例えば、研究やビジネスのために、GPUクラスタでより大規模で高品質なモデルを訓練したい場合、希望するモデルサイズとGPU数を指定するだけです。以下は、アクターモデルのパラメータ数を66Bに、GPU数を64に指定する例です。
+
+```python
+python train.py --actor-model facebook/opt-66b --reward-model facebook/opt-350m --deployment-type multi_node
+```
+
+64台のA100 (80GBメモリ) GPUを使用する場合、9時間で660億パラメータのChatGPTモデルを訓練できます。
+
+
+
+| モデルサイズ | Step 1 | Step 2 | Step 3 | Total |
+| -------------------------------- | :-----: | :----: | :----: | :---: |
+|アクターモデル: OPT-66B, 報酬モデル: OPT-350M | 82分 | 5分 | 7.5時間 | 9時間 |
+
+表5. 8台のA100(80GBメモリ)を備えたDGXサーバ8台で660億パラメータのモデルを訓練する場合のend-to-end処理時間の内訳
+
+
+
+1～2時間のコーヒータイムや昼休みに、DeepSpeed-Chatで小規模なトイモデルをトレーニングしてみるのも良いでしょう。例えば、コンシューマグレードのGPUでの訓練を動かしてみるため、1つのデータセットで1.3Bのモデルを訓練する例を用意しました。これなら、昼休みから戻ったときに、できあがったモデルのチェックポイントを試してみることができます。
+
+```python
+python train.py --actor-model facebook/opt-1.3b --reward-model facebook/opt-350m --deployment-type single_gpu
+```
+
+
+
+| モデルサイズ | Step 1 | Step 2 | Step 3 | Total |
+| --------------------------------- | :-------: | :------: | :----: | :---: |
+| Actor: OPT-1.3B, Reward: OPT-350M | 2900 秒| 670 秒 | 1.2時間 | 2.2時間 |
+
+*表6. コモディティグレードのGPU（A6000）1台でのend-to-end処理時間の内訳*
+
+
+
+## DeepSpeed-ChatのAPIを用いたRLHFパイプラインのカスタマイズ
+
+DeepSpeed-Chatでは、設定を柔軟に指定できるAPIを提供しており、ユーザーが独自のRLHF訓練パイプラインを構築することができます。このAPIを通じて、探索的な研究のための幅広いRLHFアルゴリズムを作成するための、汎用的なインターフェースとバックエンドを利用できます。
+
+```python
+
+engine = DeepSpeedRLHFEngine(
+ actor_model_name_or_path=args.actor_model_name_or_path,
+ critic_model_name_or_path=args.critic_model_name_or_path,
+ tokenizer=tokenizer,
+ num_total_iters=num_total_iters,
+ args=args)
+
+trainer = DeepSpeedPPOTrainer(engine=engine, args=args)
+
+for prompt_batch in prompt_train_dataloader:
+ out = trainer.generate_experience(prompt_batch)
+ actor_loss, critic_loss = trainer.train_rlhf(out)
+```
+
+# 3. 完全なRLHF訓練パイプライン
+
+シームレスに訓練を実行するため、DeepSpeed-Chatには、InstructGPTと同様の完全なend-to-endの訓練パイプラインが組み込まれています（図1）。
+
+
+
+
+
+*図1: DeepSpeed-Chatの訓練パイプラインの概要*
+
+
+
+
+
+*図2. RLHFで最も処理時間がかかる部分を高速化するハイブリッドエンジン（DeepSpeed-HE）*
+
+
+
+
+
+*図3. 他フレームワークとのStep 3のスループット比較（1台のA100-40Gを使用。バツ印はメモリ不足で実行できないことを示す）*
+
+
+
+
+
+*図4. ステップ3（全3ステップ処理時間の大半を占める）のEnd-to-endの訓練スループット比較 (8台のA100-40Gを備えた1台のDGXノードを使用。バツ印はメモリ不足で実行できないことを示す）*
+
+
+
+
+
+*図5. DeepSpeed-HEを用いた生成フェーズの高速化（OPT-1.3Bベースのアクターモデル + OPT-350Mベースの報酬モデル、8台のA100-40Gを備えた1台のDGXノードを使用）*
+
+
+
+
+
+*図6. DeepSpeed-HEを用いたRLHFにおける生成、訓練、および実効スループット（GPU数は最善の効率を得られるように設定）*
+
+
+
+
+
+*図7. DGXノード（ノードあたり8台のA100-40/80G）の数を増加させた場合のスケーラビリティ（13Bおよび66Bのアクターモデルと350Mの報酬モデルを使用）*
+
+
+
+# DeepSpeed-FastGen: High-throughput Text Generation for LLMs via MII and DeepSpeed-Inference
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ *Figure 1: Illustration of continuous batching strategies. Each block shows the execution of a forward pass. An arrow indicates that the forward pass has sequences with one or more tokens generated. vLLM performs either token generations or prompt processing in a forward pass; token generation preempts prompt processing. Orca runs prompts at their complete length alongside generation. Dynamic SplitFuse performs dynamic composition of fixed-sized batches composed of both generation and prompt tokens.*
+
+
+
+
+ *Figure 2: Throughput and latency of text generation using Llama 2 70B (Tensor parallelism across 4 A100-80GB GPUs). A normal distribution was applied to prompt and generation lengths with averages of 1200/2600 and 128/60, respectively, and a 30% variance*
+
+
+
+
+
+ *Figure 3: Throughput and latency of text generation using Llama 2 13B (A100-80GB GPU, no tensor parallelism). A normal distribution was applied to prompt and generation lengths with averages of 1200/2600 and 60/128, respectively, and a 30% variance*
+
+
+
+ *Figure 4: Effective throughput of DeepSpeed-FastGen and vLLM (Llama 2 70B/A100-80GB using tensor parallelism across 4 A100-80GB GPUs. A normal distribution was applied to prompt and generation lengths with averages of 2600 and 60, respectively, and a 30% variance)*
+
+
+When vLLM preempts the ongoing generation of previous requests, the generation latency experiences a notable increase. This leads to vLLM's effective throughput appearing lower than its directly measured throughput. At vLLM's peak, the effective throughput was 0.63 queries/sec and around 28% of requests failed to meet the 4 tokens/s SLA. At the same SLA, DeepSpeed-FastGen achieved 1.42 queries/sec (less than 1% of requests failed to meet the SLA), which is 2.3x higher than vLLM.
+
+### D. Token Level Timing Analysis
+
+Figure 5 displays the P50, P90, and P95 latencies of the generation processes. Both vLLM and DeepSpeed-FastGen exhibit similar P50 latencies, but vLLM demonstrates significantly higher latencies for P90 and P95.
+Regarding the P95 latencies, DeepSpeed-FastGen achieved a reduction of 3.7 times.
+
+This discrepancy is due to a noticeable spike in vLLM's generation latency when it preempts the ongoing generation to process new prompts.
+In contrast, DeepSpeed-FastGen typically processes the prompt and generation for previous requests concurrently, leading to much more consistent generation latency.
+
+
+
+
+
+ *Figure 5: Per-Token generation Latency of Llama 2 70B/A100-80GB using tensor parallelism across 4 A100-80GB GPUs, 16 clients. A normal distribution was applied to prompt and generation lengths with averages of 2600 and 128, respectively, and a 30% variance.*
+
+
+
+### E. Scalability using Load Balancing
+
+DeepSpeed-FastGen offers replica-level load balancing that evenly distributes requests across multiple servers, allowing you to effortlessly scale up your application.
+
+Figure 6 illustrates the scalability of DeepSpeed-FastGen when employing the load balancer and up to 16 replicas. Note that we utilized 4 A100 GPUs to compute the Llama 2 70B model. In total, we employed 8 nodes to run the 16 replicas. The results demonstrate nearly perfect scalability with DeepSpeed-FastGen.
+Given that the throughput of a single replica is 1.46 queries/sec, the throughput with 16 replicas reaches 23.7 queries/sec, marking a linear 16x increase compared to a single replica.
+
+
+
+
+ *Figure 6: Scalability using the load balancing feature. A normal distribution was applied to prompt and generation lengths with averages of 2600 and 60, respectively, and a 30% variance*
+
+
+
+ *Figure 7: Throughput-latency curve and effective throughput of Llama 2 70b using 8 H100 GPUs. A normal distribution was applied to prompt and generation lengths with averages of 2600 and 60, respectively, and a 30% variance*
+
+
+
+ *Figure 8: Throughput-latency curve and effective throughput of Llama 2 7b using A6000. A normal distribution was applied to prompt and generation lengths with averages of 2600 and 60, respectively, and a 30% variance*
+
+
+
+
+
+# DeepSpeed-FastGen：通过 MII 和 DeepSpeed-Inference 实现 LLM 高吞吐量文本生成
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ *图 1: 连续批处理策略的示意图。每个块显示一个前向传递的执行。箭头表示前向传递有一个或多个生成的令牌序列。vLLM 在一个前向传递中要么生成令牌，要么处理提示；令牌生成抢占提示处理。Orca 在生成过程中以完整长度处理提示。DeepSpeed-FastGen动态分割融合则执行固定大小批次的动态组合，包括生成和提示令牌。*
+
+
+
+
+ *图 2: 使用 Llama 2 70B 进行文本生成的吞吐量和延迟（使用 4 个 A100-80GB GPU 的张量并行）。提示和生成长度遵循正态分布，平均值分别为 1200/2600 和 128/60，方差为 30%*
+
+
+
+
+
+ *图 3: 使用 Llama 2 13B 进行文本生成的吞吐量和延迟（A100-80GB GPU，无张量并行）。提示和生成长度遵循正态分布，平均值分别为 1200/2600 和 60/128，并且有 30% 的方差*
+
+
+
+ *图 4: DeepSpeed-FastGen 和 vLLM 的有效吞吐量（Llama 2 70B/A100-80GB 使用张量并行在 4 个 A100-80GB GPU 上。提示和生成长度遵循正态分布，平均值分别为 2600 和 60，并且有 30% 的方差)*
+
+
+当 vLLM 抢占正在进行的先前请求的生成时，生成延迟会明显增加。这导致 vLLM 的有效吞吐量看起来低于其直接测量的吞吐量。在 vLLM 的峰值时，有效吞吐量为 0.63 查询/秒，大约 28% 的请求未能满足 4 令牌/秒的 SLA。在相同的 SLA 下，DeepSpeed-FastGen 达到了 1.42 查询/秒（不到 1% 的请求未能满足 SLA），这是 vLLM 的 2.3 倍。
+
+### D. 令牌级时间分析
+
+图 5 显示了生成过程的 P50、P90 和 P95 延迟。vLLM 和 DeepSpeed-FastGen 展示了类似的 P50 延迟，但 vLLM 的 P90 和 P95 延迟显著更高。
+
+这种差异是由于 vLLM 在抢占正在进行的生成以处理新提示时，生成延迟出现显著增加所导致的。
+相比之下，DeepSpeed-FastGen 通常会同时处理之前请求的提示和生成，导致生成延迟更加一致。
+
+
+
+
+ *图 5: 使用张量并行在 4 个 A100-80GB GPU 上的 Llama 2 70B/A100-80GB 的每令牌生成延迟，16 客户端。提示和生成长度遵循正态分布，平均值分别为 2600 和 128，并且有 30% 的方差。
+
+
+
+### E. 使用负载均衡的可扩展性
+
+DeepSpeed-FastGen 提供了副本级负载均衡，可以将请求均匀分布在多个服务器上，让您轻松扩展应用程序。
+
+图 6 展示了 DeepSpeed-FastGen 在使用负载均衡器和最多 16 个副本时的可扩展性。请注意，我们使用了 4 个 A100 GPU 来计算每个 Llama 2 70B 模型。总共，我们使用了 8 个节点来运行 16 个副本。结果展示了 DeepSpeed-FastGen 几乎完美的可扩展性。
+单个副本时DeepSpeed-FastGen的吞吐量为 1.46 查询/秒，而16 个副本的吞吐量达到了 23.7 查询/秒，与单个副本相比标志着线性的 16 倍增长。
+
+
+
+
+ *图 6: 使用负载均衡功能的可扩展性。提示和生成长度遵循正态分布，平均值分别为 2600 和 60，并且有 30% 的方差*
+
+
+
+ *图 7: 使用 8 个 H100 GPU 的 Llama 2 70b 的吞吐量-延迟曲线和有效吞吐量。提示和生成长度遵循正态分布，平均值分别为 2600 和 60，并且有 30% 的方差*
+
+
+
+ *图 8: 使用 A6000 的 Llama 2 7b 的吞吐量-延迟曲线和有效吞吐量。提示和生成长度遵循正态分布，平均值分别为 2600 和 60，并且有 30% 的方差*
+
+
+
+
+
+# DeepSpeed-FastGen: MIIとDeepSpeed-InferenceによるLLMのための高速なテキスト生成
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+*図1: 連続バッチ処理戦略のイラスト。各ブロックはフォワードパスの実行を示しています。矢印は、1つ以上のトークンが生成されたシーケンスを持つフォワードパスを示しています。vLLMはフォワードパスでトークン生成またはプロンプト処理のいずれかを実行し、トークン生成はプロンプト処理をプリエンプトします。Orcaは生成と同時に完全な長さのプロンプトを実行します。Dynamic SplitFuseは、生成トークンとプロンプトトークンの両方で構成された固定サイズのバッチの動的構成を実行します。*
+
+
+
+ *図2: テキスト生成のスループットとレイテンシ（4台のA100-80GB GPUでのテンソル並列を使用したLlama 2 70B）。プロンプトと生成の長さは、平均1200/2600と128/60の正規分布（30％の分散）に基づいて設定。*
+
+
+
+
+
+ *図3: テキスト生成のスループットとレイテンシ（1台のA100-80GB GPUでのテンソル並列なしでのLlama 2 13B）。プロンプトと生成の長さは、平均1200/2600と60/128の正規分布の正規分布（30％の分散）に基づいて設定。*
+
+
+
+ *図4: DeepSpeed-FastGenとvLLMの実効スループット。Llama 2 70B/A100-80GBを使用し、4台のA100-80GB GPU間でテンソル並列を使用。プロンプトと生成の長さは、それぞれ平均2600と60の正規分布（30％の分散）に基づいて設定。*
+
+
+vLLMが、新たなプロンプトを処理するために進行中の前のリクエストの生成を中断すると、生成のレイテンシは顕著に増加します。これにより、vLLMの実効スループットは直接測定されたスループットよりも低く見えます。vLLMのピーク時、実効スループットは0.63クエリ/秒であり、リクエストの約28％が4トークン/秒のSLAを満たすことができませんでした。同じSLAで、DeepSpeed-FastGenは1.42クエリ/秒（SLAを満たさなかったリクエストは1％未満）を達成し、これはvLLMの2.3倍です。
+
+### D. トークン単位のレイテンシ分析
+
+図5は生成プロセスのP50、P90、P95のレイテンシを表示しています。vLLMとDeepSpeed-FastGenを比べると、P50レイテンシに大きな違いはありませんが、vLLMはP90とP95で著しく高いレイテンシを示しています。
+P95レイテンシに関しては、DeepSpeed-FastGenは3.7倍の削減を達成しています。
+
+この差異は、vLLMが進行中の生成を中断して新しいプロンプトを処理する際に、生成レイテンシに顕著なスパイクが生じるためです。
+対照的に、DeepSpeed-FastGenは通常、前のリクエストのプロンプトと生成を同時に処理するため、はるかに一貫した生成のレイテンシを実現します。
+
+
+
+
+ *図5: トークンごとの生成レイテンシ。Llama 2 70B/A100-80GBを使用し、4台のA100-80GB GPU間でテンソル並列を使用。クライアント数16。プロンプトと生成の長さは、それぞれ平均2600と128の正規分布（30％の分散）に基づいて設定。*
+
+
+
+### E. ロードバランシングを使用したスケーラビリティ
+DeepSpeed-FastGenはレプリカ単位のロードバランシングの機能を備えており、複数のサーバーにリクエストを均等に分散させることで、アプリケーションを簡単にスケールアップすることができます。
+
+図6は、ロードバランサーを使用し、最大16のレプリカを適用したときのDeepSpeed-FastGenのスケーラビリティを示しています。Llama 2 70Bモデルの計算には、レプリカ一つあたりで、4台のA100 GPUを使用しました。合計で16のレプリカを実行するために8ノードを使用しました。その結果はDeepSpeed-FastGenのほぼ完璧なスケーラビリティを示しています。1つのレプリカのスループットが1.46クエリ/秒である場合、16のレプリカでのスループットは23.7クエリ/秒に達し、1つのレプリカに比べて16倍の線形増加を示しています。
+
+
+
+
+ *図6: ロードバランシング機能を使用したスケーラビリティ。プロンプトと生成の長さは、それぞれ平均2600と60の正規分布（30％の分散）に基づいて設定。*
+
+
+
+ *図7: 8つのH100 GPUを使用したLlama 2 70bのスループット・レイテンシカーブと実効スループット。プロンプトと生成の長さは、それぞれ平均2600と60の正規分布（30％の分散）に基づいて設定。*
+
+
+
+ *図8: A6000を使用したLlama 2 7bのスループット・レイテンシカーブと実効スループット。プロンプトと生成の長さは、それぞれ平均2600と60の正規分布（30％の分散）に基づいて設定。*
+
+
+
+
+
+| Hardware | Bert-base | Bert-large | Roberta-base | Roberta-large |
+|----------|:------:|:------:|:------:|:------:|
+| A100 |1.65x | 1.68x | 1.53x | 1.61x |
+| V100 | 1.29x | 1.14x | 1.23x | 1.21x |
+
+Table 1. The average speedup (see NOTE below for more detail)
+
+
+
+
+For those transformer operators in float16, we have implemented kernels written in Triton language that replace ordinary CUDA kernels or torch operators.
+The Triton kernels we implemented include softmax, layer-normalization, residual-addition and all the matrix multiplications except MLP layers (see NOTE below for details).
+In our experiments, Triton kernels help to reduce the average latecy (over difference sequence lengths) by 6\~24% (depending on model and hardware) when compared to the latency with CUDA-only kernels.
+
+
+Figures below show the latency reduction in more detail.
+Figure 1 visualizes latency reduction in different sequence lengths in A100 GPU for Bert-base model.
+The baseline (blue) is from Huggingface transformers without any kernel injection, the orange is from Deepspeed with CUDA-only kernels and the gray is from Deepspeed with Triton kernels.
+Figure 2 shows the same plot for Bert-large model in A100 GPU.
+
+
+
+
+
+*Figure 1: Normalized P90 latency for Bert-base model in A100 GPU across different sequence lengths*
+
+
+
+*Figure 2: Normalized P90 latency for Bert-large model in A100 GPU across different sequence lengths*
+
+
+
+# DeepSpeed Ulysses: System Optimizations for Enabling Training of Extreme Long Sequence Transformer Models
+
+
+
+
+
+
+
+
+
+
+*Figure 1: DeepSpeed sequence parallelism (DeepSpeed-Ulysses) design*
+
+
+
+*Figure 2: DeepSpeed sequence parallelism strong scalability evaluation
+at different sequence length and GPU count.*
+
+
+
+*Figure 3: Evaluation of Ulysses vs Megatron LM vs ColAI-SP on GPT-7B parameter
+ model with dense attention (32 GPUs).*
+
+
+
+*Figure 4: Evaluation of Ulysses vs Megatron LM vs ColAI-SP on GPT-30B parameter
+ model with dense attention (64 GPUs).*
+
+
+
+*Figure 5: Evaluation of Ulysses and Megatron LM sequence parallelism on GPT-7B
+parameter model with block sparse attention (32 GPUs).*
+
+
+
+*Figure 6: Evaluation of Ulysses and Megatron LM sequence parallelism on GPT-30B
+parameter model with block sparse attention (64 GPUs).*
+
+
+
+*Figure 7: Convergence evaluation of DeepSpeed sequence parallelism with different
+ZeRO memory optimization stages.*
+
+
+# DeepSpeed Ulysses: 训练极长序列Transformer模型的系统优化
+
+
+
+
+
+
+
+
+
+
+*图1：DeepSpeed序列并行（DeepSpeed-Ulysses）设计*
+
+
+
+*图2：DeepSpeed序列并行强化可扩展性评估，使用不同的序列长度和GPU数量。*
+
+
+
+*图3：DeepSpeed和Megatron LM序列并行在300亿参数模型上的密集注意力评估。*
+
+
+
+*图4：DeepSpeed和Megatron LM序列并行在300亿参数模型上的稀疏注意力评估。*
+
+
+
+*图5：使用不同ZeRO内存优化阶段的DeepSpeed序列并行的收敛性评估。*
+
+
+# DeepSpeed Ulysses: Transformerモデルを非常に長いシーケンスで訓練するための最適化
+
+
+
+
+
+
+
+
+
+
+*図1: DeepSpeed-Ulysses の設計*
+
+
+
+*図2: 異なるシーケンス長・GPU数での強スケーリング（strong scaling）*
+
+
+
+*図3: 300億パラメータ・DenseなアテンションでのMegatron-LMとの比較*
+
+
+
+*図4: 300億パラメータ・SparseなアテンションでのMegatron-LMとの比較*
+
+
+
+*図5: 異なるZeROのステージでの収束*
+
+
+# DeepSpeed-VisualChat：多轮图像+文字，为你展现不一样的AI聊天魅力
+
+
+
+
+
+
+
+
+
+
+ *图 1：DeepSpeed-VisualChat 的模型架构示意图。*
+
+
+
+ *图 2：不同的注意机制：使用一个输入句子“用户：请描述这个图片。”和三个图像令牌（I-token1、I-token2、I-token3）来比较不同的注意机制。在左边，我们展示了标准的因果注意力，将图像令牌视为文本。在中间，我们展示了应用于图像的交叉注意力，同时保持文本令牌的标准因果注意力。在右边，我们展示了我们的创新 MMCA 注意力机制，其中图像令牌只执行自注意，文本令牌独立地注意文本/图像令牌，橙色为图像部分。这种机制由：softmax($`QK^T \odot M_1`$)+ softmax($`QK^T \odot M_2`$) 定义，其中 Q 和 K 分别为查询和密钥，$`M_1`$=[M==1]，和 $`M_2`$=[M==2]，其中 M $`\in`$ R
10x10。*
+
+
+
+ *图 3：示例视觉和语言输入，显示了（1）标准因果注意力 (CA) （2）与交叉注意力组合的标准因果注意力 (CA+ CrA) 和（3）DeepSpeed-VisualChat 中的特殊多模态因果注意力 (MMCA) 之间的输出比较。*
+
+
+
+ *图 4：DeepSpeed-VisualChat 准确地识别了场景是一个美丽的湖泊，并提供了一组合理的建议。相比之下，其他的注意力机制误解了图像认为其包含“带船坡的码头”。*
+
+
+
+ *图 5：经过 LLaVA-Otter 数据混合后的数据样本。灰色对话框来自 LLaVA 数据集，橙色对话框来自 Otter 数据集。*
+
+
+
+ *图 6：DeepSpeed-VisualChat 可以计算图像中的人数，并读取第一张图像中的文本。它还展示了跨图像的理解。*
+
+
+
+ *图 7：DeepSpeed-VisualChat 可以识别名人并将他们与其成就联系起来。*
+
+
+
+ *图 8：DeepSpeed-VisualChat 可以讲故事并识别电影。*
+
+
+# DeepSpeed-VisualChat: 複数ラウンド・複数画像の入力が可能なAIチャット体験を実現
+
+
+
+
+
+
+
+
+
+ *図1: モデルアーキテクチャの概要*
+
+
+
+
+ *図2: 異なるアテンションの機構: 「ユーザー：画像を説明してください」という入力文と3つの画像トークン（I-token1、I-token2、I-token3）と組み合わせて与えた場合の、それぞれのattention機構の構成を示しています。左側では、標準的なcausal attentionによって、画像トークンをテキストとして扱う様子を示しています。中央は、テキストトークンに対する標準的なcausal attentionを維持しながら、画像に適用されるcross attentionを使用する様子を示しています。右側では、画像トークンはself attentionのみを行い、テキストトークンはテキスト／画像トークンへのアテンションを独立に計算するという、新しいマルチモーダルのためのアテンションの提案を、オレンジ色のマスクで強調して示しています。この仕組みは、Q, Kをクエリとキーとしたとき、 softmax($`QK^T \odot M_1`$)+ softmax($`QK^T \odot M_2`$)として定義されます。M $`\in`$ R
10x10としたとき、$`M_1`$=[M==1], and $`M_2`$=[M==2] です。*
+
+
+
+ *図3: (1) 標準的なcausal attention (CA) (2) cross attentionと組み合わせた標準的なcausal attention (CA+CrA) (3)DeepSpeed-VisualChatの特別なマルチモーダルCausal Attention (MMCA) の出力比較を示す視覚入力と言語入力の例。*
+
+
+
+ *図4：DeepSpeed-VisualChatは、示された場面を美しい湖として正確に識別し、妥当な提案のセットを提示する。対照的に、ベースラインは画像を「ボート乗り場のあるドック」と誤認識している。*
+
+
+
+ *図5: LLaVA-Otterデータブレンド後のデータサンプル。灰色のダイアログボックスはLLaVAデータセットから、オレンジ色のダイアログボックスはOtterデータセットからのもの*
+
+
+
+ *図6: DeepSpeed-VisualChatは、画像内の人数を数え、最初の画像のテキストを読み取ることができます。また、複数画像を横断的に理解することも可能です。*
+
+
+
+ *図7: DeepSpeed-VisualChatは有名人を認識し、その人物の業績と関連付けることができます*
+
+
+
+ *図8: DeepSpeed-VisualChatは、ストーリーを作ったり、映画を認識したりできます。*
+
+
+# DeepSpeed-VisualChat: Improve Your Chat Experience with Multi-Round Multi-Image Inputs
+
+
+
+
+
+
+
+
+
+
+ *Figure 1: Model architecture illustration.*
+
+
+
+
+ *Figure 2: Different Attention Mechanisms: Examine the differing attention mechanisms using an input sentence "User: Please describe the image." coupled with three Image tokens (I-token1, I-token2, I-token3). On the left, we demonstrate standard causal attention, treating image tokens as text. In the middle, we present cross attention applied to images, while maintaining standard causal attention for text tokens. On the right, we illustrate our innovative multi-modal attention proposal where image tokens only perform self-attention, and text tokens attend to text/image tokens independently, highlighted with an orange mask. This mechanism is defined by: softmax($`QK^T \odot M_1`$)+ softmax($`QK^T \odot M_2`$) with Q and K as query and key, $`M_1`$=[M==1], and $`M_2`$=[M==2], with M $`\in`$ R
10x10 in this case.*
+
+
+
+ *Figure 3: Example visual and language inputs that demonstrate the output comparison between (1) the standard causal attention (CA) (2) the standard causal attention combined with cross-attention (CA+ CrA) and (3) the special multi-modal causal attention (MMCA) in DeepSpeed-VisualChat.*
+
+
+
+
+ *Figure 4: DeepSpeed-VisualChat accurately identifies the scene as a beautiful lake and offers a set of plausible suggestions. In contrast, the baseline misinterprets the image as containing “dock with a boat ramp”.*
+
+
+
+
+ *Figure 5: A data sample after LLaVA-Otter data blending. Gray dialog boxes are from LLaVA datasets, and orange ones are from Otter dataset.*
+
+
+
+ *Figure 6: DeepSpeed-VisualChat can count the number of people in the image and read the text in the first image. It also demonstrates cross-image understanding.*
+
+
+
+ *Figure 7: DeepSpeed-VisualChat can recognize celebrities and associate them with their achievements.*
+
+
+
+ *Figure 8: DeepSpeed-VisualChat can tell stories and recognize movies.*
+
+
+# Announcing the DeepSpeed4Science Initiative: Enabling large-scale scientific discovery through sophisticated AI system technologies
+
+
+
+[https://www.microsoft.com/en-us/research/blog/announcing-the-deepspeed4science-initiative-enabling-large-scale-scientific-discovery-through-sophisticated-ai-system-technologies/](https://www.microsoft.com/en-us/research/blog/announcing-the-deepspeed4science-initiative-enabling-large-scale-scientific-discovery-through-sophisticated-ai-system-technologies/)
+
+To cite DeepSpeed4Science, please cite our [white paper](https://arxiv.org/abs/2310.04610):
+
+```
+@article{song2023deepspeed4science,
+ title={DeepSpeed4Science Initiative: Enabling Large-Scale Scientific Discovery through Sophisticated AI System Technologies},
+ author={Song, Shuaiwen Leon and Kruft, Bonnie and Zhang, Minjia and Li, Conglong and Chen, Shiyang and Zhang, Chengming and Tanaka, Masahiro and Wu, Xiaoxia and Rasley, Jeff and Awan, Ammar Ahmad and others},
+ journal={arXiv preprint arXiv:2310.04610},
+ year={2023}
+}
+```
diff --git a/blogs/deepspeed4science/chinese/README.md b/blogs/deepspeed4science/chinese/README.md
new file mode 100644
index 000000000000..dabc4ab077f2
--- /dev/null
+++ b/blogs/deepspeed4science/chinese/README.md
@@ -0,0 +1,156 @@
+
+
+# DeepSpeed4Science：利用先进的AI系统优化技术实现科学发现
+
+
+
+*此博客为英文博客[Announcing the DeepSpeed4Science Initiative: Enabling large-scale scientific discovery through sophisticated AI system technologies](https://www.microsoft.com/en-us/research/blog/announcing-the-deepspeed4science-initiative-enabling-large-scale-scientific-discovery-through-sophisticated-ai-system-technologies/)的官方翻译*
+
+
+
+
+*图1：DeepSpeed4Science方法概述：专为加速科学发现和应对其复杂性而量身定制的AI系统技术开发。*
+
+
+
+
+*图2：科学基础模型（Scientific Foundation Model，SFM）及其当前探索：Distributional Graphormer。*
+
+
+
+*图3：ClimaX是第一个设计用于执行各种天气和气候建模任务的基础模型。*
+
+
+
+*图4：一百万步的分子动力学模拟：RBD-蛋白（RBD-protein）与蛋白抑制剂（protein inhibitor）相互作用。*
+
+
+
+*图5：微软降水预报（每4分钟一次对接下来4小时进行预测）。*
+
+
+
+
+*图6：在训练过程中OpenFold对PDB链7B3A_A的预测。*
+
+
+
+*图7：在OpenFold中，对多序列比对（MSA）Attention内核（包含偏差）变体的训练峰值内存需求。 (左) 使用在AlphaFold2中的EvoformerAttention的原始OpenFold实现。对于这些类型的蛋白质结构预测模型，在训练/推理中的内存爆炸问题是常见的。最先进的FlashAttention无法有效支持这样的Attention变体。 (右) DeepSpeed4Science的一种新解决方案DS4Sci_EvoformerAttention在不影响模型品质的条件下显著地减少了OpenFold的训练峰值内存需求（最多13倍）。*
+
+
+
+*图8：GenSLMs：获2022年ACM 戈登贝尔奖的COVID基因组模型（基于GPT-NeoX的25B/33B模型）。它用于学习描述SARS-CoV-2基因组生物学意义的潜在空间。这个GIF展示了一个重要的蛋白质家族苹果酸脱氢酶（malate dehydrogenase）的根据重要特征（如序列长度和GC含量（核酸鸟嘌呤和胞嘧啶的含量与腺嘌呤和胸腺嘧啶的比率。它测量DNA链抵抗热的能力））着色的潜在空间的投影。*
+
+
+
+*图9：由不同框架在不同规模下支持的两个GenSLMs模型的最大序列长度。使用NVIDIA DGX，每个节点有八个40G A100 GPU。*
+
+
+# DeepSpeed4Scienceイニシアティブ: 洗練されたAIシステムのテクノロジーにより大規模な科学的発見を可能に
+
+
+
+*こちらは英語ブログ[Announcing the DeepSpeed4Science Initiative: Enabling large-scale scientific discovery through sophisticated AI system technologies](https://www.microsoft.com/en-us/research/blog/announcing-the-deepspeed4science-initiative-enabling-large-scale-scientific-discovery-through-sophisticated-ai-system-technologies/)の公式の翻訳です*
+
+
+
+
+*図1：DeepSpeed4Scienceのアプローチ: 汎用の言語モデルのサポートを超え、科学的発見とその複雑さの解決に特化したAI技術を開発*
+
+
+
+
+*図2: 科学基盤モデル (Scientific foundation model, SFM) とその探索: Distributional Graphormer*
+
+
+
+*図3: 天気・気候の多様なモデリングタスクのための最初の基盤モデルClimaX*
+
+
+
+*図4: 100万ステップの分子動力学シミュレーション: RBD-proteinとprotein inhibitorの相互作用*
+
+
+
+*図5: Microsoft Startにおける降水予想 (次の４時間について４分ごと)*
+
+
+
+
+*図6: モデル学習の進行に伴うPDB chain 7B3A_AについてのOpenFoldの予測*
+
+
+
+*図7: OpenFoldで可能な最大の訓練サンプル次元を持つ多重配列アライメント（MSA）アテンションカーネル（バイアス付き）のバリエーションを訓練するために必要なピークメモリ。(左）AlphaFold2で使用されているEvoformerAttentionを用いたオリジナルのOpenFold実装。この種のタンパク質構造予測モデルの訓練/推論では、極めて多くのメモリが必要とされることは一般的な課題となっている。特に、最新技術として広く知られるFlashAttentionでも、このような科学研究のためのアテンションのバリエーションを効果的にサポートできない。(右）DS4Sci_EvoformerAttentionと呼ばれるDeepSpeed4Scienceの新しい技術は、精度を落とすことなく、OpenFoldモデルの訓練に必要なピークメモリを1/13に大幅に削減する。*
+
+
+
+*図8: GenSLMs：2022年ACM Gordon Bell Special Prize受賞COVIDゲノム用モデル（GPT-NeoXに基づく25B/33Bモデル）。SARS-CoV-2ゲノムの生物学的に意味のある特性を記述する潜在空間を学習するために使用される。このGIFは、重要なタンパク質ファミリーであるリンゴ酸デヒドロゲナーゼ（malate dehydrogenase）を可視化し、配列の長さやGC含量（アデニンとチミンと比較した核酸グアニンとシトシンの含量の比率。これはDNA鎖が熱に耐える能力を測るものである。）などの重要な特徴で色付けされた潜在空間の投影を表示している。*
+
+
+
+*図9: 異なるスケールで異なるフレームワークがサポートする2つのGenSLMsモデルの最大シーケンス長。1ノードあたり8個の40G A100 GPUを搭載したNVIDIA DGXノードを使用。*
+
+
+# DeepSpeed ZeRO++：降低4倍网络通信，显著提高大模型及类ChatGPT模型训练效率
+
+
+
+
+
+
+图1: DeepSpeed ZeRO++ 简介
+
+
+
+
+图2: ZeRO optimizer 工作流程图
+
+
+
+
+图3:qwZ的分区量化图例
+
+
+
+
+图4: 权重的分层分割存储(hpZ)
+
+
+
+
+图5: qgZ 端到端的工作流程
+
+
+
+| Communication Volume | Forward all-gather on weights | Backward all-gather on weights | Backward reduce-scatter on gradients | Total |
+|:---------------------------:|:------------------------------------:|:-------------------------------------:|:-------------------------------------------:|:------------:|
+| ZeRO | M | M | M | 3M |
+| ZeRO++ | 0.5M | 0 | 0.25M | 0.75M |
+
+
+
+### **通信总量优化**
+
+通过结合以上所有三个组件，我们将跨节点通信量从 3M 减少到 0.75M。 更具体地说，我们使用 qwZ 将模型权重的前向全收集/广播从 M 减少到 0.5M。 我们使用 hpZ 消除了反向传播期间的跨节点 all-gather，将通信从 M 减少到 0。最后，我们使用 qgZ 将反向传播期间的跨节点 reduce-scatter 通信从 M 减少到 0.25M。
+
+## **ZeRO++ 加速大型语言模型训练**
+
+在这里，我们展示了 ZeRO++ 在 384 个 Nvidia V100 GPU 上的真实 LLM 训练场景的测试结果。
+
+
+
+
+
+图6: 在 384 个 V100 GPU 上的各种模型大小下 ZeRO++ 与 ZeRO 的吞吐量，节点间使用 4 个 Infiniband (IB) 进行互连，每个以 100 Gbps 运行。
+
+
+
+
+
+
+图7: 在 384 个 V00 GPU 上 100Gbps 跨节点带宽时各种 LLM 的吞吐量
+
+
+
+
+
+
+图8: ZeRO++ 以显着降低的带宽实现高带宽集群性能
+
+
+
+
+
+
+图9: ZeRO++ 加速了 RLHF 训练的生成和训练阶段
+
+
+
+# DeepSpeed ZeRO++: LLMやチャットモデルの訓練を劇的に高速化 – 通信オーバヘッドを1/4に大幅削減 -
+
+
+
+
+
+
+図1: DeepSpeed ZeRO++ の概要
+
+
+
+
+図2: ZeROによる最適化
+
+
+
+
+図3: qwZにおけるブロックベース量子化
+
+
+
+
+図4: hpZにおける階層的なパラメータの分割
+
+
+
+
+図5: qgZの処理の流れ
+
+
+
+| Communication Volume | Forward all-gather on weights | Backward all-gather on weights | Backward reduce-scatter on gradients | Total |
+|:---------------------------:|:------------------------------------:|:-------------------------------------:|:-------------------------------------------:|:------------:|
+| ZeRO | M | M | M | 3M |
+| ZeRO++ | 0.5M | 0 | 0.25M | 0.75M |
+
+
+
+### **通信量の削減**
+
+上述の3つの最適化技術をすべて組み込むことで、ノード間の通信量を3Mから0.75Mに減らすことができます。具体的には、qwZを用いて、モデルパラメータに関する順伝播のallgather/broadcast通信をMから0.5Mに削減します。また、qgZを使用して、逆伝播のノード間のreduce-scatter通信をMから0.25Mに削減します。
+
+## **ZeRO++によるLLM訓練の高速化**
+
+ここでは、384台のNVIDIA V100 GPUを使用した、実際のLLM訓練シナリオでのZeRO++の評価結果を示します。
+
+
+
+
+
+図6: 様々なモデルサイズでのZeRO++とZeROの速度の比較（384台のV100 GPU、400Gbps (100Gbps×4) のノード間接続）
+
+
+
+
+
+
+図7: 異なるサイズのLLMのスループット比較（384台のGPU・100Gbpsのノード間接続）
+
+
+
+
+
+図8: ZeRO++により、低い帯域幅のクラスタでも、ZeROを高い帯域幅のクラスタで使用した場合と同等の性能を実現
+
+
+
+
+
+
+図9: ZeRO++によりRLHF訓練の生成フェーズと訓練フェーズの両方を高速化
+
+
-
-
-## DeepSpeed-Training
-
-DeepSpeed offers a confluence of system innovations, that has made large scale DL training effective, and efficient, greatly improved ease of use, and redefined the DL training landscape in terms of scale that is possible. These innovations such as ZeRO, 3D-Parallelism, DeepSpeed-MoE, ZeRO-Infinity, etc. fall under the training pillar. Learn more: [DeepSpeed-Training](https://www.deepspeed.ai/training/)
-
-## DeepSpeed-Inference
-
-DeepSpeed brings together innovations in parallelism technology such as tensor, pipeline, expert and ZeRO-parallelism, and combines them with high performance custom inference kernels, communication optimizations and heterogeneous memory technologies to enable inference at an unprecedented scale, while achieving unparalleled latency, throughput and cost reduction. This systematic composition of system technologies for inference falls under the inference pillar. Learn more: [DeepSpeed-Inference](https://www.deepspeed.ai/inference)
-
-
-## DeepSpeed-Compression
-
-To further increase the inference efficiency, DeepSpeed offers easy-to-use and flexible-to-compose compression techniques for researchers and practitioners to compress their models while delivering faster speed, smaller model size, and significantly reduced compression cost. Moreover, SoTA innovations on compression like ZeroQuant and XTC are included under the compression pillar. Learn more: [DeepSpeed-Compression](https://www.deepspeed.ai/compression)
-
----
-
-# DeepSpeed Software Suite
-
-## DeepSpeed Library
-
- The [DeepSpeed](https://github.com/microsoft/deepspeed) library (this repository) implements and packages the innovations and technologies in DeepSpeed Training, Inference and Compression Pillars into a single easy-to-use, open-sourced repository. It allows for easy composition of multitude of features within a single training, inference or compression pipeline. The DeepSpeed Library is heavily adopted by the DL community, and has been used to enable some of the most powerful models (see [DeepSpeed Adoption](#deepspeed-adoption)).
-
-## Model Implementations for Inference (MII)
-
- [Model Implementations for Inference (MII)](https://github.com/microsoft/deepspeed-mii) is an open-sourced repository for making low-latency and high-throughput inference accessible to all data scientists by alleviating the need to apply complex system optimization techniques themselves. Out-of-box, MII offers support for thousands of widely used DL models, optimized using DeepSpeed-Inference, that can be deployed with a few lines of code, while achieving significant latency reduction compared to their vanilla open-sourced versions.
-
-## DeepSpeed on Azure
-
- DeepSpeed users are diverse and have access to different environments. We recommend to try DeepSpeed on Azure as it is the simplest and easiest method. The recommended method to try DeepSpeed on Azure is through AzureML [recipes](https://github.com/Azure/azureml-examples/tree/main/v1/python-sdk/workflows/train/deepspeed). The job submission and data preparation scripts have been made available [here](https://github.com/microsoft/Megatron-DeepSpeed/tree/main/examples/azureml). For more details on how to use DeepSpeed on Azure, please follow the [Azure tutorial](https://www.deepspeed.ai/tutorials/azure/).
-
----
-
-# DeepSpeed Adoption
-
-DeepSpeed is an important part of Microsoft’s new
-[AI at Scale](https://www.microsoft.com/en-us/research/project/ai-at-scale/)
-initiative to enable next-generation AI capabilities at scale, where you can find more
-information [here](https://innovation.microsoft.com/en-us/exploring-ai-at-scale).
-
-DeepSpeed has been used to train many different large-scale models, below is a list of several examples that we are aware of (if you'd like to include your model please submit a PR):
-
- * [Megatron-Turing NLG (530B)](https://www.microsoft.com/en-us/research/blog/using-deepspeed-and-megatron-to-train-megatron-turing-nlg-530b-the-worlds-largest-and-most-powerful-generative-language-model/)
- * [Jurassic-1 (178B)](https://uploads-ssl.webflow.com/60fd4503684b466578c0d307/61138924626a6981ee09caf6_jurassic_tech_paper.pdf)
- * [BLOOM (176B)](https://huggingface.co/blog/bloom-megatron-deepspeed)
- * [GLM (130B)](https://github.com/THUDM/GLM-130B)
- * [YaLM (100B)](https://github.com/yandex/YaLM-100B)
- * [GPT-NeoX (20B)](https://github.com/EleutherAI/gpt-neox)
- * [AlexaTM (20B)](https://www.amazon.science/blog/20b-parameter-alexa-model-sets-new-marks-in-few-shot-learning)
- * [Turing NLG (17B)](https://www.microsoft.com/en-us/research/blog/turing-nlg-a-17-billion-parameter-language-model-by-microsoft/)
- * [METRO-LM (5.4B)](https://arxiv.org/pdf/2204.06644.pdf)
-
-DeepSpeed has been integrated with several different popular open-source DL frameworks such as:
-
-| | Documentation |
-| ---------------------------------------------------------------------------------------------- | -------------------------------------------- |
-
| [Transformers with DeepSpeed](https://huggingface.co/docs/transformers/main/main_classes/deepspeed) |
-| 
| [Accelerate with DeepSpeed](https://huggingface.co/docs/accelerate/usage_guides/deepspeed) |
-| 
| [Lightning with DeepSpeed](https://pytorch-lightning.readthedocs.io/en/stable/api/pytorch_lightning.strategies.DeepSpeedStrategy.html) |
-| 
| [MosaicML with DeepSpeed](https://docs.mosaicml.com/en/latest/trainer/using_the_trainer.html?highlight=deepspeed#deepspeed-integration) |
-| 
| [Determined with DeepSpeed](https://docs.determined.ai/latest/training/apis-howto/deepspeed/overview.html) |
-
----
-
-# Build Pipeline Status
-
-| Description | Status |
-| ----------- | ------ |
-| NVIDIA | [](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-torch12-p40.yml) [](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-torch18-v100.yml) [](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-torch-latest-v100.yml) [](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-inference.yml) [](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-nightly.yml) |
-| AMD | [](https://github.com/microsoft/DeepSpeed/actions/workflows/amd.yml) |
-| PyTorch Nightly | [](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-torch-nightly-v100.yml) |
-| Integrations | [](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-transformers-v100.yml) [](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-lightning-v100.yml) [](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-accelerate-v100.yml) |
-| Misc | [](https://github.com/microsoft/DeepSpeed/actions/workflows/formatting.yml) [](https://github.com/microsoft/DeepSpeed/actions/workflows/pages/pages-build-deployment) [](https://deepspeed.readthedocs.io/en/latest/?badge=latest)|
-
-# Installation
-
-The quickest way to get started with DeepSpeed is via pip, this will install
-the latest release of DeepSpeed which is not tied to specific PyTorch or CUDA
-versions. DeepSpeed includes several C++/CUDA extensions that we commonly refer
-to as our 'ops'. By default, all of these extensions/ops will be built
-just-in-time (JIT) using [torch's JIT C++ extension loader that relies on
-ninja](https://pytorch.org/docs/stable/cpp_extension.html) to build and
-dynamically link them at runtime.
-
-## Requirements
-* [PyTorch](https://pytorch.org/) must be installed _before_ installing DeepSpeed.
-* For full feature support we recommend a version of PyTorch that is >= 1.8 and ideally the latest PyTorch stable release.
-* A CUDA or ROCm compiler such as [nvcc](https://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/#introduction) or [hipcc](https://github.com/ROCm-Developer-Tools/HIPCC) used to compile C++/CUDA/HIP extensions.
-* Specific GPUs we develop and test against are listed below, this doesn't mean your GPU will not work if it doesn't fall into this category it's just DeepSpeed is most well tested on the following:
- * NVIDIA: Pascal, Volta, Ampere, and Hopper architectures
- * AMD: MI100 and MI200
-
-## PyPI
-We regularly push releases to [PyPI](https://pypi.org/project/deepspeed/) and encourage users to install from there in most cases.
-
-```bash
-pip install deepspeed
-```
-
-After installation, you can validate your install and see which extensions/ops
-your machine is compatible with via the DeepSpeed environment report.
-
-```bash
-ds_report
-```
-
-If you would like to pre-install any of the DeepSpeed extensions/ops (instead
-of JIT compiling) or install pre-compiled ops via PyPI please see our [advanced
-installation instructions](https://www.deepspeed.ai/tutorials/advanced-install/).
-
-## Windows
-Windows support is partially supported with DeepSpeed. On Windows you can build wheel with following steps, currently only inference mode is supported.
-1. Install pytorch, such as pytorch 1.8 + cuda 11.1
-2. Install visual cpp build tools, such as VS2019 C++ x64/x86 build tools
-3. Launch cmd console with Administrator privilege for creating required symlink folders
-4. Run `python setup.py bdist_wheel` to build wheel in `dist` folder
-
-# Features
-
-Please checkout [DeepSpeed-Training](https://www.deepspeed.ai/training), [DeepSpeed-Inference](https://www.deepspeed.ai/inference) and [DeepSpeed-Compression](https://www.deepspeed.ai/compression) pages for full set of features offered along each of these three pillars.
-
-# Further Reading
-
-All DeepSpeed documentation, tutorials, and blogs can be found on our website: [deepspeed.ai](https://www.deepspeed.ai/)
-
-
-| | Description |
-| ---------------------------------------------------------------------------------------------- | -------------------------------------------- |
-| [Getting Started](https://www.deepspeed.ai/getting-started/) | First steps with DeepSpeed |
-| [DeepSpeed JSON Configuration](https://www.deepspeed.ai/docs/config-json/) | Configuring DeepSpeed |
-| [API Documentation](https://deepspeed.readthedocs.io/en/latest/) | Generated DeepSpeed API documentation |
-| [Tutorials](https://www.deepspeed.ai/tutorials/) | Tutorials |
-| [Blogs](https://www.deepspeed.ai/posts/) | Blogs |
-
-
-# Contributing
-DeepSpeed welcomes your contributions! Please see our
-[contributing](CONTRIBUTING.md) guide for more details on formatting, testing,
-etc.
-
-## Contributor License Agreement
-This project welcomes contributions and suggestions. Most contributions require you to
-agree to a Contributor License Agreement (CLA) declaring that you have the right to, and
-actually do, grant us the rights to use your contribution. For details, visit
-https://cla.opensource.microsoft.com.
-
-When you submit a pull request, a CLA bot will automatically determine whether you need
-to provide a CLA and decorate the PR appropriately (e.g., status check, comment). Simply
-follow the instructions provided by the bot. You will only need to do this once across
-all repos using our CLA.
-
-## Code of Conduct
-This project has adopted the [Microsoft Open Source Code of
-Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the
-[Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact
-[opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
-
-# Publications
-1. Samyam Rajbhandari, Jeff Rasley, Olatunji Ruwase, Yuxiong He. (2019) ZeRO: memory optimizations toward training trillion parameter models. [arXiv:1910.02054](https://arxiv.org/abs/1910.02054) and [In Proceedings of the International Conference for High Performance Computing, Networking, Storage and Analysis (SC '20)](https://dl.acm.org/doi/10.5555/3433701.3433727).
-2. Jeff Rasley, Samyam Rajbhandari, Olatunji Ruwase, and Yuxiong He. (2020) DeepSpeed: System Optimizations Enable Training Deep Learning Models with Over 100 Billion Parameters. [In Proceedings of the 26th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining (KDD '20, Tutorial)](https://dl.acm.org/doi/10.1145/3394486.3406703).
-3. Minjia Zhang, Yuxiong He. (2020) Accelerating Training of Transformer-Based Language Models with Progressive Layer Dropping. [arXiv:2010.13369](https://arxiv.org/abs/2010.13369) and [NeurIPS 2020](https://proceedings.neurips.cc/paper/2020/hash/a1140a3d0df1c81e24ae954d935e8926-Abstract.html).
-4. Jie Ren, Samyam Rajbhandari, Reza Yazdani Aminabadi, Olatunji Ruwase, Shuangyan Yang, Minjia Zhang, Dong Li, Yuxiong He. (2021) ZeRO-Offload: Democratizing Billion-Scale Model Training. [arXiv:2101.06840](https://arxiv.org/abs/2101.06840) and [USENIX ATC 2021](https://www.usenix.org/conference/atc21/presentation/ren-jie).
-5. Hanlin Tang, Shaoduo Gan, Ammar Ahmad Awan, Samyam Rajbhandari, Conglong Li, Xiangru Lian, Ji Liu, Ce Zhang, Yuxiong He. (2021) 1-bit Adam: Communication Efficient Large-Scale Training with Adam's Convergence Speed. [arXiv:2102.02888](https://arxiv.org/abs/2102.02888) and [ICML 2021](http://proceedings.mlr.press/v139/tang21a.html).
-6. Samyam Rajbhandari, Olatunji Ruwase, Jeff Rasley, Shaden Smith, Yuxiong He. (2021) ZeRO-Infinity: Breaking the GPU Memory Wall for Extreme Scale Deep Learning. [arXiv:2104.07857](https://arxiv.org/abs/2104.07857) and [SC 2021](https://dl.acm.org/doi/abs/10.1145/3458817.3476205).
-7. Conglong Li, Ammar Ahmad Awan, Hanlin Tang, Samyam Rajbhandari, Yuxiong He. (2021) 1-bit LAMB: Communication Efficient Large-Scale Large-Batch Training with LAMB's Convergence Speed. [arXiv:2104.06069](https://arxiv.org/abs/2104.06069) and [HiPC 2022](https://hipc.org/advance-program/).
-8. Conglong Li, Minjia Zhang, Yuxiong He. (2021) The Stability-Efficiency Dilemma: Investigating Sequence Length Warmup for Training GPT Models. [arXiv:2108.06084](https://arxiv.org/abs/2108.06084) and [NeurIPS 2022](https://openreview.net/forum?id=JpZ5du_Kdh).
-9. Yucheng Lu, Conglong Li, Minjia Zhang, Christopher De Sa, Yuxiong He. (2022) Maximizing Communication Efficiency for Large-scale Training via 0/1 Adam. [arXiv:2202.06009](https://arxiv.org/abs/2202.06009).
-10. Samyam Rajbhandari, Conglong Li, Zhewei Yao, Minjia Zhang, Reza Yazdani Aminabadi, Ammar Ahmad Awan, Jeff Rasley, Yuxiong He. (2022) DeepSpeed-MoE: Advancing Mixture-of-Experts Inference and Training to Power Next-Generation AI Scale [arXiv:2201.05596](https://arxiv.org/abs/2201.05596) and [ICML 2022](https://proceedings.mlr.press/v162/rajbhandari22a.html).
-11. Shaden Smith, Mostofa Patwary, Brandon Norick, Patrick LeGresley, Samyam Rajbhandari, Jared Casper, Zhun Liu, Shrimai Prabhumoye, George Zerveas, Vijay Korthikanti, Elton Zhang, Rewon Child, Reza Yazdani Aminabadi, Julie Bernauer, Xia Song, Mohammad Shoeybi, Yuxiong He, Michael Houston, Saurabh Tiwary, Bryan Catanzaro. (2022) Using DeepSpeed and Megatron to Train Megatron-Turing NLG 530B, A Large-Scale Generative Language Model [arXiv:2201.11990](https://arxiv.org/abs/2201.11990).
-12. Xiaoxia Wu, Zhewei Yao, Minjia Zhang, Conglong Li, Yuxiong He. (2022) Extreme Compression for Pre-trained Transformers Made Simple and Efficient. [arXiv:2206.01859](https://arxiv.org/abs/2206.01859) and [NeurIPS 2022](https://openreview.net/forum?id=xNeAhc2CNAl).
-13. Zhewei Yao, Reza Yazdani Aminabadi, Minjia Zhang, Xiaoxia Wu, Conglong Li, Yuxiong He. (2022) ZeroQuant: Efficient and Affordable Post-Training Quantization for Large-Scale Transformers. [arXiv:2206.01861](https://arxiv.org/abs/2206.01861) and [NeurIPS 2022](https://openreview.net/forum?id=f-fVCElZ-G1).
-14. Reza Yazdani Aminabadi, Samyam Rajbhandari, Minjia Zhang, Ammar Ahmad Awan, Cheng Li, Du Li, Elton Zheng, Jeff Rasley, Shaden Smith, Olatunji Ruwase, Yuxiong He. (2022) DeepSpeed Inference: Enabling Efficient Inference of Transformer Models at Unprecedented Scale. [arXiv:2207.00032](https://arxiv.org/abs/2207.00032) and [SC 2022](https://dl.acm.org/doi/abs/10.5555/3571885.3571946).
-15. Zhewei Yao, Xiaoxia Wu, Conglong Li, Connor Holmes, Minjia Zhang, Cheng Li, Yuxiong He. (2022) Random-LTD: Random and Layerwise Token Dropping Brings Efficient Training for Large-scale Transformers. [arXiv:2211.11586](https://arxiv.org/abs/2211.11586).
-16. Conglong Li, Zhewei Yao, Xiaoxia Wu, Minjia Zhang, Yuxiong He. (2022) DeepSpeed Data Efficiency: Improving Deep Learning Model Quality and Training Efficiency via Efficient Data Sampling and Routing. [arXiv:2212.03597](https://arxiv.org/abs/2212.03597).
-
-
-# Videos
-1. DeepSpeed KDD 2020 Tutorial
- 1. [Overview](https://www.youtube.com/watch?v=CaseqC45DNc&list=PLa85ZdUjfWS21mgibJ2vCvLziprjpKoW0&index=29)
- 2. [ZeRO + large model training](https://www.youtube.com/watch?v=y4_bCiAsIAk&list=PLa85ZdUjfWS21mgibJ2vCvLziprjpKoW0&index=28)
- 3. [17B T-NLG demo](https://www.youtube.com/watch?v=9V-ZbP92drg&list=PLa85ZdUjfWS21mgibJ2vCvLziprjpKoW0&index=27)
- 4. [Fastest BERT training + RScan tuning](https://www.youtube.com/watch?v=o1K-ZG9F6u0&list=PLa85ZdUjfWS21mgibJ2vCvLziprjpKoW0&index=26)
- 5. DeepSpeed hands on deep dive: [part 1](https://www.youtube.com/watch?v=_NOk-mBwDYg&list=PLa85ZdUjfWS21mgibJ2vCvLziprjpKoW0&index=92), [part 2](https://www.youtube.com/watch?v=sG6_c4VXLww&list=PLa85ZdUjfWS21mgibJ2vCvLziprjpKoW0&index=94), [part 3](https://www.youtube.com/watch?v=k9yPkBTayos&list=PLa85ZdUjfWS21mgibJ2vCvLziprjpKoW0&index=93)
- 6. [FAQ](https://www.youtube.com/watch?v=nsHu6vEgPew&list=PLa85ZdUjfWS21mgibJ2vCvLziprjpKoW0&index=24)
-2. Microsoft Research Webinar
- * Registration is free and all videos are available on-demand.
- * [ZeRO & Fastest BERT: Increasing the scale and speed of deep learning training in DeepSpeed](https://note.microsoft.com/MSR-Webinar-DeepSpeed-Registration-On-Demand.html).
-3. [DeepSpeed on AzureML](https://youtu.be/yBVXR8G8Bg8)
-4. Community Tutorials
- * [DeepSpeed: All the tricks to scale to gigantic models (Mark Saroufim)](https://www.youtube.com/watch?v=pDGI668pNg0)
- * [Turing-NLG, DeepSpeed and the ZeRO optimizer (Yannic Kilcher)](https://www.youtube.com/watch?v=tC01FRB0M7w)
- * [Ultimate Guide To Scaling ML Models (The AI Epiphany)](https://www.youtube.com/watch?v=hc0u4avAkuM)
+- Version 1.0 of [GPT-NeoX](https://github.com/EleutherAI/gpt-neox/releases/tag/v1.0) and [DeeperSpeed](https://github.com/EleutherAI/DeeperSpeed/releases/tag/v1.0) maintain snapshots of the old stable versions that [GPT-NeoX-20B](https://arxiv.org/abs/2204.06745) and the [Pythia Suite](https://github.com/EleutherAI/pythia) were trained on.
+- Version 2.0 of [GPT-NeoX](https://github.com/EleutherAI/gpt-neox/releases/tag/v2.0) and [DeeperSpeed](https://github.com/EleutherAI/DeeperSpeed/releases/tag/v2.0) are the latest versions built on the latest DeepSpeed, and will be maintained going forward.
diff --git a/SECURITY.md b/SECURITY.md
index e0dfff56a956..9e9391ee0bac 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -12,7 +12,7 @@ If you believe you have found a security vulnerability in any Microsoft-owned re
Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://msrc.microsoft.com/create-report).
-If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the the [Microsoft Security Response Center PGP Key page](https://www.microsoft.com/en-us/msrc/pgp-key-msrc).
+If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://www.microsoft.com/en-us/msrc/pgp-key-msrc).
You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc).
diff --git a/accelerator/__init__.py b/accelerator/__init__.py
index d29c3b15467b..efed1ef84aca 100644
--- a/accelerator/__init__.py
+++ b/accelerator/__init__.py
@@ -1,4 +1,7 @@
-'''Copyright The Microsoft DeepSpeed Team'''
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
from .abstract_accelerator import DeepSpeedAccelerator
-from .real_accelerator import get_accelerator, set_accelerator
+from .real_accelerator import get_accelerator, set_accelerator, is_current_accelerator_supported
diff --git a/accelerator/abstract_accelerator.py b/accelerator/abstract_accelerator.py
index 03dfbe9df43d..0c7ee3212fad 100644
--- a/accelerator/abstract_accelerator.py
+++ b/accelerator/abstract_accelerator.py
@@ -1,14 +1,22 @@
-'''Copyright The Microsoft DeepSpeed Team'''
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
import abc
from abc import ABC
class DeepSpeedAccelerator(ABC):
+
def __init__(self):
self._name = None
self._communication_backend_name = None
+ @abc.abstractmethod
+ def is_synchronized_device(self):
+ ...
+
# Device APIs
@abc.abstractmethod
def device_name(self, device_index):
@@ -139,6 +147,10 @@ def max_memory_reserved(self, device_index=None):
def total_memory(self, device_index=None):
...
+ @abc.abstractmethod
+ def available_memory(self, device_index=None):
+ ...
+
# Data types
@abc.abstractmethod
def is_bf16_supported(self):
@@ -148,6 +160,10 @@ def is_bf16_supported(self):
def is_fp16_supported(self):
...
+ @abc.abstractmethod
+ def supported_dtypes(self):
+ ...
+
# Misc
@abc.abstractmethod
def amp(self):
@@ -173,6 +189,10 @@ def lazy_call(self, callback):
def communication_backend_name(self):
...
+ @abc.abstractmethod
+ def is_triton_supported(self):
+ ...
+
# Tensor operations
@property
@abc.abstractmethod
@@ -210,7 +230,11 @@ def LongTensor(self):
...
@abc.abstractmethod
- def pin_memory(self, tensor):
+ def pin_memory(self, tensor, align_bytes=1):
+ ...
+
+ @abc.abstractmethod
+ def is_pinned(self, tensor):
...
@abc.abstractmethod
diff --git a/accelerator/cpu_accelerator.py b/accelerator/cpu_accelerator.py
new file mode 100644
index 000000000000..fdbbd33c07a2
--- /dev/null
+++ b/accelerator/cpu_accelerator.py
@@ -0,0 +1,282 @@
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+import torch
+from deepspeed.accelerator.abstract_accelerator import DeepSpeedAccelerator
+import oneccl_bindings_for_pytorch # noqa: F401 # type: ignore
+import psutil
+import os
+
+
+# accelerator for Intel CPU
+class CPU_Accelerator(DeepSpeedAccelerator):
+
+ def __init__(self):
+ self._name = 'cpu'
+ self._communication_backend_name = 'ccl'
+ self.max_mem = psutil.Process().memory_info().rss
+
+ def is_synchronized_device(self):
+ return True
+
+ # Device APIs
+ def device_name(self, device_index=None):
+ return 'cpu'
+
+ def device(self, device_index=None):
+ return None
+
+ def set_device(self, device_index):
+ return
+
+ def current_device(self):
+ return os.environ.get('LOCAL_RANK', 0)
+
+ def current_device_name(self):
+ return 'cpu'
+
+ def device_count(self):
+ device_count = int(os.environ.get('LOCAL_SIZE', 0))
+ if device_count > 0:
+ return device_count
+ else:
+ from deepspeed.utils.numa import get_numa_cores
+ # Count NUMA node for number of cpu accelerators. On machine with HBM
+ # In flat mode, HBM is in separate NUMA node with no cores on this node.
+ # Ignore these NUMA nodes with no cores.
+ numa_core_lists = get_numa_cores()
+ numa_count = 0
+ prev_core_list = []
+ for core_list in numa_core_lists:
+ if len(core_list) > 0 and core_list != prev_core_list:
+ numa_count += 1
+ prev_core_list = core_list
+ return numa_count
+
+ def synchronize(self, device_index=None):
+ return
+
+ # RNG APIs
+ def random(self):
+ return torch.random
+
+ def set_rng_state(self, new_state, device_index=None):
+ if device_index == None:
+ return torch.set_rng_state(new_state)
+ return torch.set_rng_state(new_state, device_index)
+
+ def get_rng_state(self, device_index=None):
+ return torch.get_rng_state()
+
+ def manual_seed(self, seed):
+ return torch.manual_seed(seed)
+
+ def manual_seed_all(self, seed):
+ return torch.manual_seed(seed)
+
+ def initial_seed(self, seed):
+ return torch.initial_seed(seed)
+
+ def default_generator(self, device_index):
+ return torch.default_generator
+
+ # Streams/Events
+ @property
+ def Stream(self):
+ return None
+
+ def stream(self, stream):
+ from deepspeed.runtime.utils import noop_context
+ return noop_context()
+
+ def current_stream(self, device_index=None):
+ return None
+
+ def default_stream(self, device_index=None):
+ return None
+
+ @property
+ def Event(self):
+ return None
+
+ # Memory management
+ def empty_cache(self):
+ return
+
+ def get_rss(self):
+ mem = psutil.Process().memory_info().rss
+ if mem > self.max_mem:
+ self.max_mem = mem
+ return mem
+
+ def reset_rss(self):
+ mem = psutil.Process().memory_info().rss
+ self.max_mem = mem
+ return mem
+
+ def memory_allocated(self, device_index=None):
+ return self.get_rss()
+
+ def max_memory_allocated(self, device_index=None):
+ self.get_rss()
+ return self.max_mem
+
+ def reset_max_memory_allocated(self, device_index=None):
+ self.reset_rss()
+ return
+
+ def memory_cached(self, device_index=None):
+ return self.get_rss()
+
+ def max_memory_cached(self, device_index=None):
+ self.get_rss()
+ return self.max_mem
+
+ def reset_max_memory_cached(self, device_index=None):
+ self.reset_rss()
+ return
+
+ def memory_stats(self, device_index=None):
+ mem = self.get_rss()
+ mem_stat = {}
+ mem_stat['allocated_bytes.all.current'] = mem
+ mem_stat['allocated_bytes.all.peak'] = self.max_mem
+ return mem_stat
+
+ def reset_peak_memory_stats(self, device_index=None):
+ self.reset_rss()
+ return
+
+ def memory_reserved(self, device_index=None):
+ return self.get_rss()
+
+ def max_memory_reserved(self, device_index=None):
+ self.get_rss()
+ return self.max_mem
+
+ def total_memory(self, device_index=None):
+ return psutil.virtual_memory().total
+
+ def available_memory(self, device_index=None):
+ return psutil.virtual_memory().available
+
+ # Misc
+ def amp(self):
+ return torch.cpu.amp
+
+ def is_available(self):
+ return True
+
+ def range_push(self, msg):
+ # TODO itt is currently not supported yet
+ # return torch.profiler.itt.range_push(msg)
+ return
+
+ def range_pop(self):
+ # TODO itt is currently not supported yet
+ # return torch.profiler.itt.range_pop()
+ return
+
+ def lazy_call(self, callback):
+ return callback()
+
+ def communication_backend_name(self):
+ return self._communication_backend_name
+
+ def is_triton_supported(self):
+ return False
+
+ # Data types
+ def is_bf16_supported(self):
+ return True
+
+ def is_fp16_supported(self):
+ return False
+
+ def supported_dtypes(self):
+ return [torch.float, torch.bfloat16]
+
+ # Tensor operations
+
+ @property
+ def BFloat16Tensor(self):
+ return torch.BFloat16Tensor
+
+ @property
+ def ByteTensor(self):
+ return torch.ByteTensor
+
+ @property
+ def DoubleTensor(self):
+ return torch.DoubleTensor
+
+ @property
+ def FloatTensor(self):
+ return torch.FloatTensor
+
+ @property
+ def HalfTensor(self):
+ return torch.HalfTensor
+
+ @property
+ def IntTensor(self):
+ return torch.IntTensor
+
+ @property
+ def LongTensor(self):
+ return torch.LongTensor
+
+ def pin_memory(self, tensor, align_bytes=1):
+ return tensor
+
+ def is_pinned(self, tensor):
+ return tensor.is_pinned()
+
+ def op_builder_dir(self):
+ try:
+ # is op_builder from deepspeed or a 3p version? this should only succeed if it's deepspeed
+ # if successful this also means we're doing a local install and not JIT compile path
+ from op_builder import __deepspeed__ # noqa: F401 # type: ignore
+ return "op_builder.cpu"
+ except ImportError:
+ return "deepspeed.ops.op_builder.cpu"
+
+ def on_accelerator(self, tensor):
+ device_str = str(tensor.device)
+ if device_str.startswith('cpu'):
+ return True
+ else:
+ return False
+
+ # create an instance of op builder and return, name specified by class_name
+ def create_op_builder(self, op_name):
+ builder_class = self.get_op_builder(op_name)
+ if builder_class != None:
+ return builder_class()
+ return None
+
+ # return an op builder class, name specified by class_name
+ def get_op_builder(self, class_name):
+ try:
+ # is op_builder from deepspeed or a 3p version? this should only succeed if it's deepspeed
+ # if successful this also means we're doing a local install and not JIT compile path
+ from op_builder import __deepspeed__ # noqa: F401 # type: ignore
+ from op_builder.cpu import CCLCommBuilder, FusedAdamBuilder, CPUAdamBuilder, NotImplementedBuilder
+ except ImportError:
+ from deepspeed.ops.op_builder.cpu import CCLCommBuilder, FusedAdamBuilder, CPUAdamBuilder, NotImplementedBuilder
+
+ if class_name == "CCLCommBuilder":
+ return CCLCommBuilder
+ elif class_name == "FusedAdamBuilder":
+ return FusedAdamBuilder
+ elif class_name == "CPUAdamBuilder":
+ return CPUAdamBuilder
+ else:
+ # return a NotImplementedBuilder to avoid get NoneType[Name] in unit tests
+ return NotImplementedBuilder
+
+ def build_extension(self):
+ from torch.utils.cpp_extension import BuildExtension
+ return BuildExtension
diff --git a/accelerator/cuda_accelerator.py b/accelerator/cuda_accelerator.py
index 945ba42a3a9c..2786b425ca7f 100644
--- a/accelerator/cuda_accelerator.py
+++ b/accelerator/cuda_accelerator.py
@@ -1,4 +1,7 @@
-'''Copyright The Microsoft DeepSpeed Team'''
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
import os
import pkgutil
@@ -12,30 +15,32 @@
except ImportError:
pass
+# Delay import pynvml to avoid import error when CUDA is not available
+pynvml = None
+
class CUDA_Accelerator(DeepSpeedAccelerator):
+
def __init__(self):
self._name = 'cuda'
self._communication_backend_name = 'nccl'
+ if pynvml is None:
+ self._init_pynvml()
- # begin initialize for create_op_builder()
- # put all valid class name <--> class type mapping into class_dict
- op_builder_dir = self.op_builder_dir()
- op_builder_module = importlib.import_module(op_builder_dir)
-
- for _, module_name, _ in pkgutil.iter_modules([os.path.dirname(op_builder_module.__file__)]):
- # avoid self references
- if module_name != 'all_ops' and module_name != 'builder':
- module = importlib.import_module("{}.{}".format(
- op_builder_dir,
- module_name))
- for member_name in module.__dir__():
- if member_name.endswith(
- 'Builder'
- ) and member_name != "OpBuilder" and member_name != "CUDAOpBuilder" and member_name != "TorchCPUOpBuilder": # avoid abstract classes
- if not member_name in self.class_dict:
- self.class_dict[member_name] = getattr(module, member_name)
- # end initialize for create_op_builder()
+ def _init_pynvml(self):
+ global pynvml
+ try:
+ import pynvml
+ except ImportError:
+ return
+ try:
+ pynvml.nvmlInit()
+ except pynvml.NVMLError:
+ pynvml = None
+ return
+
+ def is_synchronized_device(self):
+ return False
# Device APIs
def device_name(self, device_index=None):
@@ -148,6 +153,31 @@ def max_memory_reserved(self, device_index=None):
def total_memory(self, device_index=None):
return torch.cuda.get_device_properties(device_index).total_memory
+ def _get_nvml_gpu_id(self, torch_gpu_id):
+ """
+ credit: https://discuss.pytorch.org/t/making-pynvml-match-torch-device-ids-cuda-visible-devices/103020
+
+ Remap torch device id to nvml device id, respecting CUDA_VISIBLE_DEVICES.
+
+ If the latter isn't set return the same id
+ """
+ # if CUDA_VISIBLE_DEVICES is used automagically remap the id since pynvml ignores this env var
+ if "CUDA_VISIBLE_DEVICES" in os.environ:
+ ids = list(map(int, os.environ.get("CUDA_VISIBLE_DEVICES", "").split(",")))
+ return ids[torch_gpu_id] # remap
+ else:
+ return torch_gpu_id
+
+ def available_memory(self, device_index=None):
+ if pynvml:
+ if device_index is None:
+ device_index = self.current_device()
+ handle = pynvml.nvmlDeviceGetHandleByIndex(self._get_nvml_gpu_id(device_index))
+ info = pynvml.nvmlDeviceGetMemoryInfo(handle)
+ return info.free
+ else:
+ return self.total_memory(device_index) - self.memory_allocated(device_index)
+
# Data types
def is_bf16_supported(self):
return torch.cuda.is_bf16_supported()
@@ -159,6 +189,9 @@ def is_fp16_supported(self):
else:
return False
+ def supported_dtypes(self):
+ return [torch.float, torch.half, torch.bfloat16]
+
# Misc
def amp(self):
if hasattr(torch.cuda, 'amp'):
@@ -182,6 +215,13 @@ def lazy_call(self, callback):
def communication_backend_name(self):
return self._communication_backend_name
+ def is_triton_supported(self):
+ major, _ = torch.cuda.get_device_capability()
+ if major >= 8:
+ return True
+ else:
+ return False
+
# Tensor operations
@property
@@ -212,9 +252,12 @@ def IntTensor(self):
def LongTensor(self):
return torch.cuda.LongTensor
- def pin_memory(self, tensor):
+ def pin_memory(self, tensor, align_bytes=1):
return tensor.pin_memory()
+ def is_pinned(self, tensor):
+ return tensor.is_pinned()
+
def on_accelerator(self, tensor):
device_str = str(tensor.device)
if device_str.startswith('cuda:'):
@@ -226,7 +269,7 @@ def op_builder_dir(self):
try:
# is op_builder from deepspeed or a 3p version? this should only succeed if it's deepspeed
# if successful this also means we're doing a local install and not JIT compile path
- from op_builder import __deepspeed__ # noqa: F401
+ from op_builder import __deepspeed__ # noqa: F401 # type: ignore
return "op_builder"
except ImportError:
return "deepspeed.ops.op_builder"
@@ -234,10 +277,35 @@ def op_builder_dir(self):
# dict that holds class name <--> class type mapping i.e.
# 'AsyncIOBuilder':