Remove examples for deprecated features

applications/DeepSpeed-Chat/dschat/utils/module/lora.py

@@ -6,7 +6,7 @@
 import torch
 from torch import nn
 import torch.nn.functional as F
-from deepspeed.compression.helper import recursive_getattr, recursive_setattr
+from deepspeed.utils.module_utils import recursive_getattr, recursive_setattr
 import deepspeed
 
 

applications/DeepSpeed-VisualChat/utils/module/lora.py

@@ -6,7 +6,7 @@
 import torch
 from torch import nn
 import torch.nn.functional as F
-from deepspeed.compression.helper import recursive_getattr, recursive_setattr
+from deepspeed.utils.module_utils import recursive_getattr, recursive_setattr
 import deepspeed
 
 

training/BingBertGlue/nvidia/modeling.py

@@ -517,7 +517,7 @@ def __init__(self, config, args):
                 initializer_range=config.initializer_range,
                 local_rank=args.local_rank,
                 seed=args.seed,
-                fp16=ds_config.fp16_enabled,
+                fp16=ds_config.float16_config.enabled,
                 pre_layer_norm=False)
 
             self.layer = nn.ModuleList([

training/BingBertGlue/nvidia/modelingpreln.py

@@ -75,44 +75,6 @@ def get_deepspeed_config(args):
         raise RuntimeError('deepspeed_config is not found in args.')
 
 
-def get_sparse_attention_config(args, num_heads):
-    if args.deepspeed_sparse_attention:
-        ds_config = get_deepspeed_config(args)
-        if hasattr(ds_config,
-                   'sparse_attention') and ds_config.sparse_attention:
-            sa_config = ds_config.sparse_attention
-            sa_mode = sa_config.get('mode')
-            if (sa_mode == 'dense'):
-                from deepspeed.ops.sparse_attention import DenseSparsityConfig as STConfig
-            elif (sa_mode == 'fixed'):
-                from deepspeed.ops.sparse_attention import FixedSparsityConfig as STConfig
-            elif (sa_mode == 'bigbird'):
-                from deepspeed.ops.sparse_attention import BigBirdSparsityConfig as STConfig
-            elif (sa_mode == 'bslongformer'):
-                from deepspeed.ops.sparse_attention import BSLongformerSparsityConfig as STConfig
-            elif (sa_mode == 'variable'):
-                from deepspeed.ops.sparse_attention import VariableSparsityConfig as STConfig
-            else:
-                raise NotImplementedError(
-                    f'Given sparsity mode, {sa_mode}, has not been implemented yet!'
-                )
-            del sa_config['mode']
-            return STConfig(num_heads=num_heads, **sa_config)
-        else:
-            from deepspeed.ops.sparse_attention import FixedSparsityConfig as STConfig
-            print(
-                'deepspeed sparse attention is not set; Fixed sparsity is used as default.'
-            )
-            return STConfig(num_heads=num_heads)
-    else:
-        return None
-
-def get_sparse_attention_utils(sparse_attention_config):
-    if sparse_attention_config is not None:
-        from deepspeed.ops.sparse_attention import SparseAttentionUtils
-        return SparseAttentionUtils
-    return None
-
 def load_tf_weights_in_bert(model, tf_checkpoint_path):
     """ Load tf checkpoints in a pytorch model
     """
@@ -557,17 +519,12 @@ def forward(self, hidden_states, attention_mask):
 
 
 class BertEncoder(nn.Module):
-    def __init__(self, config, args, sparse_attention_config=None):
+    def __init__(self, config, args):
         super(BertEncoder, self).__init__()
 
         #Added later to make it similar to GPT-2
         self.FinalLayerNorm = BertLayerNorm(config.hidden_size, eps=1e-12)
 
-        if args.deepspeed_transformer_kernel and args.deepspeed_sparse_attention:
-            raise NotImplementedError(
-                f'Currently DeepSpeed Transformer Kernels do not support Sparse Attention. To use Sparse Attention, you need to disable Transformer Kernels!'
-            )
-
         if args.deepspeed_transformer_kernel:
             from deepspeed import DeepSpeedTransformerLayer, DeepSpeedTransformerConfig
 
@@ -584,7 +541,7 @@ def __init__(self, config, args, sparse_attention_config=None):
                 local_rank=args.local_rank
                 if hasattr(args, 'local_rank') else -1,
                 seed=args.seed,
-                fp16=ds_config.fp16_enabled,
+                fp16=ds_config.float16_config.enabled,
                 pre_layer_norm=True,
                 # attn_dropout_checkpoint=args.attention_dropout_checkpoint,
                 # normalize_invertible=args.normalize_invertible,
@@ -601,12 +558,6 @@ def __init__(self, config, args, sparse_attention_config=None):
             ])
         else:
             layer = BertLayer(config)
-            if sparse_attention_config is not None:
-                from deepspeed.ops.sparse_attention import BertSparseSelfAttention
-
-                layer.attention.self = BertSparseSelfAttention(
-                    config, sparsity_config=sparse_attention_config)
-
             self.layer = nn.ModuleList([
                 copy.deepcopy(layer) for _ in range(config.num_hidden_layers)
             ])
@@ -995,15 +946,7 @@ class BertModel(BertPreTrainedModel):
     def __init__(self, config, args=None):
         super(BertModel, self).__init__(config)
         self.embeddings = BertEmbeddings(config)
-        # set pad_token_id that is used for sparse attention padding
-        self.pad_token_id = config.pad_token_id if hasattr(
-            config, 'pad_token_id') and config.pad_token_id is not None else 0
-        # set sparse_attention_config if it has been selected
-        self.sparse_attention_config = get_sparse_attention_config(
-            args, config.num_attention_heads)
-        self.sparse_attention_utils = get_sparse_attention_utils(self.sparse_attention_config)
-        self.encoder = BertEncoder(
-            config, args, sparse_attention_config=self.sparse_attention_config)
+        self.encoder = BertEncoder(config, args)
         self.pooler = BertPooler(config)
         self.apply(self.init_bert_weights)
         logger.info("Init BERT pretrain model")
@@ -1035,18 +978,6 @@ def forward(self,
             dtype=next(self.parameters()).dtype)  # fp16 compatibility
         extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0
 
-        # If BertEncoder uses sparse attention, it needs to be padded based on the sparse attention block size
-        if self.sparse_attention_config is not None:
-            pad_len, input_ids, attention_mask, token_type_ids, position_ids, inputs_embeds = self.sparse_attention_utils.pad_to_block_size(
-                block_size=self.sparse_attention_config.block,
-                input_ids=input_ids,
-                attention_mask=extended_attention_mask,
-                token_type_ids=token_type_ids,
-                position_ids=None,
-                inputs_embeds=None,
-                pad_token_id=self.pad_token_id,
-                model_mbeddings=self.embeddings)
-
         embedding_output = self.embeddings(input_ids, token_type_ids)
         encoded_layers = self.encoder(
             embedding_output,
@@ -1056,11 +987,6 @@ def forward(self,
         sequence_output = encoded_layers[-1]
         pooled_output = self.pooler(sequence_output)
 
-        # If BertEncoder uses sparse attention, and input_ids were padded, sequence output needs to be unpadded to original length
-        if self.sparse_attention_config is not None and pad_len > 0:
-            encoded_layers[-1] = self.sparse_attention_utils.unpad_sequence_output(
-                pad_len, encoded_layers[-1])
-
         if not output_all_encoded_layers:
             encoded_layers = encoded_layers[-1]
         return encoded_layers, pooled_output