update model test case, common and module tests passing

allennlp/common/testing/model_test_case.py

@@ -7,7 +7,8 @@
 from allennlp.commands.train import train_model_from_file
 from allennlp.common import Params
 from allennlp.common.testing.test_case import AllenNlpTestCase
-from allennlp.data import DataIterator, DatasetReader, Vocabulary
+from allennlp.data import DatasetReader, Vocabulary
+from allennlp.data.samplers import DataLoader
 from allennlp.data.batch import Batch
 from allennlp.models import load_archive, Model
 
@@ -93,24 +94,27 @@ def ensure_model_can_train_save_and_load(
  params = Params.from_file(param_file, params_overrides=overrides)
  reader = DatasetReader.from_params(params["dataset_reader"])
 
- # Need to duplicate params because Iterator.from_params will consume.
- iterator_params = params["iterator"]
- iterator_params2 = Params(copy.deepcopy(iterator_params.as_dict()))
-
- iterator = DataIterator.from_params(iterator_params)
- iterator2 = DataIterator.from_params(iterator_params2)
-
- # We'll check that even if we index the dataset with each model separately, we still get
- # the same result out.
  print("Reading with original model")
  model_dataset = reader.read(params["validation_data_path"])
- iterator.index_with(model.vocab)
- model_batch = next(iterator(model_dataset, shuffle=False))
+ model_dataset.index_with(model.vocab)
 
  print("Reading with loaded model")
  loaded_dataset = reader.read(params["validation_data_path"])
- iterator2.index_with(loaded_model.vocab)
- loaded_batch = next(iterator2(loaded_dataset, shuffle=False))
+ loaded_dataset.index_with(loaded_model.vocab)
+
+ # Need to duplicate params because DataLoader.from_params will consume.
+ data_loader_params = params["data_loader"]
+ data_loader_params["shuffle"] = False
+ data_loader_params2 = Params(copy.deepcopy(data_loader_params.as_dict()))
+
+ data_loader = DataLoader.from_params(dataset=model_dataset, params=data_loader_params)
+ data_loader2 = DataLoader.from_params(dataset=loaded_dataset, params=data_loader_params2)
+
+ # We'll check that even if we index the dataset with each model separately, we still get
+ # the same result out.
+ model_batch = next(iter(data_loader))
+
+ loaded_batch = next((iter(data_loader2)))
 
  # Check gradients are None for non-trainable parameters and check that
  # trainable parameters receive some gradient if they are trainable.

allennlp/tests/common/params_test.py

@@ -38,14 +38,14 @@ def test_overrides(self):
  overrides = (
  '{ "train_data_path": "FOO", "model": { "type": "BAR" },'
  '"model.text_field_embedder.tokens.type": "BAZ",'
- '"iterator.sorting_keys.0.0": "question"}'
+ '"data_loader.batch_sampler.sorting_keys.0.0": "question"}'
  )
  params = Params.from_file(filename, overrides)
 
  assert "dataset_reader" in params
  assert "trainer" in params
  assert params["train_data_path"] == "FOO"
- assert params["iterator"]["sorting_keys"][0][0] == "question"
+ assert params["data_loader"]["batch_sampler"]["sorting_keys"][0][0] == "question"
 
  model_params = params.pop("model")
  assert model_params.pop("type") == "BAR"

allennlp/tests/fixtures/elmo/config/characters_token_embedder.json

@@ -43,7 +43,7 @@
  ]
  }
  },
- "iterator": {"type": "basic", "batch_size": 32},
+ "data_loader": {"batch_size": 32},
  "trainer": {
  "optimizer": "adam",
  "num_epochs": 5,

allennlp/tests/fixtures/language_model/characters_token_embedder.json

@@ -46,7 +46,7 @@
  ]
  }
  },
- "iterator": {"type": "basic", "batch_size": 32},
+ "data_loader": {"batch_size": 32},
  "trainer": {
  "optimizer": "adam",
  "num_epochs": 5,