[feat] Change unit-tests isolation mechanism to property-based one (a…

…imhubio#2162)

.github/workflows/pull-request.yml

@@ -63,6 +63,7 @@ jobs:
  pytest --cov-report term --cov-branch --cov-fail-under=50 --cov=aim/web --cov=aim/storage --cov=aim/sdk tests
 
  storage-performance-checks:
+ needs: run-checks
  concurrency: perf-tests
  runs-on: [self-hosted, performance-tests]
  name: Performance tests

CHANGELOG.md

@@ -13,6 +13,7 @@
 
 ### Fixes:
 
+- Change unit-tests data isolation mechanism (alberttorosyan)
 - Adjust the visibility of the run color in tables (VkoHov)
 - Fix response headers for remote tracking server (mihran113)
 

tests/api/test_project_api.py

@@ -14,14 +14,16 @@ def test_project_activity_api(self):
  with self.repo.structured_db as db:
  db.create_experiment('My experiment')
 
+ experiment_count = len(self.repo.structured_db.experiments())
+ run_count = len(self.repo.structured_db.runs())
  client = self.client
  response = client.get('/api/projects/activity')
  self.assertEqual(200, response.status_code)
  data = response.json()
  today_gmt = datetime.datetime.now().astimezone(pytz.timezone('gmt')).strftime('%Y-%m-%dT%H:00:00')
- self.assertEqual(10, data['num_runs'])
- self.assertEqual(10, data['activity_map'][today_gmt])
- self.assertEqual(2, data['num_experiments']) # count 'default' experiment
+ self.assertEqual(run_count, data['num_runs'])
+ self.assertEqual(run_count, data['activity_map'][today_gmt])
+ self.assertEqual(experiment_count, data['num_experiments']) # count 'default' experiment
 
  def test_project_params_api(self):
  client = self.client
@@ -67,8 +69,8 @@ def test_project_images_and_metric_info_api(self, qparams):
  self.assertIn('metric', data)
  self.assertIn('images', data)
 
- self.assertSetEqual({'metric1', 'metric2'}, set(data['metric'].keys()))
- self.assertSetEqual({'images1', 'images2'}, set(data['images'].keys()))
+ self.assertTrue({'metric1', 'metric2'}.issubset(set(data['metric'].keys())))
+ self.assertTrue({'images1', 'images2'}.issubset(set(data['images'].keys())))
 
  self.assertEqual(1, len(data['metric']['metric1']))
  self.assertDictEqual({'a': 1}, data['metric']['metric1'][0])

tests/api/test_run_api.py

@@ -13,7 +13,8 @@ class TestRunApi(PrefilledDataApiTestBase):
  def test_search_runs_api(self):
  client = self.client
 
- response = client.get('/api/runs/search/run/', params={'q': 'run["name"] == "Run # 3"',
+ query = self.isolated_query_patch('run["name"] == "Run # 3"')
+ response = client.get('/api/runs/search/run/', params={'q': query,
  'report_progress': False})
  self.assertEqual(200, response.status_code)
 
@@ -27,7 +28,8 @@ def test_search_runs_api(self):
  def test_search_runs_api_paginated(self):
  client = self.client
 
- response = client.get('/api/runs/search/run/', params={'q': 'run["name"] in ["Run # 2","Run # 3"]',
+ query = self.isolated_query_patch('run["name"] in ["Run # 2","Run # 3"]')
+ response = client.get('/api/runs/search/run/', params={'q': query,
  'limit': 1, 'report_progress': False})
  self.assertEqual(200, response.status_code)
 
@@ -39,7 +41,8 @@ def test_search_runs_api_paginated(self):
  offset = run_hash
  self.assertEqual('Run # 3', run['props']['name'])
 
- response = client.get('/api/runs/search/run/', params={'q': 'run["name"] in ["Run # 2","Run # 3"]',
+ query = self.isolated_query_patch('run["name"] in ["Run # 2","Run # 3"]')
+ response = client.get('/api/runs/search/run/', params={'q': query,
  'limit': 5,
  'offset': offset,
  'report_progress': False})
@@ -53,7 +56,8 @@ def test_search_runs_api_paginated(self):
  def test_search_metrics_api_default_step(self):
  client = self.client
 
- response = client.get('/api/runs/search/metric/', params={'q': 'run["name"] == "Run # 3"',
+ query = self.isolated_query_patch('run["name"] == "Run # 3"')
+ response = client.get('/api/runs/search/metric/', params={'q': query,
  'report_progress': False})
  self.assertEqual(200, response.status_code)
 
@@ -79,7 +83,8 @@ def test_search_metrics_api_default_step(self):
  def test_search_metrics_api_custom_step(self, step_count):
  client = self.client
 
- response = client.get('/api/runs/search/metric/', params={'q': 'run["name"] == "Run # 3"',
+ query = self.isolated_query_patch('run["name"] == "Run # 3"')
+ response = client.get('/api/runs/search/metric/', params={'q': query,
  'p': step_count,
  'report_progress': False})
  self.assertEqual(200, response.status_code)
@@ -197,6 +202,6 @@ def test_run_traces_batch_api(self):
  def _find_run_by_name(self, name: str) -> Run:
  repo = self.repo
  for run in repo.iter_runs():
- if run.name == name or run['name'] == name:
+ if run.name == name or run.get('name') == name:
  return run
  return None

tests/api/test_run_images_api.py

@@ -13,7 +13,8 @@ class TestNoImagesRunQueryApi(ApiTestBase):
  def test_query_images_api_empty_result(self):
  client = self.client
 
- response = client.get('/api/runs/search/images/')
+ query = self.isolated_query_patch()
+ response = client.get('/api/runs/search/images/', params={'q': query, 'report_progress': False})
  self.assertEqual(200, response.status_code)
  self.assertEqual(b'', response.content)
 
@@ -22,7 +23,7 @@ class RunImagesTestBase(ApiTestBase):
  @classmethod
  def setUpClass(cls) -> None:
  super().setUpClass()
- run = Run(repo=cls.repo)
+ run = cls.create_run(repo=cls.repo)
  run['images_per_step'] = 16
  for step in range(100):
  images = generate_image_set(img_count=16, caption_prefix=f'Image {step}')
@@ -35,7 +36,8 @@ class TestRunImagesSearchApi(RunImagesTestBase):
  def test_query_images_api_defaults(self):
  client = self.client
 
- response = client.get('/api/runs/search/images/', params={'q': '', 'report_progress': False})
+ query = self.isolated_query_patch()
+ response = client.get('/api/runs/search/images/', params={'q': query, 'report_progress': False})
  self.assertEqual(200, response.status_code)
 
  decoded_response = decode_tree(decode_encoded_tree_stream(response.iter_content(chunk_size=512 * 1024)))
@@ -67,8 +69,9 @@ def test_query_images_api_defaults(self):
  def test_query_images_api_custom_densities_dense(self):
  client = self.client
 
+ query = self.isolated_query_patch()
  response = client.get('/api/runs/search/images/',
- params={'record_density': 200, 'index_density': 10, 'report_progress': False})
+ params={'q': query, 'record_density': 200, 'index_density': 10, 'report_progress': False})
  self.assertEqual(200, response.status_code)
 
  decoded_response = decode_tree(decode_encoded_tree_stream(response.iter_content(chunk_size=512 * 1024),
@@ -84,7 +87,9 @@ def test_query_images_api_custom_densities_dense(self):
  def test_query_images_api_custom_densities_sparse(self):
  client = self.client
 
- response = client.get('/api/runs/search/images/', params={'record_density': 10,
+ query = self.isolated_query_patch()
+ response = client.get('/api/runs/search/images/', params={'q': query,
+ 'record_density': 10,
  'index_density': 4,
  'report_progress': False})
  self.assertEqual(200, response.status_code)
@@ -110,7 +115,9 @@ def test_query_images_api_custom_densities_sparse(self):
  def test_query_images_api_custom_record_ranges(self, input_range, total_range, used_range, count):
  client = self.client
 
- response = client.get('/api/runs/search/images/', params={'record_range': input_range,
+ query = self.isolated_query_patch()
+ response = client.get('/api/runs/search/images/', params={'q': query,
+ 'record_range': input_range,
  'record_density': 100,
  'report_progress': False})
  self.assertEqual(200, response.status_code)
@@ -129,7 +136,9 @@ def test_query_images_api_custom_record_ranges(self, input_range, total_range, u
  def test_query_images_api_calculate_ranges(self):
  client = self.client
 
+ query = self.isolated_query_patch()
  response = client.get('/api/runs/search/images/', params={
+ 'q': query,
  'record_range': '10:20',
  'index_range': '3:6',
  'calc_ranges': True,
@@ -247,7 +256,7 @@ class TestImageListsAndSingleImagesSearchApi(ApiTestBase):
  def setUpClass(cls) -> None:
  super().setUpClass()
 
- run = Run(system_tracking_interval=None)
+ run = cls.create_run(system_tracking_interval=None)
  cls.run_hash = run.hash
 
  for step in range(5):
@@ -258,8 +267,9 @@ def setUpClass(cls) -> None:
  def test_search_simgle_image_only_default_index_range(self):
  client = self.client
 
+ query = self.isolated_query_patch('images.name == "single_images"')
  response = client.get('/api/runs/search/images/',
- params={'q': 'images.name == "single_images"', 'report_progress': False})
+ params={'q': query, 'report_progress': False})
  self.assertEqual(200, response.status_code)
 
  decoded_response = decode_tree(decode_encoded_tree_stream(response.iter_content(chunk_size=512 * 1024)))
@@ -280,7 +290,8 @@ def test_search_simgle_image_only_default_index_range(self):
  def test_mixed_search_default_index_range(self):
  client = self.client
 
- response = client.get('/api/runs/search/images/', params={'q': '', 'report_progress': False})
+ query = self.isolated_query_patch()
+ response = client.get('/api/runs/search/images/', params={'q': query, 'report_progress': False})
  self.assertEqual(200, response.status_code)
 
  decoded_response = decode_tree(decode_encoded_tree_stream(response.iter_content(chunk_size=512 * 1024)))
@@ -307,8 +318,9 @@ def test_mixed_search_default_index_range(self):
  def test_mixed_search_custom_index_range(self):
  client = self.client
 
+ query = self.isolated_query_patch()
  response = client.get('/api/runs/search/images/',
- params={'q': '', 'index_range': '3:5', 'report_progress': False})
+ params={'q': query, 'index_range': '3:5', 'report_progress': False})
  self.assertEqual(200, response.status_code)
 
  decoded_response = decode_tree(decode_encoded_tree_stream(response.iter_content(chunk_size=512 * 1024)))
@@ -347,15 +359,15 @@ def setUpClass(cls) -> None:
  # | -> floats
  # -> context {'subset': 'val'} -> floats
 
- run1 = Run(system_tracking_interval=None)
+ run1 = cls.create_run(system_tracking_interval=None)
  cls.run1_hash = run1.hash
  images = generate_image_set(img_count=2, caption_prefix=f'Image 0')
  run1.track(images, name='image_lists', context={'subset': 'train'})
  run1.track(random.random(), name='floats', context={'subset': 'train'})
  run1.track(random.randint(100, 200), name='integers', context={'subset': 'train'})
  run1.track(random.random(), name='floats', context={'subset': 'val'})
 
- run2 = Run(system_tracking_interval=None)
+ run2 = cls.create_run(system_tracking_interval=None)
  run2.track(images[0], name='single_images', context={'subset': 'val'})
  run2.track(random.random(), name='floats', context={'subset': 'train'})
  run2.track(random.random(), name='floats', context={'subset': 'val'})