reformatting

.pre-commit-config.yaml

@@ -13,7 +13,7 @@ repos:
  hooks:
  - id: flake8
 - repo: https://github.com/odwyersoftware/brunette
- rev: b8fc75f460885f986a01842664e0571769b2cc12
+ rev: 238bead5ec5c58935d6bb12c70f435f70b2bf785
  hooks:
  - id: brunette
  entry: brunette --config=setup.cfg

setup.cfg

@@ -87,10 +87,7 @@ single-quotes = true
 
 [flake8]
 max-line-length = 120
-ignore = E231,
 per-file-ignores =
  __init__.py:F401,
  experiment.py:W605,E501,
- statistics.py:E501,W503,
- parser.py:W503,
- test_dao.py:W503
+ statistics.py:E501,

src/epstats/config.py

@@ -8,9 +8,22 @@
  },
  },
  'handlers': {
- 'console': {'class': 'logging.StreamHandler', 'formatter': 'standard', 'stream': 'ext:https://sys.stdout',},
+ 'console': {
+ 'class': 'logging.StreamHandler',
+ 'formatter': 'standard',
+ 'stream': 'ext:https://sys.stdout',
+ },
+ },
+ 'loggers': {
+ 'epstats': {
+ 'level': 'INFO',
+ 'handlers': ['console'],
+ 'propagate': False,
+ },
+ },
+ 'root': {
+ 'level': 'INFO',
+ 'handlers': ['console'],
  },
- 'loggers': {'epstats': {'level': 'INFO', 'handlers': ['console'], 'propagate': False,},},
- 'root': {'level': 'INFO', 'handlers': ['console'],},
  },
 }

src/epstats/locust.py

@@ -31,6 +31,12 @@ def evaluate(self):
  'denominator': 'count(test_unit_type.global.exposure)',
  },
  ],
- 'checks': [{'id': 1, 'name': 'SRM', 'denominator': 'count(test_unit_type.global.exposure)',}],
+ 'checks': [
+ {
+ 'id': 1,
+ 'name': 'SRM',
+ 'denominator': 'count(test_unit_type.global.exposure)',
+ }
+ ],
  },
  )

src/epstats/server/api_evaluate.py

@@ -39,7 +39,8 @@ def _evaluate(experiment: EvExperiment, dao: Dao, statsd: StatsClient):
  _logger.exception(e)
  statsd.incr('errors.experiment')
  raise HTTPException(
- status_code=500, detail=f'Cannot evaluate experiment [{experiment.id}] because of {e}',
+ status_code=500,
+ detail=f'Cannot evaluate experiment [{experiment.id}] because of {e}',
  )
 
  router = APIRouter()

src/epstats/server/req.py

@@ -143,7 +143,9 @@ class Experiment(BaseModel):
  Experiment to evaluate.
  """
 
- id: str = Field(title='Experiment Id',)
+ id: str = Field(
+ title='Experiment Id',
+ )
  control_variant: str = Field(
  title='Identification of the control variant',
  description="""All other variant data
@@ -278,6 +280,12 @@ class Config:
  'denominator': 'count(test_unit_type.global.exposure)',
  }
  ],
- 'checks': [{'id': 1, 'name': 'SRM', 'denominator': 'count(test_unit_type.global.exposure)',}],
+ 'checks': [
+ {
+ 'id': 1,
+ 'name': 'SRM',
+ 'denominator': 'count(test_unit_type.global.exposure)',
+ }
+ ],
  }
  }

src/epstats/server/res.py

@@ -69,7 +69,8 @@ class MetricResult(BaseModel):
  """
 
  id: int = Field(
- title='Metric Id', description="""Database id of the metric, not used at the moment in ep-stats""",
+ title='Metric Id',
+ description="""Database id of the metric, not used at the moment in ep-stats""",
  )
  name: str = Field(
  title='Metric Name',
@@ -112,7 +113,8 @@ class CheckResult(BaseModel):
  """
 
  id: int = Field(
- title='Check Id', description='Database id of the check, not used at the moment.',
+ title='Check Id',
+ description='Database id of the check, not used at the moment.',
  )
  name: str = Field(
  title='Check Name',
@@ -137,7 +139,8 @@ class ExposureStat(BaseModel):
 
  exp_variant_id: str = Field(title='Variant in the Experiment')
  count: int = Field(
- title='Per-variant exposures', description="""Exposure count of experiment (randomization) unit.""",
+ title='Per-variant exposures',
+ description="""Exposure count of experiment (randomization) unit.""",
  )
 
  @staticmethod
@@ -156,7 +159,8 @@ class ExposureResult(BaseModel):
  needed to correctly retrieve number of exposures per experiment variant.""",
  )
  stats: List[ExposureStat] = Field(
- title='Experiment Exposures', description="""List with experiment variant exposure counts per entry.""",
+ title='Experiment Exposures',
+ description="""List with experiment variant exposure counts per entry.""",
  )
 
  @staticmethod
@@ -171,12 +175,16 @@ class Result(BaseModel):
  Top-level element in the response.
  """
 
- id: str = Field(title='Experiment Id',)
+ id: str = Field(
+ title='Experiment Id',
+ )
  metrics: List[MetricResult] = Field(
- title='Metric Results', description="""List with one entry per evaluated metric.""",
+ title='Metric Results',
+ description="""List with one entry per evaluated metric.""",
  )
  checks: List[CheckResult] = Field(
- title='Check Results', description="""List with one entry per evaluated check.""",
+ title='Check Results',
+ description="""List with one entry per evaluated check.""",
  )
  exposure: ExposureResult = Field(title='Experiment Exposures')
 

src/epstats/toolkit/check.py

@@ -94,7 +94,11 @@ class SrmCheck(Check):
  """
 
  def __init__(
- self, id: int, name: str, denominator: str, confidence_level: float = 0.999,
+ self,
+ id: int,
+ name: str,
+ denominator: str,
+ confidence_level: float = 0.999,
  ):
  super().__init__(id, name, denominator)
  self.confidence_level = confidence_level

src/epstats/toolkit/experiment.py

@@ -127,8 +127,30 @@ def __init__(
  self.confidence_level = confidence_level
  self.variants = variants
  self._exposure_goals = [
- EpGoal(['count', '(', UnitType([unit_type]), '.', AggType(['global']), '.', Goal(['exposure']), ')',]),
- EpGoal(['count', '(', UnitType([unit_type]), '.', AggType(['unit']), '.', Goal(['exposure']), ')',]),
+ EpGoal(
+ [
+ 'count',
+ '(',
+ UnitType([unit_type]),
+ '.',
+ AggType(['global']),
+ '.',
+ Goal(['exposure']),
+ ')',
+ ]
+ ),
+ EpGoal(
+ [
+ 'count',
+ '(',
+ UnitType([unit_type]),
+ '.',
+ AggType(['unit']),
+ '.',
+ Goal(['exposure']),
+ ')',
+ ]
+ ),
  ]
  self.statsd = statsd
 
@@ -210,7 +232,10 @@ def evaluate_agg(self, goals: pd.DataFrame) -> Evaluation:
  """
  g = self._fix_missing_agg(goals)
  return self._evaluate(
- g, Experiment._metrics_column_fce_agg, Experiment._checks_fce_agg, Experiment._exposures_fce_agg,
+ g,
+ Experiment._metrics_column_fce_agg,
+ Experiment._checks_fce_agg,
+ Experiment._exposures_fce_agg,
  )
 
  def evaluate_by_unit(self, goals: pd.DataFrame) -> Evaluation:
@@ -294,7 +319,15 @@ def evaluate_by_unit(self, goals: pd.DataFrame) -> Evaluation:
  pd.pivot_table(
  g,
  values=['count', 'sum_value'],
- index=['exp_id', 'exp_variant_id', 'unit_type', 'agg_type', 'unit_id', 'dimension', 'dimension_value',],
+ index=[
+ 'exp_id',
+ 'exp_variant_id',
+ 'unit_type',
+ 'agg_type',
+ 'unit_id',
+ 'dimension',
+ 'dimension_value',
+ ],
  columns='goal',
  aggfunc=np.sum,
  fill_value=0,
@@ -445,7 +478,16 @@ def _fix_missing_agg(self, goals: pd.DataFrame) -> pd.DataFrame:
  # join to existing data and use zeros for only missing variants and goals
  m = (
  pd.concat([g, empty_df], axis=0)
- .groupby(['exp_variant_id', 'unit_type', 'agg_type', 'dimension', 'dimension_value', 'goal',])
+ .groupby(
+ [
+ 'exp_variant_id',
+ 'unit_type',
+ 'agg_type',
+ 'dimension',
+ 'dimension_value',
+ 'goal',
+ ]
+ )
  .sum()
  .reset_index()
  )

src/epstats/toolkit/testing/utils.py

@@ -40,25 +40,49 @@ def assert_experiment(experiment: Experiment, target: Evaluation, test_dao: Test
  assert_exposures(experiment.id, target.exposures, test_dao, unit_type=experiment.unit_type)
 
 
-def assert_metrics(experiment_id: str, metric_id: int, target: pd.DataFrame, test_dao: TestDao, precision=3,) -> None:
+def assert_metrics(
+ experiment_id: str,
+ metric_id: int,
+ target: pd.DataFrame,
+ test_dao: TestDao,
+ precision=3,
+) -> None:
  target = target[(target.exp_id == experiment_id) & (target.metric_id == metric_id)]
 
  expected = test_dao.load_evaluations_metrics(experiment_id)
  expected = expected[expected.metric_id == metric_id]
 
  assert_array_equal(target.exp_variant_id, expected.exp_variant_id)
- t = target[['sum_value', 'diff', 'mean', 'p_value', 'confidence_interval', 'confidence_level',]].astype(float)
+ t = target[
+ [
+ 'sum_value',
+ 'diff',
+ 'mean',
+ 'p_value',
+ 'confidence_interval',
+ 'confidence_level',
+ ]
+ ].astype(float)
  atol = 10 ** -precision
  assert allclose(t['sum_value'], expected['sum_value'], atol=atol, equal_nan=True)
  assert allclose(t['diff'], expected['diff'], atol=atol, equal_nan=True)
  assert allclose(t['mean'], expected['mean'], atol=atol, equal_nan=True)
  assert allclose(t['p_value'], expected['p_value'], atol=atol * 10, equal_nan=True)
- assert allclose(t['confidence_interval'], expected['confidence_interval'], atol=atol * 10, equal_nan=True,)
+ assert allclose(
+ t['confidence_interval'],
+ expected['confidence_interval'],
+ atol=atol * 10,
+ equal_nan=True,
+ )
  assert allclose(t['confidence_level'], expected['confidence_level'], atol=atol, equal_nan=True)
 
 
 def assert_checks(
- experiment_id: str, check_id: int, target: pd.DataFrame, test_dao: TestDao, precision: int = 4,
+ experiment_id: str,
+ check_id: int,
+ target: pd.DataFrame,
+ test_dao: TestDao,
+ precision: int = 4,
 ) -> None:
  target = target[(target.exp_id == experiment_id) & (target.check_id == check_id)]
 

tests/epstats/server/test_api_evaluate.py

@@ -34,7 +34,13 @@ def test_conversion_evaluate():
  'denominator': 'count(test_unit_type.global.exposure)',
  }
  ],
- 'checks': [{'id': 1, 'name': 'SRM', 'denominator': 'count(test_unit_type.global.exposure)',}],
+ 'checks': [
+ {
+ 'id': 1,
+ 'name': 'SRM',
+ 'denominator': 'count(test_unit_type.global.exposure)',
+ }
+ ],
  }
 
  resp = client.post('/evaluate', json=json_blob)
@@ -55,7 +61,13 @@ def test_real_valued_evaluate():
  'denominator': 'count(test_unit_type.global.exposure)',
  }
  ],
- 'checks': [{'id': 1, 'name': 'SRM', 'denominator': 'count(test_unit_type.global.exposure)',}],
+ 'checks': [
+ {
+ 'id': 1,
+ 'name': 'SRM',
+ 'denominator': 'count(test_unit_type.global.exposure)',
+ }
+ ],
  }
 
  resp = client.post('/evaluate', json=json_blob)
@@ -82,7 +94,13 @@ def test_multiple_evaluate():
  'denominator': 'count(test_unit_type.global.exposure)',
  },
  ],
- 'checks': [{'id': 1, 'name': 'SRM', 'denominator': 'count(test_unit_type.global.exposure)',}],
+ 'checks': [
+ {
+ 'id': 1,
+ 'name': 'SRM',
+ 'denominator': 'count(test_unit_type.global.exposure)',
+ }
+ ],
  }
 
  resp = client.post('/evaluate', json=json_blob)