PR from branch users/fomars/YANDEXTANK-283

fix phantom reader test

ini config defaults and sections duplicates ARE NOT SUPPORTED any longer, tests fixed

change netort distributions calculation

fix telegraf resource usage

fix netort column ordering in distribution processing

bfg/bin py3programm

change plugin imports paths

Queue -> queue renaming; fix plugin import

projects/bfg py2

py3test -> pytest for firestarter

py3 tests

style fix

fix bfg ya make

style fix

add logging

remove __init__.py from Phantom/config

review fixes

bfg 2020 py3

delete rm 'Bfg2020' from oss_postprocess

version fix

refactor consoleworker

PR from branch users/ival83/yatank-fix-tests
YANDEXTANK-314
mv plugin complete
remove None from bfg2020 plugin
turn on lint and import tests

tankapi multiprocess

consoleworker config load with resource

py3 from github

add decode utf-8

add logging

tankapi py3

kwargs logging

remove extension

add missing parentheses

tankapi py3

__init__.py from resources to srcs

add init py to yamake

add init

correct module naming

fix resource directory naming

fix py schema

fix failures of load_schema

fix yaml resources

configparser

py3 build

commit to merge git changes with py3

ref:6bf7ec34ce4a9546cd674dbd31bb20b09859f034

Dockerfile-test

@@ -1,9 +1,9 @@
 FROM load/yandex-tank-pip:testing
 WORKDIR /yandextank
 RUN apt-get update && \
- apt-get install -y python-pip
-RUN pip install --upgrade setuptools
-RUN pip install --upgrade pip
-RUN pip install pytest
-CMD pip install . && pytest -s
+ apt-get install -y python3-pip
+RUN pip3 install --upgrade setuptools
+RUN pip3 install --upgrade pip
+RUN pip3 install pytest
+CMD pip3 install . && pytest -s
 # docker run -v /path/to/yandextank:/yandextank --name my_container my_image

README.md

@@ -1,8 +1,10 @@
-# Yandex Tank [Gitter](https://gitter.im/yandex/yandex-tank)
+# Yandex Tank [![Build Status](https://travis-ci.org/yandex/yandex-tank.svg?branch=master)](https://travis-ci.org/yandex/yandex-tank) [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/yandex/yandex-tank?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+
+#### Yandextank 1.9.0 now available! Read [release notes](https://github.com/yandex/yandex-tank/wiki/Release-1.9.0).
 
 Yandex.Tank is an extensible open source load testing tool for advanced linux users which is especially good as a part of an automated load testing suite
 
-![Quantiles chart example](https://a.yandex-team.ru/api/tree/blob/trunk/arcadia/load/projects/yandex-tank/logos/screen.png)
+![Quantiles chart example](/logos/screen.png)
 
 ## Main features
 * different load generators supported:

data/PKGBUILD

@@ -7,7 +7,7 @@ pkgdesc='Performance measurement tool'
 arch=('any')
 url='https://github.com/yandex/yandex-tank'
 license=('GPL')
-depends=('python2' 'python2-psutil' 'python2-ipaddr' 'phantom-engine-git')
+depends=('python3' 'python3-psutil' 'python3-ipaddr' 'phantom-engine-git')
 source=("https://ppa.launchpad.net/yandex-load/main/ubuntu/pool/main/y/yandextank/yandextank_${pkgver}.tar.gz")
 sha256sums=("5c2d9d948e1583183a623f430bf0b5327baca4fcbc9d7893f230b364a7aedc70")
 

docker/Dockerfile

@@ -29,7 +29,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
  atop \
  openssh-client \
  git \
- python-pip && \
+ python3-pip && \
  add-apt-repository ppa:yandex-load/main -y && \
  apt-get update -q && \
  apt-get install -yq \
@@ -46,13 +46,13 @@ RUN gpg --keyserver hkp:https://ha.pool.sks-keyservers.net --recv-keys 05CE15085FC09D1
  dpkg -i telegraf_${TELEGRAF_VERSION}_amd64.deb && \
  rm -f telegraf_${TELEGRAF_VERSION}_amd64.deb*
 
-ENV BUILD_DEPS="python-dev build-essential gfortran libssl-dev libffi-dev"
+ENV BUILD_DEPS="python3-dev build-essential gfortran libssl-dev libffi-dev"
 RUN export DEBIAN_FRONTEND=noninteractive && \
  apt-get update && \
  apt-get install -yq --no-install-recommends ${BUILD_DEPS} && \
- pip install --upgrade setuptools && \
- pip install --upgrade pip==9.0.3 &&  \
- pip install https://api.github.com/repos/yandex/yandex-tank/tarball/${BRANCH} && \
+ pip3 install --upgrade setuptools && \
+ pip3 install --upgrade pip==19.2.3 && \
+ pip3 install https://api.github.com/repos/yandex/yandex-tank/tarball/${BRANCH} && \
  apt-get autoremove -y ${BUILD_DEPS} && \
  apt-get clean && \
  rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* /tmp/* /var/tmp/* /root/.cache/*

docker/Dockerfile.mobile

@@ -38,7 +38,7 @@ VOLUME "$USER_HOME_DIR/.m2"
 # some street magic
 RUN echo "y" | android update sdk --no-ui --force --filter platform-tools
 
-RUN pip install uiautomator Appium-Python-Client
+RUN pip3 install uiautomator Appium-Python-Client
 
 RUN curl -sL https://deb.nodesource.com/setup_6.x | bash - && \
  apt install -y nodejs && npm install -g appium

docs/ammo_generators.rst

@@ -4,13 +4,13 @@ Ammo generators
 
 **sample req-style ammo generator (python):**
 
-``usage: cat data | python make_ammo.py``
+``usage: cat data | python3 make_ammo.py``
 For each line of 'data' file this script will generate phantom ammo.
 Line format: ``GET||/url||case_tag||body(optional)``
 
 .. code-block:: python
 
- #!/usr/bin/python
+ #!/usr/bin/env python3
  # -*- coding: utf-8 -*-
  
  import sys
@@ -74,7 +74,7 @@ Line format: ``GET||/url||case_tag||body(optional)``
 
 .. code-block:: python
 
- #!/usr/bin/python
+ #!/usr/bin/python3
  # -*- coding: utf-8 -*-
  import requests
  

docs/core_and_modules.rst

@@ -669,7 +669,7 @@ yaml file section: **bfg**
  Default: ``caseline``.
 
 :pip:
- Install python modules with ``pip install --user`` before the test. If you need multiple modules use multiline options, i.e.:
+ Install python modules with ``pip3 install --user`` before the test. If you need multiple modules use multiline options, i.e.:
 
 ::
 
@@ -1122,7 +1122,7 @@ Example:
  <Memory fielddrop='["active", "inactive", "total", "used_per*", "avail*"]'></Memory>
  <Disk devices='["vda1","sda1","sda2","sda3"]'></Disk>
  <Netstat />
- <Custom diff="1" measure="call" label="test">curl -s -H 'Host: host.tld' 'https://localhost:6100/stat' | python -c 'import sys, json; j = json.load(sys.stdin); print "\n".join(`c["values"]["accept"]` for c in j["charts"] if c["name"] == "localqueue_wait_time")'</Custom>
+ <Custom diff="1" measure="call" label="test">curl -s -H 'Host: host.tld' 'https://localhost:6100/stat' | python3 -c 'import sys, json; j = json.load(sys.stdin); print("\n".join(`c["values"]["accept"]` for c in j["charts"] if c["name"] == "localqueue_wait_time"))'</Custom>
  <Source>/path/to/file</Source>
  <TelegrafRaw>
  [[inputs.ping]]

docs/install.rst

@@ -55,25 +55,25 @@ These are the packages that are required to build different python libraries. In
 
 .. code-block:: bash
 
- sudo apt-get install python-pip build-essential python-dev libffi-dev gfortran libssl-dev
+ sudo apt-get install python3-pip build-essential python3-dev libffi-dev gfortran libssl-dev
 
 Update your pip:
 
 .. code-block:: bash
 
- sudo -H pip install --upgrade pip
+ sudo -H pip3 install --upgrade pip
 
 Update/install your setuptools:
 
 .. code-block:: bash
 
- sudo -H pip install --upgrade setuptools
+ sudo -H pip3 install --upgrade setuptools
 
 Install latest Yandex.Tank from master branch:
 
 .. code-block:: bash
 
- sudo -H pip install https://api.github.com/repos/yandex/yandex-tank/tarball/master
+ sudo -H pip3 install https://api.github.com/repos/yandex/yandex-tank/tarball/master
 
 You'll probably need Phantom load generator, so install it from our ppa:
 
@@ -103,7 +103,7 @@ or this way
 
 .. code-block:: bash
 
- sudo apt-get install python-software-properties
+ sudo apt-get install python3-software-properties
  sudo apt-get install software-properties-common
  sudo add-apt-repository ppa:yandex-load/main
 

docs/intro.rst

@@ -38,10 +38,6 @@ See also
 Evgeniy Mamchits' `phantom <https://github.com/mamchits/phantom>`_ -
 Phantom scalable IO Engine
 
-Phantom `Documentation <https://phantom-doc-ru.readthedocs.io/en/latest/>`_ 
-(sorry, russian only).
-Most useful docs are `result's data format <https://phantom-doc-ru.readthedocs.io/en/latest/analyzing_result_data.html#id1>`_ and `stpd format <https://phantom-doc-ru.readthedocs.io/en/latest/writing-a-phantom-conf.html#ammo-stpd>`_
-
 Alexey Lavrenuke's `pandora <https://github.com/yandex/pandora>`_ -
 A load generator in Go language
 

mocks/shootexec-shooter.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import sys
 import time

setup.py

@@ -19,7 +19,7 @@
  install_requires=[
  'cryptography>=2.2.1', 'pyopenssl==18.0.0',
  'psutil>=1.2.1', 'requests>=2.5.1', 'paramiko>=1.16.0',
- 'pandas==0.24.2', 'numpy==1.15.4', 'future>=0.16.0',
+ 'pandas==0.24.2', 'numpy==1.15.4',
  'pip>=8.1.2',
  'pyyaml>=4.2b1', 'cerberus==1.3.1', 'influxdb>=5.0.0', 'netort>=0.7.10',
  'retrying>=1.3.3', 'pytest-runner', 'typing'
@@ -42,7 +42,7 @@
  'Topic :: Software Development :: Quality Assurance',
  'Topic :: Software Development :: Testing',
  'Topic :: Software Development :: Testing :: Traffic Generation',
- 'Programming Language :: Python :: 2',
+ 'Programming Language :: Python :: 3.7',
  ],
  entry_points={
  'console_scripts': [

yandextank/aggregator/aggregator.py

@@ -160,12 +160,12 @@ def __iter__(self):
  by_tag = list(chunk.groupby([self.groupby]))
  start_time = time.time()
  result = {
- u"ts": ts,
- u"tagged":
+ "ts": ts,
+ "tagged":
  {tag: self.worker.aggregate(data)
  for tag, data in by_tag},
- u"overall": self.worker.aggregate(chunk),
- u"counted_rps": rps
+ "overall": self.worker.aggregate(chunk),
+ "counted_rps": rps
  }
  logger.debug(
  "Aggregation time: %.2fms", (time.time() - start_time) * 1000)

yandextank/aggregator/tests/test_aggregator.py

@@ -1,15 +1,17 @@
-from threading import Event
 import os
+from threading import Event
+
 import pytest as pytest
 
 from yandextank.aggregator import TankAggregator
 from yandextank.common.util import FileMultiReader
 from yandextank.plugins.Phantom.reader import PhantomReader
+
 try:
  from yatest import common
- PATH = common.source_path('load/projects/yandex-tank/yandextank/aggregator/tests/')
+ PATH = common.source_path('load/projects/yandex-tank/yandextank/aggregator/tests')
 except ImportError:
- PATH = 'yandextank/aggregator/tests/'
+ PATH = os.path.dirname(__file__)
 
 
 class PhantomMock(object):
@@ -43,10 +45,10 @@ def on_aggregated_data(self, data, stats):
 
 
 @pytest.mark.parametrize('phout, expected_rps', [
- (os.path.join(PATH, 'phout1'), 300)
+ ('phout1', 300)
 ])
 def test_agregator(phout, expected_rps):
- generator = PhantomMock(phout)
+ generator = PhantomMock(os.path.join(PATH, phout))
  aggregator = TankAggregator(generator)
  listener = ListenerMock(expected_rps)
  aggregator.add_result_listener(listener)

yandextank/aggregator/tests/test_pipeline.py

@@ -1,21 +1,23 @@
 import json
+import os
+
 import numpy as np
 import pytest
-import os
+
 from queue import Queue
 from conftest import MAX_TS, random_split
 
 from yandextank.aggregator import TankAggregator
 from yandextank.aggregator.aggregator import Aggregator, DataPoller
 from yandextank.aggregator.chopper import TimeChopper
 from yandextank.plugins.Phantom.reader import string_to_df
-
 from netort.data_processing import Drain
+
 try:
  from yatest import common
- PATH = common.source_path('load/projects/yandex-tank/yandextank/aggregator/tests/')
+ PATH = common.source_path('load/projects/yandex-tank/yandextank/aggregator/tests')
 except ImportError:
- PATH = 'yandextank/aggregator/tests/'
+ PATH = os.path.dirname(__file__)
 
 AGGR_CONFIG = TankAggregator.load_config()
 
@@ -58,18 +60,18 @@ def producer():
  assert results_queue.qsize() == MAX_TS
 
  @pytest.mark.parametrize('phout, expected_results', [
- (os.path.join(PATH, 'phout2927'), os.path.join(PATH, 'phout2927res.jsonl'))
+ ('phout2927', 'phout2927res.jsonl')
  ])
  def test_invalid_ammo(self, phout, expected_results):
- with open(phout) as fp:
+ with open(os.path.join(PATH, phout)) as fp:
  reader = [string_to_df(line) for line in fp.readlines()]
  pipeline = Aggregator(
  TimeChopper(
  DataPoller(source=reader, poll_period=0),
  cache_size=3),
  AGGR_CONFIG,
  True)
- with open(expected_results) as fp:
+ with open(os.path.join(PATH, expected_results)) as fp:
  expected_results_parsed = json.load(fp)
  for item, expected_result in zip(pipeline, expected_results_parsed):
  for key, expected_value in expected_result.items():

yandextank/api/__init__.py

@@ -1,4 +1,4 @@
 '''
 Package contains all tank tool core code
 '''
-from apiworker import * # noqa
+from .apiworker import * # noqa

yandextank/api/apiworker.py

@@ -2,8 +2,7 @@
 """ Provides class to run TankCore from python """
 import ctypes
 import logging
-from multiprocessing import Value
-from multiprocessing.process import Process
+from multiprocessing import Value, Process
 
 from yandextank.common.util import Cleanup, Finish, Status
 from yandextank.core.tankworker import TankWorker
@@ -15,46 +14,44 @@ class ApiWorker(Process, TankWorker):
  SECTION = 'core'
  FINISH_FILENAME = 'finish_status.yaml'
 
- def __init__(self, configs, cli_options=None, cfg_patches=None, cli_args=None, no_local=False,
+ def __init__(self, manager, config_paths, cli_options=None, cfg_patches=None, cli_args=None, no_local=False,
  log_handlers=None, wait_lock=False, files=None, ammo_file=None):
  Process.__init__(self)
- TankWorker.__init__(self, configs=configs, cli_options=cli_options, cfg_patches=cfg_patches,
+ TankWorker.__init__(self, configs=config_paths, cli_options=cli_options, cfg_patches=cfg_patches,
  cli_args=cli_args, no_local=no_local, log_handlers=log_handlers,
- wait_lock=wait_lock, files=files, ammo_file=ammo_file, api_start=True)
+ wait_lock=wait_lock, files=files, ammo_file=ammo_file, api_start=True, manager=manager)
  self._status = Value(ctypes.c_char_p, Status.TEST_INITIATED)
- self._test_id = Value(ctypes.c_char_p, self.core.test_id)
- self._retcode = None
- self._msg = Value(ctypes.c_char_p, '')
+ self._test_id = Value(ctypes.c_char_p, self.core.test_id.encode('utf8'))
+ self._retcode = Value(ctypes.c_int, 0)
+ self._msg = Value(ctypes.c_char_p, b'')
 
  @property
  def test_id(self):
- return self._test_id.value
+ return self._test_id.value.decode('utf8')
 
  @property
  def status(self):
- return self._status.value
+ return self._status.value.decode('utf8')
 
  @status.setter
  def status(self, val):
  self._status.value = val
 
  @property
  def retcode(self):
- return self._retcode.value if self._retcode is not None else None
+ return self._retcode.value
 
  @retcode.setter
  def retcode(self, val):
- if self._retcode is None:
- self._retcode = Value(ctypes.c_int, val)
- else:
- self._retcode.value = val
+ self._retcode.value = val
 
  @property
  def msg(self):
- return self._msg.value
+ return self._msg.value.decode('utf8')
 
  @msg.setter
  def msg(self, val):
+ val = val.encode('utf8')
  self._msg.value = val
 
  def run(self):

yandextank/common/interfaces.py

@@ -98,7 +98,7 @@ def get_multiline_option(self, option_name, default_value=None):
  def publish(self, key, value):
  """publish value to status"""
  self.log.debug(
- "Publishing status: %s/%s: %s", self.__class__.__name__, key, value)
+ "Publishing status: %s/%s: %s", self.__class__.SECTION, key, value)
  self.core.publish(self.__class__.SECTION, key, value)
 
  def close(self):
@@ -304,9 +304,9 @@ def get_value(self, keys, default=None):
  return value or default
 
  @classmethod
- def _update_dict(cls, plugin_status_dict, keys, value):
+ def _update_dict(cls, status_dict, keys, value):
  if len(keys) > 1:
- cls._update_dict(plugin_status_dict.setdefault(keys[0], {}), keys[1:], value)
+ cls._update_dict(status_dict.setdefault(keys[0], {}), keys[1:], value)
  else:
- plugin_status_dict[keys[0]] = value
- return plugin_status_dict
+ status_dict[keys[0]] = value
+ return status_dict