[build-system] requires = ["setuptools", "wheel"] # uncomment to enable pep517 after versioneer problem is fixed. # https://github.com/python-versioneer/python-versioneer/issues/193 # build-backend = "setuptools.build_meta" [tool.mypy] python_version = "3.9" plugins = [ "pydantic.mypy", "sqlalchemy.ext.mypy.plugin" ] files = [ "great_expectations", "docs", "tests", # "contrib" # ignore entire `contrib` package ] warn_unused_configs = true ignore_missing_imports = false follow_imports = 'normal' warn_redundant_casts = true show_error_codes = true enable_error_code = [ 'ignore-without-code', 'explicit-override', 'possibly-undefined' ] # The following list of codes are globally ignored, do not add to this list disable_error_code = [ # annotation-unchecked are 'warning notes', not errors and won't cause mypy to fail # but it does create a lot of noise in the CI mypy step. # https://mypy-lang.blogspot.com/2022/11/mypy-0990-released.html 'annotation-unchecked', ] exclude = [ # BEGIN ALWAYS EXCLUDE SECTION ##################################################### # If pattern should always be excluded add comment explaining why and put # Docs should not be type checked with the rest of the library. 'docs/docusaurus', 'docs/expectation_gallery', 'docs/readme_assets', 'docs/sphinx_api_docs_source', '_version\.py', # generated by `versioneer` 'v012', # legacy code 'datasource/data_connector/configured_asset_sql_data_connector\.py', # 37 - This is legacy code and will not be typed. 'core/usage_statistics/anonymizers/batch_anonymizer\.py', # 10 - This code will be removed in 1.0 'core/usage_statistics/anonymizers/batch_request_anonymizer\.py', # 16 - This code will be removed in 1.0 'core/usage_statistics/anonymizers/checkpoint_anonymizer\.py', # 16 - This code will be removed in 1.0 'core/usage_statistics/anonymizers/data_docs_anonymizer\.py', # 5 - This code will be removed in 1.0 'core/usage_statistics/anonymizers/datasource_anonymizer\.py', # 9 - This code will be removed in 1.0 'core/usage_statistics/anonymizers/expectation_anonymizer\.py', # 6 - This code will be removed in 1.0 'core/usage_statistics/anonymizers/validation_operator_anonymizer\.py', # 5 - This code will be removed in 1.0 # END ALWAYS EXCLUDE SECTION ###################################################### # # ################################################################################# # TODO: complete typing for the following modules and remove from exclude list # number is the current number of typing errors for the excluded pattern 'expectations/core/expect_column_values_to_be_of_type\.py', # 6 'expectations/core/expect_column_values_to_not_match_regex_list\.py', # 2 'expectations/core/expect_column_values_to_not_match_regex\.py', # 2 'expectations/core/expect_column_values_to_not_match_like_pattern_list\.py', # 3 'expectations/core/expect_column_values_to_not_match_like_pattern\.py', # 2 'expectations/core/expect_column_values_to_not_be_in_set\.py', # 2 'expectations/core/expect_column_values_to_match_strftime_format\.py', # 2 'expectations/core/expect_column_values_to_match_regex_list\.py', # 2 'expectations/core/expect_column_values_to_match_regex\.py', # 1 'expectations/core/expect_column_values_to_match_like_pattern_list\.py', # 3 'expectations/core/expect_column_values_to_match_like_pattern\.py', # 2 'expectations/core/expect_column_values_to_match_json_schema\.py', # 1 'expectations/core/expect_column_values_to_be_null\.py', # 3 'expectations/core/expect_column_values_to_be_json_parseable\.py', # 1 'expectations/core/expect_column_values_to_be_increasing\.py', # 1 'expectations/core/expect_column_values_to_be_in_type_list\.py', # 6 'expectations/core/expect_column_values_to_be_decreasing\.py', # 1 'expectations/core/expect_column_values_to_be_dateutil_parseable\.py', # 1 'expectations/core/expect_column_unique_value_count_to_be_between\.py', # 1 'expectations/core/expect_column_stdev_to_be_between\.py', # 1 'expectations/core/expect_column_quantile_values_to_be_between\.py', # 15 'expectations/core/expect_column_value_lengths_to_equal\.py', # 1 'expectations/core/expect_column_value_lengths_to_be_between\.py', # 5 'expectations/core/expect_column_proportion_of_unique_values_to_be_between\.py', # 1 'expectations/core/expect_column_values_to_be_in_set\.py', # 2 'expectations/core/expect_column_values_to_be_equal\.py', # 3 'expectations/core/expect_column_values_a_to_be_greater_than_b\.py', # 3 'expectations/core/expect_column_pair_cramers_phi_value_to_be_less_than\.py', # 7 'expectations/core/expect_column_most_common_value_to_be_in_set\.py', # 3 'expectations/core/expect_column_kl_divergence_to_be_less_than\.py', # 22 'expectations/core/expect_column_pair_values_to_be_in_set\.py', # 2 'expectations/core/expect_column_pair_values_to_be_equal\.py', # 6 'expectations/core/expect_column_pair_values_a_to_be_greater_than_b\.py', # 3 'expectations/core/expect_column_distinct_values_to_be_in_set\.py', # 1 'expectations/core/expect_compound_columns_to_be_unique\.py', # 3 'expectations/core/expect_multicolumn_sum_to_equal\.py', # 4 'expectations/core/expect_multicolumn_values_to_be_unique\.py', # 3 'expectations/core/expect_select_column_values_to_be_unique_within_record\.py', # 3 'expectations/core/expect_table_columns_to_match_set\.py', # 8 'expectations/core/expect_table_columns_to_match_ordered_list\.py', # 11 'expectations/core/expect_table_column_count_to_equal\.py', # 5 'expectations/set_based_column_map_expectation\.py', # 3 'render/renderer/content_block/content_block\.py', # 5 'render/renderer/content_block/exception_list_content_block\.py', # 4 'render/renderer/page_renderer\.py', # 10 'render/renderer/profiling_results_overview_section_renderer\.py', # 2 'render/renderer/site_builder\.py', # 3 'render/renderer/slack_renderer\.py', # 9 'rule_based_profiler/domain_builder/map_metric_column_domain_builder\.py', # 8 'rule_based_profiler/estimators/bootstrap_numeric_range_estimator\.py', # 8 'rule_based_profiler/estimators/kde_numeric_range_estimator\.py', # 7 'rule_based_profiler/expectation_configuration_builder', # 13 'rule_based_profiler/helpers/util\.py', # 46 'rule_based_profiler/parameter_builder/unexpected_count_statistics_multi_batch_parameter_builder\.py', # 69 'rule_based_profiler/parameter_builder/mean_unexpected_map_metric_multi_batch_parameter_builder\.py', # 19 'rule_based_profiler/parameter_builder/metric_multi_batch_parameter_builder\.py', # 15 'rule_based_profiler/parameter_builder/numeric_metric_range_multi_batch_parameter_builder\.py', # 27 'rule_based_profiler/parameter_builder/parameter_builder\.py', # 40 'rule_based_profiler/parameter_builder/partition_parameter_builder\.py', # 9 'rule_based_profiler/parameter_builder/regex_pattern_string_parameter_builder\.py', # 21 'rule_based_profiler/parameter_builder/simple_date_format_string_parameter_builder\.py', # 20 'rule_based_profiler/rule_based_profiler\.py', # 40 # tests 'tests/actions/test_core_actions\.py', 'tests/checkpoint/test_checkpoint_result_format\.py', 'tests/checkpoint/test_checkpoint\.py', 'tests/core/test_batch\.py', 'tests/core/test_expectation_configuration\.py', 'tests/core/test_expectation_suite\.py', 'tests/core/test_expectation_validation_result\.py', 'tests/core/test_validation_definition\.py', 'tests/core/test_yaml_handler\.py', 'tests/data_context/abstract_data_context/test_abstract_data_context_datasource_crud\.py', 'tests/data_context/abstract_data_context/test_data_docs_config_crud\.py', 'tests/data_context/cloud_data_context/test_include_rendered_content\.py', 'tests/data_context/fixtures/plugins/extended_checkpoint\.py', 'tests/data_context/migrator', 'tests/data_context/store/test_configuration_store\.py', 'tests/data_context/store/test_data_asset_store\.py', 'tests/data_context/store/test_datasource_store\.py', 'tests/data_context/store/test_store_backends\.py', 'tests/data_context/test_data_context_in_code_config\.py', 'tests/data_context/test_data_context_state_management\.py', 'tests/data_context/test_data_context_variables\.py', 'tests/datasource/data_connector', 'tests/datasource/fluent/tasks\.py', 'tests/datasource/test_datasource_dict\.py', 'tests/datasource/test_new_datasource_with_sql_data_connector\.py', 'tests/datasource/test_simple_sqlalchemy_datasource\.py', 'tests/execution_engine/partition_and_sample', 'tests/expectations', 'tests/experimental/metric_repository/test_column_descriptive_metrics_metric_retriever_integration\.py', 'tests/experimental/metric_repository/test_column_descriptive_metrics_metric_retriever\.py', 'tests/experimental/metric_repository/test_metric_list_metric_retriever_integration\.py', 'tests/experimental/metric_repository/test_metric_list_metric_retriever\.py', 'tests/integration/common_workflows', 'tests/integration/db', 'tests/integration/docusaurus/connecting_to_your_data', 'tests/integration/docusaurus/deployment_patterns', 'tests/integration/docusaurus/expectations', 'tests/integration/docusaurus/reference', 'tests/integration/docusaurus/setup', 'tests/integration/docusaurus/tutorials', 'tests/integration/docusaurus/validation', 'tests/integration/fixtures/partition_and_sample_data', 'tests/integration/fixtures/yellow_tripdata_pandas_fixture', 'tests/integration/spark/test_spark_config\.py', 'tests/integration/test_definitions', 'tests/integration/test_script_runner\.py', 'tests/performance', 'tests/render', 'tests/rule_based_profiler', 'tests/test_fixtures/rule_based_profiler/plugins', 'tests/test_utils\.py', 'tests/validator/test_metric_configuration\.py', 'tests/validator/test_metrics_calculator\.py', 'tests/validator/test_validation_graph\.py', ] [[tool.mypy.overrides]] # TODO: remove these overrides once we have typed the modules module = [ "contrib.experimental.great_expectations_experimental.*", "great_expectations.checkpoint.checkpoint", "great_expectations.compatibility.docstring_parser", "great_expectations.data_context.*", "great_expectations.dataset.*", "great_expectations.expectations.core.*", "great_expectations.expectations.metrics.column_map_metrics.*", "great_expectations.expectations.metrics.column_pair_map_metrics.*", "great_expectations.experimental.rule_based_profiler.*", "great_expectations.render.renderer.*", "great_expectations.self_check.*", "tests.rule_based_profiler.*", "tests.test_utils", ] follow_imports = 'silent' [[tool.mypy.overrides]] # need to use override because a mypy bug prevents ignoring an assignment warning inline # for `from azure import storage` module = ["great_expectations.compatibility.azure"] disable_error_code = [ 'assignment', # cannot assign NotImported to a ModuleType ] [[tool.mypy.overrides]] module = [ "altair.*", "boto3.*", "botocore.*", "clickhouse_sqlalchemy.*", "databricks.*", "google.*", "great_expectations.compatibility.pydantic.*", "mistune.*", "moto.*", "pact.*", "posthog.*", "pyarrow.*", "pyfakefs.*", "pypd.*", "pytest_timeout.*", "ruamel.*", "scipy.*", "sempy.*", "shapely.*", "snowflake.*", "sqlalchemy_bigquery.*", "sqlalchemy_dremio.*", "sqlalchemy_redshift.*", "sqlalchemy.*", # remove once we are using sqlalchemy 2 in type-checking step "teradatasqlalchemy.*", "trino.*", ] ignore_missing_imports = true [tool.pydantic-mypy] # https://pydantic-docs.helpmanual.io/mypy_plugin/#plugin-settings init_typed = true warn_required_dynamic_aliases = true warn_untyped_fields = true [tool.ruff] target-version = "py39" line-length = 100 lint.preview = true # This enables preview rules for specified rules e.g. NPY201 lint.explicit-preview-rules = true # https://docs.astral.sh/ruff/preview/#selecting-single-preview-rules lint.select = [ # https://beta.ruff.rs/docs/rules/#pyflakes-f "F", # Pyflakes # https://beta.ruff.rs/docs/rules/#pycodestyle-e-w "E", # pycodestyle "W", # Warning # https://beta.ruff.rs/docs/rules/#flake8-comprehensions-c4 # https://beta.ruff.rs/docs/rules/#mccabe-c90 "C", # Complexity (mccabe+) & comprehensions # https://beta.ruff.rs/docs/rules/#pyupgrade-up "UP", # pyupgrade # https://beta.ruff.rs/docs/rules/#isort-i "I", # isort # https://beta.ruff.rs/docs/rules/#flake8-type-checking-tch "TCH", # flake8-type-checking-tch # https://beta.ruff.rs/docs/rules/#flake8-tidy-imports-tid "TID", # flake8-tidy-imports # https://beta.ruff.rs/docs/rules/#flake8-pyi-pyi "PYI", # flake8-pyi - type stub files # https://beta.ruff.rs/docs/rules/#flake8-use-pathlib-pth "PTH", # use-pathlib - use pathlib for os.path and glob operations # https://beta.ruff.rs/docs/rules/#flake8-bugbear-b "B", # bugbear - common python bugs & design issues # https://beta.ruff.rs/docs/rules/#flake8-datetimez-dtz "DTZ", # flake8-datetimez-dtz - prevent use of tz naive datetimes # https://beta.ruff.rs/docs/rules/#pylint-pl "PL", # pylint # https://beta.ruff.rs/docs/rules/#ruff-specific-rules-ruf "RUF", # custom ruff rules # https://docs.astral.sh/ruff/rules/#numpy-specific-rules-npy "NPY", # NumPy-specific rules TODO: Enable after fixing or ignoring errors "NPY201", # numpy 2.0 compatibility (see https://numpy.org/devdocs/numpy_2_0_migration_guide.html#ruff-plugin) -- Explicitly listed to view preview rule errors # https://docs.astral.sh/ruff/rules/#tryceratops-try "TRY", # TRYceratops # https://docs.astral.sh/ruff/rules/#flake8-simplify-sim "SIM", # flake8-simplify ] lint.ignore = [ # formatting related ignores # https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules "W191", # tab-identation "E111", # indentation-with-invalid-multiple "E114", # indentation-with-invalid-multiple-comment "E117", # over-idented # https://beta.ruff.rs/docs/rules/#pyflakes-f "F842", # variable annotated but unused # TODO enable # https://beta.ruff.rs/docs/rules/#pycodestyle-e-w "E402", # module level import not at top of file # https://beta.ruff.rs/docs/rules/#flake8-comprehensions-c4 "C400", # TODO enable "C408", # TODO enable "C409", # TODO enable "C413", # TODO enable "C414", # TODO enable "C416", # TODO enable "C417", # TODO enable # https://beta.ruff.rs/docs/rules/#pyupgrade-up "UP006", # use-pep585-annotation "UP007", # use-pep604-annotation # https://beta.ruff.rs/docs/rules/#flake8-type-checking-tch # minimal cost for standard lib imports; keep this disabled "TCH003", # typing-only-standard-library-import # gives false positives if we use try imports and type-checking import "TCH004", # runtime-import-in-type-checking-block "TID252", # Relative imports from parent modules are banned # https://beta.ruff.rs/docs/rules/#flake8-use-pathlib-pth "PTH123", # pathlib-open - this would force pathlib usage anytime open or with open was used. # https://beta.ruff.rs/docs/rules/#flake8-pyi-pyi "PYI053", # string-or-bytes-too-long - causes mypy to fail on some of our type stubs "PYI054", # numeric-literal-too-long - causes mypy to fail on some of our type stubs "UP035", # TODO: remove once min version of pydantic supports using collections.abc types # https://beta.ruff.rs/docs/rules/#flake8-bugbear-b # TODO: enable these "B904", # raise-without-from-inside-except "B028", # no-explicit-stacklevel - https://beta.ruff.rs/docs/rules/no-explicit-stacklevel/ "B007", # unused-loop-control-variable # TODO: enable remaining ruf rules in followup PRs "RUF005", # collection-literal-concatenation "RUF012", # mutable-class-default - too many violations "RUF015", # unnecessary-iterable-allocation-for-first-element - requires more careful review "TRY300", # Consider moving this statement to an `else` block - we don't use this; this is kind of weird # TODO: enable these a few at a time (ordered by number of violations) "SIM108", # if-else-block-instead-of-if-exp # Not enabling: This is not compatible with type narrowing "SIM102", # collapsible-if "SIM105", # suppressible-exception "SIM117", # multiple-with-statements "SIM910", # dict-get-with-none-default "SIM401", # if-else-block-instead-of-dict-get "SIM115", # open-file-with-context-handler "SIM202", # negate-not-equal-op "SIM210", # if-expr-with-true-false "SIM112", # uncapitalized-environment-variables ] extend-exclude = [ "docs/docusaurus/versioned_docs/version-0.18", "docs/docusaurus/versioned_docs/version-0.17", "build/*", "versioneer*", "examples/*", # TODO: remove the items below and fix linting issues "tests/data_asset", # 10 warnings "tests/dataset", # 10 warnings ] [tool.ruff.lint.per-file-ignores] "ci/checks" = [ "E501", # line too long - too many violation & not relevant for CI checks ] "assets/benchmark/benchmark.py" = [ "DTZ", # flake8-datetimez-dtz - doesn't matter for benchmark tests ] "assets/scripts/build_gallery.py" = [ "PLR0912", # Too many branches - scripts are not part of the main codebase "PLR0913", # Too many arguments - scripts are not part of the main codebase "PLR0915", # Too many statements - scripts are not part of the main codebase "E501", # line too long - scripts are not part of the main codebase ] "__init__.py" = [ "F401", # unused import "F403", # star imports "PTH207", # use glob - __all__ needs to be list of str, not Path ] "*.pyi" = [ "TID251", # flake8-banned-api - type stubs are not executed ] "great_expectations/_version.py" = [ "PLR", # pylint - versioneer code ] "great_expectations/compatibility/*.py" = [ "TID251", # flake8-banned-api ] "tasks.py" = [ "PLR0913", "TID251", # not part of core code ] "tests/integration/docusaurus/*.py" = [ "F841", # local variable is assigned to but never used - ignored for code used in snippets "E501", # may affect how code is displayed in the docs ] [tool.ruff.lint.flake8-type-checking] # pydantic models use annotations at runtime runtime-evaluated-base-classes = [ # NOTE: ruff is unable to detect that these are subclasses of pydantic.BaseModel "pydantic.BaseModel", "great_expectations.checkpoint.checkpoint.Checkpoint", "great_expectations.datasource.fluent.fluent_base_model.FluentBaseModel", "great_expectations.datasource.fluent.interfaces.Datasource", "great_expectations.datasource.fluent.sql_datasource.SQLDatasource", "great_expectations.compatibility.pydantic.BaseModel", ] runtime-evaluated-decorators = ["pydantic.dataclasses.dataclass"] [tool.ruff.lint.mccabe] # https://docs.astral.sh/ruff/rules/complex-structure/ max-complexity = 8 [tool.ruff.lint.pydocstyle] convention = "google" [tool.ruff.lint.flake8-tidy-imports.banned-api] "os.environ".msg = """Please do not use os.environ outside of configuration files. If you are working in a configuration file you may use the inline comment \ "# noqa: TID251 # os.environ allowed in config files" to ignore this error.""" "sqlalchemy".msg = "Please do not import sqlalchemy directly, import from great_expectations.compatibility.sqlalchemy instead." "pyspark".msg = "Please do not import pyspark directly, import from great_expectations.compatibility.pyspark instead." "boto3".msg = "Please do not import boto3 directly, import from great_expectations.compatibility.aws instead." "google".msg = "Please do not import google directly, import from great_expectations.compatibility.google instead." "azure".msg = "Please do not import azure directly, import from great_expectations.compatibility.azure instead." "trino".msg = "Please do not import trino directly, import from great_expectations.compatibility.trino instead." "pyarrow".msg = "Please do not import pyarrow directly, import from great_expectations.compatibility.pyarrow instead." "typing_extensions.override".msg = "Do not import typing_extensions.override directly, import `override` from great_expectations.compatibility.typing_extensions instead." # TODO: remove pydantic once our min version is pydantic v2 "pydantic".msg = "Please do not import pydantic directly, import from great_expectations.compatibility.pydantic instead." "pkg_resources".msg = "pkg_resources module has been deprecated. Use importlib.resources or importlib.metada instead." "great_expectations.util.convert_to_json_serializable".msg = "Use pydantic for serialization." "great_expectations.util.ensure_json_serializable".msg = "Use pydantic for serialization." # ----------------------------------------------------------------- [tool.pytest.ini_options] # https://docs.python.org/3/library/warnings.html#the-warnings-filter filterwarnings = [ # Turn all warnings not explicitly filtered below into errors "error", # PendingDeprecationWarning: these relate to the use of methods that are not even deprecated yet. # We want to see the warnings in the logs, but not have them cause CI to fail. "once::PendingDeprecationWarning", # Generally we shouldn't need to use a context manger with `mocker` but it can be useful # if we want the patch/mock to end before the test finishes. # Example: Mocks returned by pytest-mock do not need to be used as context managers. # The mocker fixture automatically undoes mocking at the end of a test. This warning can be ignored if it was triggered by mocking a context manager. # https://pytest-mock.readthedocs.io/en/latest/remarks.html#usage-as-context-manager "default::pytest_mock.plugin.PytestMockWarning", # Find and fix ruff DTZ violations 'once: datetime.datetime.utc.*\(\) is deprecated .*:DeprecationWarning', # the pkg_resources module distributed with setuptools has been deprecated # we should never use it but it is used in some of our dependencies # https://setuptools.pypa.io/en/latest/pkg_resources.html "once:pkg_resources is deprecated as an API:DeprecationWarning", # ruamel, google, etc. "once:Deprecated call to `pkg_resources.declare_namespace:DeprecationWarning", # This warning is common during testing where we intentionally use a COMPLETE format even in cases that would # be potentially overly resource intensive in standard operation "ignore:Setting result format to COMPLETE for a SqlAlchemyDataset:UserWarning", # This deprecation warning was fixed in moto release 1.3.15, and the filter should be removed once we migrate # to that minimum version "ignore:Using or importing the ABCs:DeprecationWarning:moto.cloudformation.parsing", # This deprecation warning comes from getsentry/responses, a mocking utility for requests. It is a dependency in moto. "ignore:stream argument is deprecated. Use stream parameter in request directly:DeprecationWarning", # We likely won't be updating to `marhsmallow` 4, these errors should be filtered out "error::marshmallow.warnings.RemovedInMarshmallow4Warning", # pkg_resources is deprecated as an API, but third party libraries still use it 'ignore: Deprecated call to `pkg_resources.declare_namespace\(.*\)`', # --------------------------------------- Great Expectations Warnings ---------------------------------- # This warning is for configuring the result_format parameter at the Validator-level, which will not be persisted, # but is still useful for building the configuration. "ignore:`result_format` configured at the Validator-level will not be persisted:UserWarning", # This warning is for configuring the result_format parameter at the Expectation-level, which will not be persisted, # but is still useful for building the configuration. "ignore:`result_format` configured at the Expectation-level will not be persisted:UserWarning", # This warning can be emitted when configuring splitters with fluent datasources "ignore:The same option name is applied for your batch regex and splitter config:UserWarning", # --------------------------------------- Great Expectations Warnings ---------------------------------- # --------------------------------------- Great Expectations Deprecation Warnings ---------------------------------- # Ignores in this section are for items in Great Expectations that are deprecated but not yet removed. Once the # relevant code is removed, the warning ignore should also be removed. # By ignoring these warnings, we will be able to turn on "warnings are errors" in our pipelines. # Example Actual Warning: UserWarning: Your query appears to have hard-coded references to your data. By not parameterizing your query with `{batch}`, {col}, etc., you may not be validating against your intended data asset, or the expectation may fail. 'ignore: Your query appears to have hard-coded references to your data. By not parameterizing your query with `{batch}`, {col}, etc., you may not be validating against your intended data asset, or the expectation may fail.:UserWarning', # Example Actual Warning: UserWarning: Your query appears to not be parameterized for a data asset. By not parameterizing your query with `{batch}`, you may not be validating against your intended data asset, or the expectation may fail. 'ignore: Your query appears to not be parameterized for a data asset. By not parameterizing your query with `{batch}`, you may not be validating against your intended data asset, or the expectation may fail.:UserWarning', # Example Actual Warning: (found in test_expect_queried_column_value_frequency_to_meet_threshold_override_query_sqlite) # UserWarning: `row_condition` is an experimental feature. Combining this functionality with QueryExpectations may result in unexpected behavior. 'ignore: `row_condition` is an experimental feature. Combining this functionality with QueryExpectations may result in unexpected behavior.:UserWarning', # Example Actual Warning: (found in test_golden_path_sql_datasource_configuration) # DeprecationWarning: get_batch is deprecated for the V3 Batch Request API as of v0.13.20 and will be removed in v0.16. Please use get_batch_list instead. 'ignore: get_batch is deprecated for the V3 Batch Request API as of v0.13.20 and will be removed in v0.16.:DeprecationWarning', # Example Actual Warning: (found in test_data_context) # UserWarning: Warning. An existing `great_expectations.yml` was found here 'ignore: Warning. An existing `great_expectations.yml` was found here:UserWarning', # Example Actual Warning: (found in test_data_context) # UserWarning: Warning. An existing `config_variables.yml` was found here 'ignore: Warning. An existing `config_variables.yml` was found here:UserWarning', # --------------------------------------- Great Expectations Deprecation Warnings ---------------------------------- # --------------------------------------- TEMPORARY IGNORES -------------------------------------------------------- # The warnings in this section should be addressed (fixed or ignored) but are ignored here temporarily to help allow # turning on "warnings are errors" so new warnings become errors and are addressed during PRs. 'ignore: The distutils package is deprecated and slated for removal in Python 3.12. Use setuptools or check PEP 632 for potential alternatives:DeprecationWarning', # python # Example Actual Warning: RuntimeWarning: divide by zero encountered in divide, RuntimeWarning: divide by zero encountered in true_divide # Found in test_atomic_prescriptive_summary_expect_column_kl_divergence_to_be_less_than, test_case_runner_v2_api 'ignore: divide by zero encountered:RuntimeWarning', # Example Actual Warning: Found running tests/test_definitions/test_expectations_v3_api.py # DeprecationWarning: NotImplemented should not be used in a boolean context 'ignore: NotImplemented should not be used in a boolean context:DeprecationWarning', # python 3.10 # Example Actual Warning: Found in tests/datasource/fluent/data_asset/data_connector/test_s3_data_connector.py # DeprecationWarning: ssl.PROTOCOL_TLS is deprecated 'ignore: ssl.PROTOCOL_TLS is deprecated:DeprecationWarning', # python 3.11 # data_connector/util.py imports deprecated modules that will be removed in Python 3.12 "ignore: module 'sre_constants' is deprecated:DeprecationWarning", "ignore: module 'sre_parse' is deprecated:DeprecationWarning", # trino # example actual warning: found in great_expectations/self_check/util.py:2752: in _create_trino_engine # sqlalchemy.exc.SADeprecationWarning: The dbapi() classmethod on dialect classes has been renamed to import_dbapi(). Implement an import_dbapi() classmethod directly on class to remove this warning; the old .dbapi() classmethod may be maintained for backwards compatibility. 'ignore: The dbapi\(\) classmethod on dialect classes has been renamed to import_dbapi\(\):DeprecationWarning', # six # Example Actual Warning: Found in ImportError while loading conftest '/great_expectations/tests/conftest.py'. # ImportWarning: _SixMetaPathImporter.exec_module() not found; falling back to load_module() 'ignore: _SixMetaPathImporter.exec_module\(\) not found:ImportWarning', # distutils # Example Actual Warning: Found in tests/datasource/test_batch_generators.py, test spark python 3.10 # DeprecationWarning: distutils Version classes are deprecated. Use packaging.version instead. 'ignore: distutils Version classes are deprecated. Use packaging.version instead.:DeprecationWarning', # pandas # Example Actual Warning: FutureWarning: In a future version of pandas, parsing datetimes with mixed time zones will raise an error unless `utc=True`. Please specify `utc=True` to opt in to the new behaviour and silence this warning. To create a `Series` with mixed offsets and `object` dtype, please use `apply` and `datetime.datetime.strptime` # Found when running pytest tests/test_definitions/test_expectations_v3_api.py 'ignore: In a future version of pandas, parsing datetimes with mixed time zones:FutureWarning', # Example Actual Warning: FutureWarning: The behavior of `series[i:j]` with an integer-dtype index is deprecated. In a future version, this will be treated as *label-based* indexing, consistent with e.g. `series[i]` lookups. To retain the old behavior, use `series.iloc[i:j]`. To get the future behavior, use `series.loc[i:j]`. # Found when running pytest tests/test_definitions/test_expectations_v3_api.py 'ignore: The behavior of `series:FutureWarning', # Example Actual Warning: UserWarning: Unknown extension is not supported and will be removed # Found when running TestIO.test_read_excel 'ignore: Unknown extension is not supported and will be removed:UserWarning', # Example Actual Warning: DeprecationWarning: The default dtype for empty Series will be 'object' instead of 'float64' in a future version. Specify a dtype explicitly to silence this warning. # Found when running test_value_counts_metric_spark "ignore: The default dtype for empty Series will be 'object' instead of 'float64' in a future version.:DeprecationWarning", # Example Actual Warning: Found by running tests/expectations/metrics/test_core.py::test_value_counts_metric_spark # FutureWarning: The default dtype for empty Series will be 'object' instead of 'float64' in a future version. Specify a dtype explicitly to silence this warning. 'ignore: The default dtype for empty Series will be:FutureWarning', # Example Actual Warning: Found by running tests/render/test_column_section_renderer.py::test_ProfilingResultsColumnSectionRenderer_render_bar_chart_table with Pandas 2.0. The warning is emitted through an Altair v5 codepath. # FutureWarning: the convert_dtype parameter is deprecated and will be removed in a future version. Do ``ser.astype(object).apply()`` instead if you want ``convert_dtype=False``. # GH Issue: https://github.com/altair-viz/altair/issues/3181 'ignore: the convert_dtype parameter is deprecated and will be removed in a future version:FutureWarning', # Example Actual Warning: FutureWarning: The behavior of DataFrame concatenation with empty or all-NA entries is deprecated. In a future version, this will no longer exclude empty or all-NA columns when determining the result dtypes. To retain the old behavior, exclude the relevant entries before the concat operation. 'ignore: The behavior of DataFrame concatenation with empty or all-NA entries is deprecated. In a future version, this will no longer exclude empty or all-NA columns when determining the result dtypes. To retain the old behavior, exclude the relevant entries before the concat operation.', # Example Actual Warning: FutureWarning: Returning a DataFrame from Series.apply when the supplied function returns a Series is deprecated and will be removed in a future version. 'ignore: Returning a DataFrame from Series.apply when the supplied function returns a Series is deprecated and will be removed in a future version.', # Example Actual Warning: UserWarning: pandas only supports SQLAlchemy connectable (engine/connection) or database string URI or sqlite3 DBAPI2 connection. Other DBAPI2 objects are not tested. Please consider using SQLAlchemy. # Found when runing pytest tests/test_definitions/test_expectations_v3_api.py 'ignore: pandas only supports SQLAlchemy connectable \(engine/connection\) or database string URI or sqlite3 DBAPI2 connection:UserWarning', # Example Actual Warning: FutureWarning: In a future version of pandas, parsing datetimes with mixed time zones will raise an error unless `utc=True`. Please specify `utc=True` to opt in to the new behaviour and silence this warning. To create a `Series` with mixed offsets and `object` dtype, please use `apply` and `datetime.datetime.strptime` 'ignore: In a future version of pandas, parsing datetimes with mixed time zones will raise an error:FutureWarning', # Example Actual Warning: DeprecationWarning: Passing a BlockManager to PandasDataset is deprecated and will raise in a future version. Use public APIs instead. # This is a legacy pattern that will be removed from GX 'ignore: Passing a BlockManager to PandasDataset is deprecated and will raise in a future version. Use public APIs instead.:DeprecationWarning', # Example Actual Warning: DeprecationWarning: Passing a BlockManager to CustomPandasDataset is deprecated and will raise in a future version. Use public APIs instead. 'ignore: Passing a BlockManager to CustomPandasDataset is deprecated and will raise in a future version. Use public APIs instead.:DeprecationWarning', # numpy # Example Actual Warning: RuntimeWarning: Mean of empty slice. # Found when running test_case_runner_v3_api[spark/column_aggregate_expectations/expect_column_median_to_be_between:test_empty_column_should_be_false_no_observed_value_with_which_to_compare] 'ignore: Mean of empty slice.:RuntimeWarning', # Example Actual Warning: RuntimeWarning: invalid value encountered in double_scalars # Found when running test_case_runner_v3_api[spark/column_aggregate_expectations/expect_column_median_to_be_between:test_empty_column_should_be_false_no_observed_value_with_which_to_compare] 'ignore: invalid value encountered:RuntimeWarning', # spark # Example Actual Warning: FutureWarning: Deprecated in 3.0.0. Use SparkSession.builder.getOrCreate() instead. # Found when running test_case_runner_v2_api[SparkDFDataset/column_pair_map_expectations/expect_column_pair_values_to_be_in_set:basic_positive_test_without_nulls] 'ignore: Deprecated in 3.0.0. Use SparkSession.builder.getOrCreate\(\) instead.:FutureWarning', # Example Acutal Warning: FutureWarning: is_datetime64tz_dtype is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.DatetimeTZDtype)` instead. 'ignore: is_datetime64tz_dtype is deprecated and will be removed in a future version. Check `isinstance\(dtype, pd.DatetimeTZDtype\)` instead.', # Example Actual Warning: # ResourceWarning: unclosed "ignore: unclosed =10.0.1; extra == "pandas"' "ignore: You have an incompatible version of 'pyarrow' installed:UserWarning", # jupyter # Example Actual Warning: # DeprecationWarning: Jupyter is migrating its paths to use standard platformdirs # given by the platformdirs library. To remove this warning and # see the appropriate new directories, set the environment variable # `JUPYTER_PLATFORM_DIRS=1` and then run `jupyter --paths`. # The use of platformdirs will be the default in `jupyter_core` v6 'ignore: Jupyter is migrating its paths to use standard platformdirs:DeprecationWarning', # pytest # Example Actual Warning: # pytest.PytestCollectionWarning: cannot collect test class 'TestConnectionError' because it has a __init__ constructor (from: tests/datasource/fluent/test_pandas_azure_blob_storage_datasource.py) "ignore: cannot collect test class 'TestConnectionError' because it has a __init__ constructor:UserWarning", # Example Actual Warning: # pytest.PytestUnraisableExceptionWarning: Exception ignored in: "ignore: Exception ignored in:UserWarning", # jsonschema (altair dependency) # Example Actual Warning: # DeprecationWarning: jsonschema.RefResolver is deprecated as of v4.18.0, in favor of the # https://github.com/python-jsonschema/referencing library, # which provides more compliant referencing behavior as well as more flexible APIs for customization. # A future release will remove RefResolver. Please file a feature request (on referencing) if you are missing an API for the kind of customization you need. "ignore: jsonschema.RefResolver is deprecated as of v4.18.0:DeprecationWarning", # Example Actual Warning: # DeprecationWarning: Importing ErrorTree directly from the jsonschema package is deprecated and will become an ImportError. Import it from jsonschema.exceptions instead. "ignore: Importing ErrorTree directly from the jsonschema package is deprecated and will become an ImportError. Import it from jsonschema.exceptions instead.:DeprecationWarning", # sqlalchemy # Example Actual Warning: # sqlalchemy.exc.RemovedIn20Warning: Deprecated API features detected! These feature(s) are not compatible with SQLAlchemy 2.0. To prevent incompatible upgrades prior to updating applications, ensure requirements files are pinned to "sqlalchemy<2.0". Set environment variable SQLALCHEMY_WARN_20=1 to show all deprecation warnings. Set environment variable SQLALCHEMY_SILENCE_UBER_WARNING=1 to silence this message. (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) 'ignore: Deprecated API features detected! These feature\(s\) are not compatible with SQLAlchemy 2\.0\.', # snowflake # Warning is emitted when snowflake-sqlalchemy 1.6 is installed alongside sqlalchemy 1 "module: The GenericFunction 'flatten' is already registered and is going to be overridden:sqlalchemy.exc.SAWarning" # --------------------------------------- TEMPORARY IGNORES -------------------------------------------------------- ] junit_family = "xunit2" markers = [ "all_backends: mark tests that run against all execution engine backends", "athena: mark a test as AWS Athena-dependent.", "aws_creds: mark all tests that require aws credentials", "aws_deps: mark tests that need aws dependencies like boto", "big: mark tests that don't have external dependencies but aren't unit tests", "bigquery: mark test as bigquery-dependent.", "clickhouse: mark a test as Clickhouse-dependent.", "cloud: mark test as being relevant to Great Expectations Cloud.", "docs-basic: mark a test as a docs test, that does not require additional credentials.", "docs-creds-needed: mark a test as a docs test that needs additional cloud credentials (these will not run on public forks).", "databricks: mark test as requiring databricks.", "docs: mark a test as a docs test.", "docs-spark: temporarily mark a test as a docs test that depends on spark.", "integration: mark test as an integration test.", "external_sqldialect: mark test as requiring install of an external sql dialect.", "filesystem: mark tests using the filesystem as the storage backend.", "mssql: mark a test as mssql-dependent.", "mysql: mark a test as mysql-dependent.", "openpyxl: mark a test for openpyxl-dependent, which is for Excel files.", "performance: mark a test as a performance test for BigQuery. These aren't run in our PR or release pipeline", "postgresql: mark a test as postgresql-dependent.", "project: mark a test that verifies properties of the gx project", "pyarrow: mark a test as PyArrow-dependent.", "snowflake: mark a test as snowflake-dependent.", "sqlalchemy_version_compatibility: mark test as required for sqlalchemy version compatibility.", "sqlite: mark test requiring sqlite", "slow: mark tests taking longer than 1 second.", "spark: mark a test as Spark-dependent.", "spark_connect: mark a test as Spark Connect-dependent.", "trino: mark a test as trino-dependent.", "unit: mark a test as a unit test.", "v2_api: mark test as specific to the v2 api (e.g. pre Data Connectors).", ] testpaths = "tests" # use `pytest-mock` drop-in replacement for `unittest.mock` # https://pytest-mock.readthedocs.io/en/latest/configuration.html#use-standalone-mock-package mock_use_standalone_module = false # https://docs.pytest.org/en/7.1.x/how-to/logging.html#how-to-manage-logging log_level = "info" [tool.coverage.report] # https://coverage.readthedocs.io/en/7.5.1/config.html # Regexes for lines to exclude from consideration exclude_also = [ # type-checking imports don't exist at runtime "if TYPE_CHECKING:", # Don't complain if tests don't hit NotImplementedError: "raise NotImplementedError", # Don't complain if non-runnable code isn't run: "if __name__ == .__main__.:", # Don't complain about abstract methods, they aren't run: "@(abc\\.)?abstractmethod", ]