YAML versioning (#2209)

* Make YAML files get the same version as Haystack and throw warning at load in case of mismatch

* Update version of most YAMLs in the codebase (aesthethic chamge, only to avoid the warning).

* Remove quotes from version in tests

* Fix version in generate_json_schema.py

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

.github/utils/generate_json_schema.py

@@ -4,6 +4,7 @@
 from pathlib import Path
 from typing import Any, Dict, Optional, Set, Tuple
 
+from haystack import __version__
 import haystack.document_stores
 import haystack.nodes
 import pydantic.schema
@@ -15,8 +16,8 @@
 from pydantic.typing import is_callable_type
 from pydantic.utils import lenient_issubclass
 
-schema_version = "0.7"
-filename = f"haystack-pipeline.{schema_version}.schema.json"
+schema_version = __version__
+filename = f"haystack-pipeline-{schema_version}.schema.json"
 destination_path = Path(__file__).parent.parent.parent / "json-schemas" / filename
 
 

docs/_src/api/api/pipelines.md

@@ -55,7 +55,7 @@ Here's a sample configuration:
 
  ```python
  | {
- | "version": "0.9",
+ | "version": "1.0",
  | "components": [
  | { # define all the building-blocks for Pipeline
  | "name": "MyReader", # custom-name for the component; helpful for visualization & debugging
@@ -110,7 +110,7 @@ be passed.
 Here's a sample configuration:
 
  ```yaml
- | version: '0.9'
+ | version: '1.0'
  |
  | components: # define all the building-blocks for Pipeline
  | - name: MyReader # custom-name for the component; helpful for visualization & debugging
@@ -137,6 +137,9 @@ Here's a sample configuration:
  | inputs: [MyESRetriever]
  ```
 
+Note that, in case of a mismatch in version between Haystack and the YAML, a warning will be printed.
+If the pipeline loads correctly regardless, save again the pipeline using `Pipeline.save_to_yaml()` to remove the warning.
+
 **Arguments**:
 
 - `path`: path of the YAML file.
@@ -605,6 +608,10 @@ Here's a sample configuration:
  | inputs: [MyESRetriever]
  ```
 
+
+Note that, in case of a mismatch in version between Haystack and the YAML, a warning will be printed.
+If the pipeline loads correctly regardless, save again the pipeline using `RayPipeline.save_to_yaml()` to remove the warning.
+
 **Arguments**:
 
 - `path`: path of the YAML file.

haystack/pipelines/base.py

@@ -29,6 +29,7 @@
  ray = None # type: ignore
  serve = None # type: ignore
 
+from haystack import __version__
 from haystack.schema import EvaluationResult, MultiLabel, Document
 from haystack.nodes.base import BaseComponent
 from haystack.document_stores.base import BaseDocumentStore
@@ -81,7 +82,7 @@ def load_from_config(
 
  ```python
  | {
- | "version": "0.9",
+ | "version": "1.0",
  | "components": [
  | { # define all the building-blocks for Pipeline
  | "name": "MyReader", # custom-name for the component; helpful for visualization & debugging
@@ -146,7 +147,7 @@ def load_from_yaml(cls, path: Path, pipeline_name: Optional[str] = None, overwri
  Here's a sample configuration:
 
  ```yaml
- | version: '0.9'
+ | version: '1.0'
  |
  | components: # define all the building-blocks for Pipeline
  | - name: MyReader # custom-name for the component; helpful for visualization & debugging
@@ -173,6 +174,9 @@ def load_from_yaml(cls, path: Path, pipeline_name: Optional[str] = None, overwri
  | inputs: [MyESRetriever]
  ```
 
+ Note that, in case of a mismatch in version between Haystack and the YAML, a warning will be printed.
+ If the pipeline loads correctly regardless, save again the pipeline using `Pipeline.save_to_yaml()` to remove the warning.
+
  :param path: path of the YAML file.
  :param pipeline_name: if the YAML contains multiple pipelines, the pipeline_name to load must be set.
  :param overwrite_with_env_variables: Overwrite the YAML configuration with environment variables. For example,
@@ -182,6 +186,14 @@ def load_from_yaml(cls, path: Path, pipeline_name: Optional[str] = None, overwri
  """
 
  pipeline_config = cls._read_pipeline_config_from_yaml(path)
+ if pipeline_config["version"] != __version__:
+ logger.warning(
+ f"YAML version ({pipeline_config['version']}) does not match with Haystack version ({__version__}). "
+ "Issues may occur during loading. "
+ "To fix this warning, save again this pipeline with the current Haystack version using Pipeline.save_to_yaml(), "
+ "check out our migration guide at https://haystack.deepset.ai/overview/migration "
+ f"or downgrade to haystack version {__version__}."
+ )
  return cls.load_from_config(
  pipeline_config=pipeline_config,
  pipeline_name=pipeline_name,
@@ -1037,7 +1049,11 @@ def get_config(self, return_defaults: bool = False) -> dict:
  # create the Pipeline definition with how the Component are connected
  pipelines[pipeline_name]["nodes"].append({"name": node, "inputs": list(self.graph.predecessors(node))})
 
- config = {"components": list(components.values()), "pipelines": list(pipelines.values()), "version": "0.8"}
+ config = {
+ "components": list(components.values()),
+ "pipelines": list(pipelines.values()),
+ "version": __version__,
+ }
  return config
 
  def _format_document_answer(self, document_or_answer: dict):
@@ -1290,6 +1306,10 @@ def load_from_yaml(
  | inputs: [MyESRetriever]
  ```
 
+
+ Note that, in case of a mismatch in version between Haystack and the YAML, a warning will be printed.
+ If the pipeline loads correctly regardless, save again the pipeline using `RayPipeline.save_to_yaml()` to remove the warning.
+
  :param path: path of the YAML file.
  :param pipeline_name: if the YAML contains multiple pipelines, the pipeline_name to load must be set.
  :param overwrite_with_env_variables: Overwrite the YAML configuration with environment variables. For example,
@@ -1299,6 +1319,14 @@ def load_from_yaml(
  :param address: The IP address for the Ray cluster. If set to None, a local Ray instance is started.
  """
  pipeline_config = cls._read_pipeline_config_from_yaml(path)
+ if pipeline_config["version"] != __version__:
+ logger.warning(
+ f"YAML version ({pipeline_config['version']}) does not match with Haystack version ({__version__}). "
+ "Issues may occur during loading. "
+ "To fix this warning, save again this pipeline with the current Haystack version using Pipeline.save_to_yaml(), "
+ "check out our migration guide at https://haystack.deepset.ai/overview/migration "
+ f"or downgrade to haystack version {__version__}."
+ )
  return RayPipeline.load_from_config(
  pipeline_config=pipeline_config,
  pipeline_name=pipeline_name,