diff --git a/.buildkite/pipeline.build.yml b/.buildkite/pipeline.build.yml index 5d58c27011597..139ac7196a693 100644 --- a/.buildkite/pipeline.build.yml +++ b/.buildkite/pipeline.build.yml @@ -577,7 +577,7 @@ # Horovod needs to be installed separately (needed for API ref imports) - ./ci/env/install-horovod.sh # See https://stackoverflow.com/questions/63383400/error-cannot-uninstall-ruamel-yaml-while-creating-docker-image-for-azure-ml-a - - pip install mosaicml==0.10.1 --ignore-installed + - pip install mosaicml==0.12.1 --ignore-installed - ./ci/ci.sh build - label: ":octopus: Tune multinode tests" diff --git a/.buildkite/pipeline.ml.yml b/.buildkite/pipeline.ml.yml index 4cdd62ea2b606..a0cc925feab67 100644 --- a/.buildkite/pipeline.ml.yml +++ b/.buildkite/pipeline.ml.yml @@ -287,6 +287,7 @@ conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_PYTHON_AFFECTED", "RAY_CI_DATA_AFFECTED"] commands: - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT + - ./ci/env/install-java.sh - DATA_PROCESSING_TESTING=1 ARROW_VERSION=9.* ARROW_MONGO_VERSION=0.5.* ./ci/env/install-dependencies.sh - ./ci/env/env_info.sh - sudo apt-get purge -y mongodb* @@ -302,6 +303,7 @@ instance_size: medium commands: - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT + - ./ci/env/install-java.sh - DATA_PROCESSING_TESTING=1 ARROW_VERSION=7.* ./ci/env/install-dependencies.sh - ./ci/env/env_info.sh - bazel test --config=ci $(./ci/run/bazel_export_options) --action_env=RAY_DATASET_USE_STREAMING_EXECUTOR=1 --build_tests_only --test_tag_filters=-dataset_integration python/ray/data/... diff --git a/ci/env/install-dependencies.sh b/ci/env/install-dependencies.sh index 743279c42492a..6b99044207552 100755 --- a/ci/env/install-dependencies.sh +++ b/ci/env/install-dependencies.sh @@ -367,7 +367,6 @@ install_pip_packages() { # Additional Train test dependencies. if [ "${TRAIN_TESTING-}" = 1 ] || [ "${DOC_TESTING-}" = 1 ]; then - rm -rf "${SITE_PACKAGES}"/ruamel* # https://stackoverflow.com/questions/63383400/error-cannot-uninstall-ruamel-yaml-while-creating-docker-image-for-azure-ml-a pip install -U -c "${WORKSPACE_DIR}"/python/requirements.txt -r "${WORKSPACE_DIR}"/python/requirements/ml/requirements_train.txt fi diff --git a/cpp/BUILD.bazel b/cpp/BUILD.bazel index 52db9f78bc4dc..d09b9a3f0e240 100644 --- a/cpp/BUILD.bazel +++ b/cpp/BUILD.bazel @@ -21,13 +21,13 @@ cc_binary( linkstatic = 1, visibility = ["//visibility:public"], deps = [ - ":ray_api", + ":ray_api_lib", ":symbols/ray_api_exported_symbols_linux.lds", ], ) cc_library( - name = "ray_api", + name = "ray_api_lib", srcs = glob([ "src/ray/api.cc", "src/ray/api/*.cc", @@ -95,7 +95,7 @@ cc_binary( "@bazel_tools//src/conditions:windows": [ # TODO(SongGuyang): Change to use dynamic library # "ray_cpp_lib" when we make it work on Windows. - "ray_api", + "ray_api_lib", ], "//conditions:default": [ "ray_cpp_lib", @@ -111,7 +111,6 @@ genrule( name = "ray_cpp_pkg", srcs = [ "default_worker", - "ray_api", "libray_api.so", ], outs = ["ray_cpp_pkg.out"], @@ -175,7 +174,7 @@ cc_test( linkstatic = True, tags = ["team:core"], deps = [ - "ray_api", + "ray_api_lib", "@com_google_googletest//:gtest_main", ], ) @@ -205,7 +204,7 @@ cc_test( linkstatic = True, tags = ["team:core"], deps = [ - "ray_api", + "ray_api_lib", "@com_google_googletest//:gtest_main", ], ) @@ -228,7 +227,7 @@ cc_test( linkstatic = True, tags = ["team:core"], deps = [ - "ray_api", + "ray_api_lib", "@com_google_googletest//:gtest_main", ], ) @@ -293,7 +292,7 @@ cc_test( linkstatic = True, tags = ["team:core"], deps = [ - "ray_api", + "ray_api_lib", ], ) @@ -327,7 +326,7 @@ cc_binary( linkstatic = True, tags = ["team:core"], deps = [ - ":ray_api", + ":ray_api_lib", ], ) diff --git a/dashboard/agent.py b/dashboard/agent.py index 94c12a4b10110..f24b74582dfd4 100644 --- a/dashboard/agent.py +++ b/dashboard/agent.py @@ -243,7 +243,7 @@ async def _check_parent(): error = True except Exception as e: msg += f"Failed to read Raylet logs at {log_path}: {e}!" - logger.exception() + logger.exception(msg) error = True if error: logger.error(msg) diff --git a/dashboard/state_aggregator.py b/dashboard/state_aggregator.py index c7fa98ce4f10a..56a633b8c01f5 100644 --- a/dashboard/state_aggregator.py +++ b/dashboard/state_aggregator.py @@ -421,7 +421,7 @@ def _to_task_state(task_attempt: dict) -> dict: ], ), (task_attempt, ["task_id", "attempt_number", "job_id"]), - (state_updates, ["node_id", "worker_id"]), + (state_updates, ["node_id", "worker_id", "error_type"]), ] for src, keys in mappings: for key in keys: diff --git a/doc/BUILD b/doc/BUILD index 8417eb6c0e010..d4527dd6bf173 100644 --- a/doc/BUILD +++ b/doc/BUILD @@ -27,6 +27,13 @@ py_test( tags = ["exclusive", "team:ml"] ) +py_test( + name = "tensor", + size = "small", + srcs = ["source/data/doc_code/tensor.py"], + tags = ["exclusive", "team:ml"] +) + py_test( name = "big_data_ingestion", size = "small", diff --git a/doc/source/_toc.yml b/doc/source/_toc.yml index 7889f9cd676a5..fd9dbde9a9825 100644 --- a/doc/source/_toc.yml +++ b/doc/source/_toc.yml @@ -54,6 +54,7 @@ parts: - file: ray-air/examples/index sections: - file: ray-air/examples/torch_image_example + - file: ray-air/examples/torch_detection - file: ray-air/examples/convert_existing_pytorch_code_to_ray_air - file: ray-air/examples/convert_existing_tf_code_to_ray_air - file: ray-air/examples/tfx_tabular_train_to_serve @@ -74,7 +75,10 @@ parts: - file: ray-air/examples/batch_forecasting - file: ray-air/examples/pytorch_resnet_batch_prediction - file: ray-air/examples/stablediffusion_batch_prediction + - file: ray-air/examples/gptj_deepspeed_fine_tuning - file: ray-air/examples/gptj_batch_prediction + - file: ray-air/examples/gptj_serving + - file: ray-air/examples/dreambooth_finetuning - file: ray-air/api/api - file: ray-air/benchmarks diff --git a/doc/source/cluster/kubernetes/configs/static-ray-cluster.with-fault-tolerance.yaml b/doc/source/cluster/kubernetes/configs/static-ray-cluster.with-fault-tolerance.yaml index 7619cb0ed250f..3a941bf5f19d5 100644 --- a/doc/source/cluster/kubernetes/configs/static-ray-cluster.with-fault-tolerance.yaml +++ b/doc/source/cluster/kubernetes/configs/static-ray-cluster.with-fault-tolerance.yaml @@ -73,6 +73,7 @@ metadata: labels: app: ray-cluster-head spec: + clusterIP: None ports: - name: client protocol: TCP @@ -198,7 +199,7 @@ spec: imagePullPolicy: Always command: ["/bin/bash", "-c", "--"] args: - - "ray start --num-cpus=$MY_CPU_REQUEST --address=$SERVICE_RAY_CLUSTER_SERVICE_HOST:$SERVICE_RAY_CLUSTER_SERVICE_PORT_GCS_SERVER --object-manager-port=8076 --node-manager-port=8077 --dashboard-agent-grpc-port=8078 --dashboard-agent-listen-port=52365 --block" + - "ray start --num-cpus=$MY_CPU_REQUEST --address=service-ray-cluster:6380 --object-manager-port=8076 --node-manager-port=8077 --dashboard-agent-grpc-port=8078 --dashboard-agent-listen-port=52365 --block" # This volume allocates shared memory for Ray to use for its plasma # object store. If you do not provide this, Ray will fall back to diff --git a/doc/source/data/creating-datasets.rst b/doc/source/data/creating-datasets.rst index 2d1a8a702412c..199e3ea79b0e5 100644 --- a/doc/source/data/creating-datasets.rst +++ b/doc/source/data/creating-datasets.rst @@ -77,9 +77,8 @@ Supported File Formats Read Parquet files into a tabular ``Dataset``. The Parquet data will be read into `Arrow Table `__ blocks. Although this simple example demonstrates reading a single file, note that - Datasets can also read directories of Parquet files, with one tabular block created - per file. For Parquet in particular, we also support reading partitioned Parquet - datasets with partition column values pulled from the file paths. + Datasets can also read directories of Parquet files. We also support reading partitioned + Parquet datasets with partition column values pulled from the file paths. .. literalinclude:: ./doc_code/creating_datasets.py :language: python diff --git a/doc/source/data/dask-on-ray.rst b/doc/source/data/dask-on-ray.rst index d13c2bdb7d7a3..587175c68abf6 100644 --- a/doc/source/data/dask-on-ray.rst +++ b/doc/source/data/dask-on-ray.rst @@ -31,8 +31,10 @@ workload. Using the Dask-on-Ray scheduler, the entire Dask ecosystem can be exec * - Ray Version - Dask Version + * - ``2.3.0`` + - ``2022.10.1`` * - ``2.2.0`` - - ``2022.2.0`` + - ``2022.10.1`` * - ``2.1.0`` - ``2022.2.0`` * - ``2.0.0`` diff --git a/doc/source/data/doc_code/tensor.py b/doc/source/data/doc_code/tensor.py index 6b41c6bfbd358..fc2945ba836e3 100644 --- a/doc/source/data/doc_code/tensor.py +++ b/doc/source/data/doc_code/tensor.py @@ -489,17 +489,21 @@ def add_one(batch: Dict[str, Any]) -> Dict[str, Any]: # fmt: off # __create_variable_shaped_tensors_begin___ # Create a Dataset of variable-shaped tensors. -arr = np.array([np.ones((2, 2)), np.ones((3, 3))], dtype=object) -ds = ray.data.from_numpy([arr, arr]) +ragged_array = np.array([np.ones((2, 2)), np.ones((3, 3))], dtype=object) +df = pd.DataFrame({"feature": ragged_array, "label": [1, 1]}) +ds = ray.data.from_pandas([df, df]) # -> Dataset(num_blocks=2, num_rows=4, -# schema={__value__: ArrowVariableShapedTensorType(dtype=double)}) +# schema={feature: TensorDtype(shape=(None, None), dtype=float64), +# label: int64}) ds.take(2) -# -> [array([[1., 1.], -# [1., 1.]]), -# array([[1., 1., 1.], -# [1., 1., 1.], -# [1., 1., 1.]])] +# -> [{'feature': array([[1., 1.], +# [1., 1.]]), +# 'label': 1}, +# {'feature': array([[1., 1., 1.], +# [1., 1., 1.], +# [1., 1., 1.]]), +# 'label': 1}] # __create_variable_shaped_tensors_end__ # fmt: off @@ -507,13 +511,16 @@ def add_one(batch: Dict[str, Any]) -> Dict[str, Any]: # Convert Ray Dataset to a TensorFlow Dataset. tf_ds = ds.to_tf( batch_size=2, - output_signature=tf.RaggedTensorSpec(shape=(None, None, None), dtype=tf.float64), + feature_columns="feature", + label_columns="label" ) # Iterate through the tf.RaggedTensors. for ragged_tensor in tf_ds: print(ragged_tensor) -# -> -# +# -> (, +# ) +# (, +# ) # __tf_variable_shaped_tensors_end__ diff --git a/doc/source/data/getting-started.rst b/doc/source/data/getting-started.rst index 6ff45965180be..20839ed0207c8 100644 --- a/doc/source/data/getting-started.rst +++ b/doc/source/data/getting-started.rst @@ -62,7 +62,17 @@ transform datasets. Ray executes transformations in parallel for performance at .. testoutput:: MapBatches(transform_batch) - +- Dataset(num_blocks=1, num_rows=150, schema={sepal length (cm): double, sepal width (cm): double, petal length (cm): double, petal width (cm): double, target: int64}) + +- Dataset( + num_blocks=..., + num_rows=150, + schema={ + sepal length (cm): double, + sepal width (cm): double, + petal length (cm): double, + petal width (cm): double, + target: int64 + } + ) To learn more about transforming datasets, read :ref:`Transforming datasets `. diff --git a/doc/source/data/glossary.rst b/doc/source/data/glossary.rst index 067491ec9cd89..e1de9b0596605 100644 --- a/doc/source/data/glossary.rst +++ b/doc/source/data/glossary.rst @@ -116,7 +116,11 @@ Ray Datasets Glossary >>> import numpy as np >>> import ray >>> ray.data.from_numpy(np.zeros((100, 32, 32, 3))) - Dataset(num_blocks=1, num_rows=100, schema={__value__: ArrowTensorType(shape=(32, 32, 3), dtype=double)}) + Dataset( + num_blocks=1, + num_rows=100, + schema={__value__: ArrowTensorType(shape=(32, 32, 3), dtype=double)} + ) Tabular Dataset A Dataset that represents columnar data. @@ -125,7 +129,17 @@ Ray Datasets Glossary >>> import ray >>> ray.data.read_csv("s3://anonymous@air-example-data/iris.csv") - Dataset(num_blocks=1, num_rows=150, schema={sepal length (cm): double, sepal width (cm): double, petal length (cm): double, petal width (cm): double, target: int64}) + Dataset( + num_blocks=1, + num_rows=150, + schema={ + sepal length (cm): double, + sepal width (cm): double, + petal length (cm): double, + petal width (cm): double, + target: int64 + } + ) User-defined function (UDF) A callable that transforms batches or :term:`records ` of data. UDFs let you arbitrarily transform datasets. diff --git a/doc/source/images/detection.jpeg b/doc/source/images/detection.jpeg new file mode 100644 index 0000000000000..1692c91956d8b Binary files /dev/null and b/doc/source/images/detection.jpeg differ diff --git a/doc/source/ray-air/examples/BUILD b/doc/source/ray-air/examples/BUILD index 7e7cba21877fa..ff2f4930cdb59 100644 --- a/doc/source/ray-air/examples/BUILD +++ b/doc/source/ray-air/examples/BUILD @@ -46,8 +46,11 @@ py_test_run_all_notebooks( "feast_example.ipynb", # REGRESSION "upload_to_comet_ml.ipynb", # Needs credentials "upload_to_wandb.ipynb", # Needs credentials + "torch_detection.ipynb", # Requires GPUs "gptj_batch_prediction.ipynb", # Requires GPUs + "gptj_serving.ipynb", # Requires GPUs "stablediffusion_batch_prediction.ipynb", # Requires GPUs + "gptj_deepspeed_fine_tuning.ipynb", # Requires release test ], data = ["//doc/source/ray-air/examples:air_examples"], tags = ["exclusive", "team:ml", "ray_air"], @@ -79,7 +82,9 @@ py_test_run_all_notebooks( include = [ "huggingface_text_classification.ipynb", "pytorch_resnet_batch_prediction.ipynb", + "torch_detection.ipynb", "gptj_batch_prediction.ipynb", + "gptj_serving.ipynb", "stablediffusion_batch_prediction.ipynb", ], exclude = [], diff --git a/doc/source/ray-air/examples/dreambooth_finetuning.rst b/doc/source/ray-air/examples/dreambooth_finetuning.rst new file mode 100644 index 0000000000000..4c8b58b49264b --- /dev/null +++ b/doc/source/ray-air/examples/dreambooth_finetuning.rst @@ -0,0 +1,183 @@ +Fine-tuning DreamBooth with Ray AIR +=================================== + +.. include:: ../../../../python/ray/air/examples/dreambooth/README.rst + :start-after: section_intro + :end-before: How it works + +How it works +------------ + +This example leverages Ray Data for data loading and Ray Train for distributed training. + +Data loading +^^^^^^^^^^^^ + +.. note:: + You can find the latest version of the code here: `dataset.py `_ + + The latest version might differ slightly from the code presented here. + + +We use Ray Data for data loading. The code has three interesting parts. + +First, we load two datasets using :func:`ray.data.read_images`: + +.. literalinclude:: ../../../../python/ray/air/examples/dreambooth/dataset.py + :language: python + :start-at: instance_dataset = read + :end-at: class_dataset = read + :dedent: 4 + +Then, we tokenize the prompt that generated these images: + +.. literalinclude:: ../../../../python/ray/air/examples/dreambooth/dataset.py + :language: python + :start-at: tokenizer = AutoTokenizer + :end-at: instance_prompt_ids = _tokenize + :dedent: 4 + + +And lastly, we apply a ``torchvision`` preprocessing pipeline to the images: + +.. literalinclude:: ../../../../python/ray/air/examples/dreambooth/dataset.py + :language: python + :start-at: transform = transforms.Compose + :end-at: preprocessor = TorchVisionPreprocessor + :dedent: 4 + +We apply all of this in final step: + + +.. literalinclude:: ../../../../python/ray/air/examples/dreambooth/dataset.py + :language: python + :start-at: instance_dataset = preprocessor + :end-before: --- + :dedent: 4 + + + +Distributed training +^^^^^^^^^^^^^^^^^^^^ + + +.. note:: + You can find the latest version of the code here: `train.py `_ + + The latest version might differ slightly from the code presented here. + + +The central part of the training code is the *training function*. This function accepts a configuration dict that contains the hyperparameters. It then defines a regular PyTorch training loop. + +There are only a few locations where we interact with the Ray AIR API. We marked them with in-line comments in the snippet below. + +Remember that we want to do data-parallel training for all our models. + + +#. We load the data shard for each worker with session.get_dataset_shard("train") +#. We iterate over the dataset with train_dataset.iter_torch_batches() +#. We report results to Ray AIR with session.report(results) + +The code was compacted for brevity. The `full code `_ is more thoroughly annotated. + + +.. literalinclude:: ../../../../python/ray/air/examples/dreambooth/train.py + :language: python + :start-at: def train_fn(config) + :end-at: session.report(results) + +We can then run this training loop with Ray AIR's TorchTrainer: + + +.. literalinclude:: ../../../../python/ray/air/examples/dreambooth/train.py + :language: python + :start-at: args = train_arguments + :end-at: trainer.fit() + :dedent: 4 + +Configuring the scale +^^^^^^^^^^^^^^^^^^^^^ + +In the TorchTrainer, we can easily configure our scale. +The above example uses the ``num_workers`` argument to specify the number +of workers. This defaults to 2 workers with 2 GPUs each - so 4 GPUs in total. + +To run the example on 8 GPUs, just set the number of workers to 4 using ``--num-workers=4``! +Or you can change the scaling config directly: + +.. code-block:: diff + + scaling_config=ScalingConfig( + use_gpu=True, + - num_workers=args.num_workers, + + num_workers=4, + resources_per_worker={ + "GPU": 2, + }, + ) + +If you're running multi-node training, you should make sure that all nodes have access to a shared +storage (e.g. via NFS or EFS). In the example script below, you can adjust this location with the +``DATA_PREFIX`` environment variable. + +Training throughput +~~~~~~~~~~~~~~~~~~~ + +We ran training using 1, 2, 4, and 8 workers (and 2, 4, 8, and 16 GPUs, respectively) to compare throughput. + +Setup: + +* 2 x g5.12xlarge nodes with 4 A10G GPUs each +* Model as configured below +* Data from this example +* 200 regularization images +* Training for 4 epochs (800 steps) +* Use a mounted External File System to share data between nodes +* 3 runs per configuration + +Because network storage can be slow, we excluded the time it takes to save the final model from the training time. + +We expect that the training time should benefit from scale and decreases when running with +more workers and GPUs. + + +.. image:: images/dreambooth_training.png + :target: images/dreambooth_training.png + :alt: DreamBooth training times + + +.. list-table:: + :header-rows: 1 + + * - Number of workers + - Number of GPUs + - Training time + * - 1 + - 2 + - 458.16 (3.82) + * - 2 + - 4 + - 364.61 (1.65) + * - 4 + - 8 + - 252.37 (3.18) + * - 8 + - 16 + - 160.97 (1.36) + + +While the training time decreases linearly with the amount of workers/GPUs, we observe some penalty. +Specifically, with double the amount of workers we don't get half of the training time. + +This is most likely due to additional communication between processes and the transfer of large model +weights. We are also only training with a batch size of one because our GPU memory is limited. On larger +GPUs with higher batch sizes we would expect a greater benefit from scaling out. + + +Run the example +--------------- + +.. include:: ../../../../python/ray/air/examples/dreambooth/README.rst + :start-after: section_run_example + + diff --git a/doc/source/ray-air/examples/gptj_batch_prediction.ipynb b/doc/source/ray-air/examples/gptj_batch_prediction.ipynb index 9a9cf77b19b88..9ce17b1db9c46 100644 --- a/doc/source/ray-air/examples/gptj_batch_prediction.ipynb +++ b/doc/source/ray-air/examples/gptj_batch_prediction.ipynb @@ -5,14 +5,16 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# GPT-J6-B Batch Prediction with Ray AIR\n", + "# GPT-J-6B Batch Prediction with Ray AIR\n", "\n", - "In this example, we will showcase how to use the Ray AIR for **GPT-J batch inference**. GPT-J is a GPT-2-like causal language model trained on the Pile dataset. This particular model has 6 billion parameters.\n", + "This example showcases how to use the Ray AIR for **GPT-J batch inference**. GPT-J is a GPT-2-like causal language model trained on the Pile dataset. This model has 6 billion parameters. For more information on GPT-J, click [here](https://huggingface.co/docs/transformers/model_doc/gptj).\n", "\n", - "We will use Ray Data to carry out this task and a pretrained model from Hugging Face hub. Note that you can easily adapt this example to use other similar models.\n", + "We use Ray Data and a pretrained model from Hugging Face hub. Note that you can easily adapt this example to use other similar models.\n", "\n", "It is highly recommended to read [Ray AIR Key Concepts](air-key-concepts) and [Ray Data Key Concepts](data_key_concepts) before starting this example.\n", "\n", + "If you are interested in serving (online inference), see {doc}`/ray-air/examples/gptj_serving`.\n", + "\n", "```{note}\n", "In order to run this example, make sure your Ray cluster has access to at least one GPU with 16 or more GBs of memory. The amount of memory needed will depend on the model.\n", "```" diff --git a/doc/source/ray-air/examples/gptj_deepspeed_fine_tuning.ipynb b/doc/source/ray-air/examples/gptj_deepspeed_fine_tuning.ipynb new file mode 100644 index 0000000000000..8002a71dc9c8e --- /dev/null +++ b/doc/source/ray-air/examples/gptj_deepspeed_fine_tuning.ipynb @@ -0,0 +1,1148 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# GPT-J-6B Fine-Tuning with Ray AIR and DeepSpeed\n", + "\n", + "In this example, we will showcase how to use the Ray AIR for **GPT-J fine-tuning**. GPT-J is a GPT-2-like causal language model trained on the Pile dataset. This particular model has 6 billion parameters. For more information on GPT-J, click [here](https://huggingface.co/docs/transformers/model_doc/gptj).\n", + "\n", + "We will use Ray AIR (with the 🤗 Transformers integration) and a pretrained model from Hugging Face hub. Note that you can easily adapt this example to use other similar models.\n", + "\n", + "This example focuses more on the performance and distributed computing aspects of Ray AIR. If you are looking for a more beginner friendly introduction to Ray AIR 🤗 Transformers integration, see {doc}`this example `.\n", + "\n", + "It is highly recommended to read [Ray AIR Key Concepts](air-key-concepts) and [Ray Data Key Concepts](data_key_concepts) before starting this example.\n", + "\n", + "```{note}\n", + "In order to run this example, make sure your Ray cluster has access to at least one GPU with 16 or more GBs of memory. The amount of memory needed will depend on the model. This notebook is being tested with 16 g4dn.4xlarge instances.\n", + "```\n", + "\n", + "In this notebook, we will:\n", + "1. [Set up Ray](#setup)\n", + "2. [Load the dataset](#load)\n", + "3. [Preprocess the dataset with Ray AIR](#preprocess)\n", + "4. [Run the training with Ray AIR](#train)\n", + "5. [Generate text from prompt with Ray AIR](#predict)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Uncomment and run the following line in order to install all the necessary dependencies (this notebook is being tested with `transformers==4.26.0`):" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "#! pip install \"datasets\" \"evaluate\" \"accelerate>=0.16.0\" \"transformers>=4.26.0\" \"torch>=1.12.0\" \"deepspeed\"" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import os" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Set up Ray \n", + "\n", + "First, let's set some global variables. We will use 16 workers, each being assigned 1 GPU and 8 CPUs." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "model_name = \"EleutherAI/gpt-j-6B\"\n", + "use_gpu = True\n", + "num_workers = 16\n", + "cpus_per_worker = 8" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will use `ray.init()` to initialize a local cluster. By default, this cluster will be comprised of only the machine you are running this notebook on. You can also run this notebook on an Anyscale cluster.\n", + "\n", + "We define a {ref}`runtime environment ` to ensure that the Ray workers have access to all the necessary packages. You can omit the `runtime_env` argument if you have all of the packages already installed on each node in your cluster." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "
\n", + "

Ray

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + "\n", + "
Python version:3.8.16
Ray version: 3.0.0.dev0
Dashboard:http://console.anyscale-staging.com/api/v2/sessions/ses_sedlspnpy16naa5lm9kf2cmi2y/services?redirect_to=dashboard
\n", + "
\n", + "
\n" + ], + "text/plain": [ + "RayContext(dashboard_url='console.anyscale-staging.com/api/v2/sessions/ses_sedlspnpy16naa5lm9kf2cmi2y/services?redirect_to=dashboard', python_version='3.8.16', ray_version='3.0.0.dev0', ray_commit='4ddbbb3c4b19c2d27bbf54f8c5ffc100dceafbcf', address_info={'node_ip_address': '10.0.30.196', 'raylet_ip_address': '10.0.30.196', 'redis_address': None, 'object_store_address': '/tmp/ray/session_2023-03-06_15-55-37_997701_162/sockets/plasma_store', 'raylet_socket_name': '/tmp/ray/session_2023-03-06_15-55-37_997701_162/sockets/raylet', 'webui_url': 'console.anyscale-staging.com/api/v2/sessions/ses_sedlspnpy16naa5lm9kf2cmi2y/services?redirect_to=dashboard', 'session_dir': '/tmp/ray/session_2023-03-06_15-55-37_997701_162', 'metrics_export_port': 8085, 'gcs_address': '10.0.30.196:6379', 'address': '10.0.30.196:6379', 'dashboard_agent_listen_port': 52365, 'node_id': '77de483c435bf4987fd6f1e91d47602554e876fd41230d8d50c05333'})" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ray\n", + "\n", + "ray.init(\n", + " runtime_env={\n", + " \"pip\": [\n", + " \"datasets\",\n", + " \"evaluate\",\n", + " \"accelerate>=0.16.0\",\n", + " \"transformers>=4.26.0\",\n", + " \"torch>=1.12.0\",\n", + " \"deepspeed\",\n", + " ]\n", + " }\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "tags": [ + "hide-cell" + ] + }, + "outputs": [], + "source": [ + "# THIS SHOULD BE HIDDEN IN DOCS AND ONLY RAN IN CI\n", + "# Download the model from our S3 mirror as it's faster\n", + "\n", + "import ray\n", + "import subprocess\n", + "import ray.util.scheduling_strategies\n", + "\n", + "\n", + "def force_on_node(node_id: str, remote_func_or_actor_class):\n", + " scheduling_strategy = ray.util.scheduling_strategies.NodeAffinitySchedulingStrategy(\n", + " node_id=node_id, soft=False\n", + " )\n", + " options = {\"scheduling_strategy\": scheduling_strategy}\n", + " return remote_func_or_actor_class.options(**options)\n", + "\n", + "\n", + "def run_on_every_node(remote_func_or_actor_class, **remote_kwargs):\n", + " refs = []\n", + " for node in ray.nodes():\n", + " if node[\"Alive\"] and node[\"Resources\"].get(\"GPU\", None):\n", + " refs.append(\n", + " force_on_node(node[\"NodeID\"], remote_func_or_actor_class).remote(\n", + " **remote_kwargs\n", + " )\n", + " )\n", + " return ray.get(refs)\n", + "\n", + "\n", + "@ray.remote(num_gpus=1)\n", + "def download_model():\n", + " from transformers.utils.hub import TRANSFORMERS_CACHE\n", + "\n", + " path = os.path.expanduser(\n", + " os.path.join(TRANSFORMERS_CACHE, \"models--EleutherAI--gpt-j-6B\")\n", + " )\n", + " subprocess.run([\"mkdir\", \"-p\", os.path.join(path, \"snapshots\", \"main\")])\n", + " subprocess.run([\"mkdir\", \"-p\", os.path.join(path, \"refs\")])\n", + " if os.path.exists(os.path.join(path, \"refs\", \"main\")):\n", + " return\n", + " subprocess.run(\n", + " [\n", + " \"aws\",\n", + " \"s3\",\n", + " \"sync\",\n", + " \"--quiet\",\n", + " \"s3://large-dl-models-mirror/models--EleutherAI--gpt-j-6B/main/\",\n", + " os.path.join(path, \"snapshots\", \"main\"),\n", + " ]\n", + " )\n", + " with open(os.path.join(path, \"snapshots\", \"main\", \"hash\"), \"r\") as f:\n", + " f_hash = f.read().strip()\n", + " with open(os.path.join(path, \"refs\", \"main\"), \"w\") as f:\n", + " f.write(f_hash)\n", + " os.rename(\n", + " os.path.join(path, \"snapshots\", \"main\"), os.path.join(path, \"snapshots\", f_hash)\n", + " )\n", + "\n", + "\n", + "_ = run_on_every_node(download_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Loading the dataset \n", + "\n", + "We will be fine-tuning the model on the [`tiny_shakespeare` dataset](https://huggingface.co/datasets/tiny_shakespeare), comprised of 40,000 lines of Shakespeare from a variety of Shakespeare's plays. The aim will be to make the GPT-J model better at generating text in the style of Shakespeare." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loading tiny_shakespeare dataset\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Found cached dataset tiny_shakespeare (/home/ray/.cache/huggingface/datasets/tiny_shakespeare/default/1.0.0/b5b13969f09fe8707337f6cb296314fbe06960bd9a868dca39e713e163d27b5e)\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "65894225f3b84e5caa117c4d08d9f99d", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/3 [00:00 pd.DataFrame:\n", + " text = list(batch[\"text\"])\n", + " flat_text = \"\".join(text)\n", + " split_text = [\n", + " x.strip()\n", + " for x in flat_text.split(\"\\n\")\n", + " if x.strip() and not x.strip()[-1] == \":\"\n", + " ]\n", + " return pd.DataFrame(split_text, columns=[\"text\"])\n", + "\n", + "\n", + "def tokenize(batch: pd.DataFrame) -> dict:\n", + " tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False)\n", + " tokenizer.pad_token = tokenizer.eos_token\n", + " ret = tokenizer(\n", + " list(batch[\"text\"]),\n", + " truncation=True,\n", + " max_length=block_size,\n", + " padding=\"max_length\",\n", + " return_tensors=\"np\",\n", + " )\n", + " ret[\"labels\"] = ret[\"input_ids\"].copy()\n", + " return dict(ret)\n", + "\n", + "\n", + "splitter = BatchMapper(split_text, batch_format=\"pandas\")\n", + "tokenizer = BatchMapper(tokenize, batch_format=\"pandas\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Fine-tuning the model with Ray AIR \n", + "\n", + "We can now configure Ray AIR's {class}`~ray.train.huggingface.huggingface_trainer.HuggingFaceTrainer` to perform distributed fine-tuning of the model. In order to do that, we specify a `trainer_init_per_worker` function, which creates a 🤗 Transformers `Trainer` that will be distributed by Ray using Distributed Data Parallelism (using PyTorch Distributed backend internally). This means that each worker will have its own copy of the model, but operate on different data, At the end of each step, all the workers will sync gradients.\n", + "\n", + "Because GPT-J is a relatively large model, it may not be possible to fit it on smaller GPU types (<=16 GB GRAM). To deal with that issue, we can use [DeepSpeed](https://github.com/microsoft/DeepSpeed), a library to optimize the training process and allow us to (among other things) offload and partition optimizer and parameter states, reducing GRAM usage. Furthermore, DeepSpeed ZeRO Stage 3 allows us to load large models without running out of memory.\n", + "\n", + "🤗 Transformers and Ray AIR's integration ({class}`~ray.train.huggingface.huggingface_trainer.HuggingFaceTrainer`) allow you to easily configure and use DDP and DeepSpeed. All you need to do is specify the DeepSpeed configuration in the [`TrainingArguments`](https://huggingface.co/docs/transformers/en/main_classes/trainer#transformers.TrainingArguments) object.\n", + "\n", + "```{tip}\n", + "There are many DeepSpeed settings that allow you to trade-off speed for memory usage. The settings used below are tailored to the cluster setup used (16 g4dn.4xlarge nodes) and per device batch size of 16. Some things to keep in mind:\n", + "- If your GPUs support bfloat16, use that instead of float16 mixed precision to get better performance and prevent overflows. Replace `fp16=True` with `bf16=True` in `TrainingArguments`.\n", + "- If you are running out of GRAM: try reducing batch size (defined in the cell below the next one), set `\"overlap_comm\": False` in DeepSpeed config.\n", + "- If you are running out of RAM, add more nodes to your cluster, use nodes with more RAM, set `\"pin_memory\": False` in the DeepSpeed config, reduce the batch size, and remove `\"offload_param\"` from the DeepSpeed config.\n", + "\n", + "For more information on DeepSpeed configuration, refer to [Hugging Face documentation](https://huggingface.co/docs/transformers/main_classes/deepspeed) and [DeepSpeed documentation](https://www.deepspeed.ai/docs/config-json/).\n", + "\n", + "Additionally, if you prefer a lower-level API, the logic below can be expressed as an [Accelerate training loop](https://github.com/huggingface/accelerate/blob/main/examples/by_feature/deepspeed_with_config_support.py) distributed by a Ray AIR {class}`~ray.train.torch.torch_trainer.TorchTrainer`.\n", + "```\n", + "\n", + "#### Training speed\n", + "\n", + "As we are using data parallelism, each worker operates on its own shard of the data. The batch size set in `TrainingArguments` is the **per device batch size** (per worker batch size). By changing the number of workers, we can change the **effective batch size** and thus the time needed for training to complete. The effective batch size is then calculated as `per device batch size * number of workers * number of gradient accumulation steps`. As we add more workers, the effective batch size rises and thus we need less time to complete a full epoch. While the speedup is not exactly linear due to extra communication overheads, in many cases it can be close to linear.\n", + "\n", + "The preprocessed dataset has 1348 examples. We have set per device batch size to 16.\n", + "\n", + "* With 16 g4dn.4xlarge nodes, the effective batch size was 256, which equals to 85 steps per epoch. One epoch took **~2440 seconds** (including initialization time).\n", + "\n", + "* With 32 g4dn.4xlarge nodes, the effective batch size was 512, which equals to 43 steps per epoch. One epoch took **~1280 seconds** (including initialization time)." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "import evaluate\n", + "from transformers import Trainer, TrainingArguments\n", + "from transformers import (\n", + " GPTJForCausalLM,\n", + " AutoTokenizer,\n", + " default_data_collator,\n", + ")\n", + "from transformers.utils.logging import disable_progress_bar, enable_progress_bar\n", + "import torch\n", + "\n", + "from ray.air import session\n", + "\n", + "\n", + "def trainer_init_per_worker(train_dataset, eval_dataset=None, **config):\n", + " # Use the actual number of CPUs assigned by Ray\n", + " os.environ[\"OMP_NUM_THREADS\"] = str(\n", + " session.get_trial_resources().bundles[-1].get(\"CPU\", 1)\n", + " )\n", + " # Enable tf32 for better performance\n", + " torch.backends.cuda.matmul.allow_tf32 = True\n", + "\n", + " batch_size = config.get(\"batch_size\", 4)\n", + " epochs = config.get(\"epochs\", 2)\n", + " warmup_steps = config.get(\"warmup_steps\", 0)\n", + " learning_rate = config.get(\"learning_rate\", 0.00002)\n", + " weight_decay = config.get(\"weight_decay\", 0.01)\n", + "\n", + " deepspeed = {\n", + " \"fp16\": {\n", + " \"enabled\": \"auto\",\n", + " \"initial_scale_power\": 8,\n", + " },\n", + " \"bf16\": {\"enabled\": \"auto\"},\n", + " \"optimizer\": {\n", + " \"type\": \"AdamW\",\n", + " \"params\": {\n", + " \"lr\": \"auto\",\n", + " \"betas\": \"auto\",\n", + " \"eps\": \"auto\",\n", + " },\n", + " },\n", + " \"zero_optimization\": {\n", + " \"stage\": 3,\n", + " \"offload_optimizer\": {\n", + " \"device\": \"cpu\",\n", + " \"pin_memory\": True,\n", + " },\n", + " \"offload_param\": {\n", + " \"device\": \"cpu\",\n", + " \"pin_memory\": True,\n", + " },\n", + " \"overlap_comm\": True,\n", + " \"contiguous_gradients\": True,\n", + " \"reduce_bucket_size\": \"auto\",\n", + " \"stage3_prefetch_bucket_size\": \"auto\",\n", + " \"stage3_param_persistence_threshold\": \"auto\",\n", + " \"gather_16bit_weights_on_model_save\": True,\n", + " \"round_robin_gradients\": True,\n", + " },\n", + " \"gradient_accumulation_steps\": \"auto\",\n", + " \"gradient_clipping\": \"auto\",\n", + " \"steps_per_print\": 10,\n", + " \"train_batch_size\": \"auto\",\n", + " \"train_micro_batch_size_per_gpu\": \"auto\",\n", + " \"wall_clock_breakdown\": False,\n", + " }\n", + "\n", + " print(\"Preparing training arguments\")\n", + " training_args = TrainingArguments(\n", + " \"output\",\n", + " per_device_train_batch_size=batch_size,\n", + " logging_steps=1,\n", + " save_strategy=\"no\",\n", + " per_device_eval_batch_size=batch_size,\n", + " learning_rate=learning_rate,\n", + " weight_decay=weight_decay,\n", + " warmup_steps=warmup_steps,\n", + " label_names=[\"input_ids\", \"attention_mask\"],\n", + " num_train_epochs=epochs,\n", + " push_to_hub=False,\n", + " disable_tqdm=True, # declutter the output a little\n", + " fp16=True,\n", + " gradient_checkpointing=True,\n", + " deepspeed=deepspeed,\n", + " )\n", + " disable_progress_bar()\n", + "\n", + " tokenizer = AutoTokenizer.from_pretrained(model_name)\n", + " tokenizer.pad_token = tokenizer.eos_token\n", + "\n", + " print(\"Loading model\")\n", + "\n", + " model = GPTJForCausalLM.from_pretrained(model_name, use_cache=False)\n", + " model.resize_token_embeddings(len(tokenizer))\n", + "\n", + " print(\"Model loaded\")\n", + "\n", + " enable_progress_bar()\n", + "\n", + " metric = evaluate.load(\"accuracy\")\n", + "\n", + " def compute_metrics(eval_pred):\n", + " logits, labels = eval_pred\n", + " predictions = np.argmax(logits, axis=-1)\n", + " return metric.compute(predictions=predictions, references=labels)\n", + "\n", + " trainer = Trainer(\n", + " model=model,\n", + " args=training_args,\n", + " train_dataset=train_dataset,\n", + " eval_dataset=eval_dataset,\n", + " compute_metrics=compute_metrics,\n", + " tokenizer=tokenizer,\n", + " data_collator=default_data_collator,\n", + " )\n", + " return trainer" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "With our `trainer_init_per_worker` complete, we can now instantiate the {class}`~ray.train.huggingface.huggingface_trainer.HuggingFaceTrainer`. Aside from the function, we set the `scaling_config`, controlling the amount of workers and resources used, and the `datasets` we will use for training and evaluation.\n", + "\n", + "We pass the preprocessors we have defined earlier as an argument, wrapped in a {class}`~ray.data.preprocessors.chain.Chain`. The preprocessor will be included with the returned {class}`~ray.air.checkpoint.Checkpoint`, meaning it will also be applied during inference.\n", + "\n", + "```{note}\n", + "If you want to upload checkpoints to cloud storage (eg. S3), use {class}`~ray.tune.syncer.SyncConfig` - see {ref}`train-config-sync` for an example. Using cloud storage is highly recommended, especially for production.\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "from ray.train.huggingface import HuggingFaceTrainer\n", + "from ray.air.config import ScalingConfig\n", + "from ray.data.preprocessors import Chain\n", + "\n", + "\n", + "trainer = HuggingFaceTrainer(\n", + " trainer_init_per_worker=trainer_init_per_worker,\n", + " trainer_init_config={\n", + " \"batch_size\": 16, # per device\n", + " \"epochs\": 1,\n", + " },\n", + " scaling_config=ScalingConfig(\n", + " num_workers=num_workers,\n", + " use_gpu=use_gpu,\n", + " resources_per_worker={\"GPU\": 1, \"CPU\": cpus_per_worker},\n", + " ),\n", + " datasets={\"train\": ray_datasets[\"train\"], \"evaluation\": ray_datasets[\"validation\"]},\n", + " preprocessor=Chain(splitter, tokenizer),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, we call the {meth}`~ray.train.huggingface.huggingface_trainer.HuggingFaceTrainer.fit` method to start training with Ray AIR. We will save the {class}`~ray.air.Result` object to a variable so we can access metrics and checkpoints." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "
\n", + "
\n", + "

Tune Status

\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Current time:2023-03-06 17:18:41
Running for: 00:43:11.46
Memory: 31.9/62.0 GiB
\n", + "
\n", + "
\n", + "
\n", + "

System Info

\n", + " Using FIFO scheduling algorithm.
Resources requested: 0/256 CPUs, 0/16 GPUs, 0.0/675.29 GiB heap, 0.0/291.99 GiB objects (0.0/16.0 accelerator_type:T4)\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "

Trial Status

\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Trial name status loc iter total time (s) loss learning_rate epoch
HuggingFaceTrainer_f623d_00000TERMINATED10.0.30.196:30861 85 2579.30.0715 4.70588e-07 1
\n", + "
\n", + "
\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "(RayTrainWorker pid=31281) 2023-03-06 16:36:00,447\tINFO bulk_executor.py:41 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[BatchMapper]\n", + "(RayTrainWorker pid=1964, ip=10.0.26.83) /tmp/ray/session_2023-03-06_15-55-37_997701_162/runtime_resources/py_modules_files/_ray_pkg_f864ba6869d6802c/ray/train/_internal/dataset_iterator.py:64: UserWarning: session.get_dataset_shard returns a ray.data.DatasetIterator instead of a Dataset/DatasetPipeline as of Ray v2.3. Use iter_torch_batches(), to_tf(), or iter_batches() to iterate over one epoch. See https://docs.ray.io/en/latest/data/api/dataset_iterator.html for full DatasetIterator docs.\n", + "(RayTrainWorker pid=1964, ip=10.0.26.83) warnings.warn(\n", + "(RayTrainWorker pid=1964, ip=10.0.26.83) 2023-03-06 16:36:00,453\tINFO bulk_executor.py:41 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[BatchMapper]\n", + "(RayTrainWorker pid=1963, ip=10.0.54.163) /tmp/ray/session_2023-03-06_15-55-37_997701_162/runtime_resources/py_modules_files/_ray_pkg_f864ba6869d6802c/ray/train/_internal/dataset_iterator.py:64: UserWarning: session.get_dataset_shard returns a ray.data.DatasetIterator instead of a Dataset/DatasetPipeline as of Ray v2.3. Use iter_torch_batches(), to_tf(), or iter_batches() to iterate over one epoch. See https://docs.ray.io/en/latest/data/api/dataset_iterator.html for full DatasetIterator docs.\n", + "(RayTrainWorker pid=1963, ip=10.0.54.163) warnings.warn(\n", + "(RayTrainWorker pid=1963, ip=10.0.54.163) 2023-03-06 16:36:00,452\tINFO bulk_executor.py:41 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[BatchMapper]\n", + "(RayTrainWorker pid=1954, ip=10.0.15.115) /tmp/ray/session_2023-03-06_15-55-37_997701_162/runtime_resources/py_modules_files/_ray_pkg_f864ba6869d6802c/ray/train/_internal/dataset_iterator.py:64: UserWarning: session.get_dataset_shard returns a ray.data.DatasetIterator instead of a Dataset/DatasetPipeline as of Ray v2.3. Use iter_torch_batches(), to_tf(), or iter_batches() to iterate over one epoch. See https://docs.ray.io/en/latest/data/api/dataset_iterator.html for full DatasetIterator docs.\n", + "(RayTrainWorker pid=1954, ip=10.0.15.115) warnings.warn(\n", + "(RayTrainWorker pid=1954, ip=10.0.15.115) 2023-03-06 16:36:00,452\tINFO bulk_executor.py:41 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[BatchMapper]\n", + "(RayTrainWorker pid=1955, ip=10.0.58.255) /tmp/ray/session_2023-03-06_15-55-37_997701_162/runtime_resources/py_modules_files/_ray_pkg_f864ba6869d6802c/ray/train/_internal/dataset_iterator.py:64: UserWarning: session.get_dataset_shard returns a ray.data.DatasetIterator instead of a Dataset/DatasetPipeline as of Ray v2.3. Use iter_torch_batches(), to_tf(), or iter_batches() to iterate over one epoch. See https://docs.ray.io/en/latest/data/api/dataset_iterator.html for full DatasetIterator docs.\n", + "(RayTrainWorker pid=1955, ip=10.0.58.255) warnings.warn(\n", + "(RayTrainWorker pid=1955, ip=10.0.58.255) 2023-03-06 16:36:00,453\tINFO bulk_executor.py:41 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[BatchMapper]\n", + "(RayTrainWorker pid=1942, ip=10.0.57.85) 2023-03-06 16:36:00,452\tINFO bulk_executor.py:41 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[BatchMapper]\n", + "(RayTrainWorker pid=1963, ip=10.0.29.205) 2023-03-06 16:36:00,452\tINFO bulk_executor.py:41 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[BatchMapper]\n", + "(RayTrainWorker pid=1942, ip=10.0.51.113) 2023-03-06 16:36:00,454\tINFO bulk_executor.py:41 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[BatchMapper]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(RayTrainWorker pid=31281) Preparing training arguments\n", + "(RayTrainWorker pid=31281) Loading model\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:37:21,252] [INFO] [partition_parameters.py:415:__exit__] finished initializing model with 6.05B parameters\n", + "(RayTrainWorker pid=31281) Model loaded\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "(RayTrainWorker pid=31281) Using cuda_amp half precision backend\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(RayTrainWorker pid=31281) [2023-03-06 16:38:03,431] [INFO] [logging.py:75:log_dist] [Rank 0] DeepSpeed info: version=0.8.1, git-hash=unknown, git-branch=unknown\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:03,450] [INFO] [logging.py:75:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "(RayTrainWorker pid=31281) ***** Running training *****\n", + "(RayTrainWorker pid=31281) Num examples = 1348\n", + "(RayTrainWorker pid=31281) Num Epochs = 1\n", + "(RayTrainWorker pid=31281) Instantaneous batch size per device = 16\n", + "(RayTrainWorker pid=31281) Total train batch size (w. parallel, distributed & accumulation) = 256\n", + "(RayTrainWorker pid=31281) Gradient Accumulation steps = 1\n", + "(RayTrainWorker pid=31281) Total optimization steps = 85\n", + "(RayTrainWorker pid=31281) Number of trainable parameters = 0\n", + "(RayTrainWorker pid=31281) /home/ray/anaconda3/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py:2387: UserWarning: torch.distributed._all_gather_base is a private function and will be deprecated. Please use torch.distributed.all_gather_into_tensor instead.\n", + "(RayTrainWorker pid=31281) warnings.warn(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,024] [INFO] [logging.py:75:log_dist] [Rank 0] DeepSpeed Final Optimizer = adamw\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,024] [INFO] [logging.py:75:log_dist] [Rank 0] DeepSpeed using client callable to create LR scheduler\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,025] [INFO] [logging.py:75:log_dist] [Rank 0] DeepSpeed LR Scheduler = \n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,025] [INFO] [logging.py:75:log_dist] [Rank 0] step=0, skipped=0, lr=[2e-05], mom=[[0.9, 0.999]]\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,025] [INFO] [config.py:1009:print] DeepSpeedEngine configuration:\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,026] [INFO] [config.py:1013:print] activation_checkpointing_config {\n", + "(RayTrainWorker pid=31281) \"partition_activations\": false, \n", + "(RayTrainWorker pid=31281) \"contiguous_memory_optimization\": false, \n", + "(RayTrainWorker pid=31281) \"cpu_checkpointing\": false, \n", + "(RayTrainWorker pid=31281) \"number_checkpoints\": null, \n", + "(RayTrainWorker pid=31281) \"synchronize_checkpoint_boundary\": false, \n", + "(RayTrainWorker pid=31281) \"profile\": false\n", + "(RayTrainWorker pid=31281) }\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,026] [INFO] [config.py:1013:print] aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True}\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,026] [INFO] [config.py:1013:print] amp_enabled .................. False\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,026] [INFO] [config.py:1013:print] amp_params ................... False\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,027] [INFO] [config.py:1013:print] autotuning_config ............ {\n", + "(RayTrainWorker pid=31281) \"enabled\": false, \n", + "(RayTrainWorker pid=31281) \"start_step\": null, \n", + "(RayTrainWorker pid=31281) \"end_step\": null, \n", + "(RayTrainWorker pid=31281) \"metric_path\": null, \n", + "(RayTrainWorker pid=31281) \"arg_mappings\": null, \n", + "(RayTrainWorker pid=31281) \"metric\": \"throughput\", \n", + "(RayTrainWorker pid=31281) \"model_info\": null, \n", + "(RayTrainWorker pid=31281) \"results_dir\": \"autotuning_results\", \n", + "(RayTrainWorker pid=31281) \"exps_dir\": \"autotuning_exps\", \n", + "(RayTrainWorker pid=31281) \"overwrite\": true, \n", + "(RayTrainWorker pid=31281) \"fast\": true, \n", + "(RayTrainWorker pid=31281) \"start_profile_step\": 3, \n", + "(RayTrainWorker pid=31281) \"end_profile_step\": 5, \n", + "(RayTrainWorker pid=31281) \"tuner_type\": \"gridsearch\", \n", + "(RayTrainWorker pid=31281) \"tuner_early_stopping\": 5, \n", + "(RayTrainWorker pid=31281) \"tuner_num_trials\": 50, \n", + "(RayTrainWorker pid=31281) \"model_info_path\": null, \n", + "(RayTrainWorker pid=31281) \"mp_size\": 1, \n", + "(RayTrainWorker pid=31281) \"max_train_batch_size\": null, \n", + "(RayTrainWorker pid=31281) \"min_train_batch_size\": 1, \n", + "(RayTrainWorker pid=31281) \"max_train_micro_batch_size_per_gpu\": 1.024000e+03, \n", + "(RayTrainWorker pid=31281) \"min_train_micro_batch_size_per_gpu\": 1, \n", + "(RayTrainWorker pid=31281) \"num_tuning_micro_batch_sizes\": 3\n", + "(RayTrainWorker pid=31281) }\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,027] [INFO] [config.py:1013:print] bfloat16_enabled ............. False\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,027] [INFO] [config.py:1013:print] checkpoint_parallel_write_pipeline False\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,027] [INFO] [config.py:1013:print] checkpoint_tag_validation_enabled True\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,027] [INFO] [config.py:1013:print] checkpoint_tag_validation_fail False\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,027] [INFO] [config.py:1013:print] comms_config ................. \n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,027] [INFO] [config.py:1013:print] communication_data_type ...... None\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,027] [INFO] [config.py:1013:print] compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}}\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,027] [INFO] [config.py:1013:print] curriculum_enabled_legacy .... False\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,027] [INFO] [config.py:1013:print] curriculum_params_legacy ..... False\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,027] [INFO] [config.py:1013:print] data_efficiency_config ....... {'enabled': False, 'seed': 1234, 'data_sampling': {'enabled': False, 'num_epochs': 1000, 'num_workers': 0, 'curriculum_learning': {'enabled': False}}, 'data_routing': {'enabled': False, 'random_ltd': {'enabled': False, 'layer_token_lr_schedule': {'enabled': False}}}}\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,027] [INFO] [config.py:1013:print] data_efficiency_enabled ...... False\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,027] [INFO] [config.py:1013:print] dataloader_drop_last ......... False\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,027] [INFO] [config.py:1013:print] disable_allgather ............ False\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,027] [INFO] [config.py:1013:print] dump_state ................... False\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,027] [INFO] [config.py:1013:print] dynamic_loss_scale_args ...... {'init_scale': 256, 'scale_window': 1000, 'delayed_shift': 2, 'min_scale': 1}\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,027] [INFO] [config.py:1013:print] eigenvalue_enabled ........... False\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,027] [INFO] [config.py:1013:print] eigenvalue_gas_boundary_resolution 1\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,027] [INFO] [config.py:1013:print] eigenvalue_layer_name ........ bert.encoder.layer\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,027] [INFO] [config.py:1013:print] eigenvalue_layer_num ......... 0\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,027] [INFO] [config.py:1013:print] eigenvalue_max_iter .......... 100\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,027] [INFO] [config.py:1013:print] eigenvalue_stability ......... 1e-06\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,027] [INFO] [config.py:1013:print] eigenvalue_tol ............... 0.01\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,027] [INFO] [config.py:1013:print] eigenvalue_verbose ........... False\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,027] [INFO] [config.py:1013:print] elasticity_enabled ........... False\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,027] [INFO] [config.py:1013:print] flops_profiler_config ........ {\n", + "(RayTrainWorker pid=31281) \"enabled\": false, \n", + "(RayTrainWorker pid=31281) \"profile_step\": 1, \n", + "(RayTrainWorker pid=31281) \"module_depth\": -1, \n", + "(RayTrainWorker pid=31281) \"top_modules\": 1, \n", + "(RayTrainWorker pid=31281) \"detailed\": true, \n", + "(RayTrainWorker pid=31281) \"output_file\": null\n", + "(RayTrainWorker pid=31281) }\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,027] [INFO] [config.py:1013:print] fp16_auto_cast ............... False\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,027] [INFO] [config.py:1013:print] fp16_enabled ................. True\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,027] [INFO] [config.py:1013:print] fp16_master_weights_and_gradients False\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,027] [INFO] [config.py:1013:print] global_rank .................. 0\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,027] [INFO] [config.py:1013:print] grad_accum_dtype ............. None\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,027] [INFO] [config.py:1013:print] gradient_accumulation_steps .. 1\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,028] [INFO] [config.py:1013:print] gradient_clipping ............ 1.0\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,028] [INFO] [config.py:1013:print] gradient_predivide_factor .... 1.0\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,028] [INFO] [config.py:1013:print] initial_dynamic_scale ........ 256\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,028] [INFO] [config.py:1013:print] load_universal_checkpoint .... False\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,028] [INFO] [config.py:1013:print] loss_scale ................... 0\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,028] [INFO] [config.py:1013:print] memory_breakdown ............. False\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,028] [INFO] [config.py:1013:print] monitor_config ............... tensorboard=TensorBoardConfig(enabled=False, output_path='', job_name='DeepSpeedJobName') wandb=WandbConfig(enabled=False, group=None, team=None, project='deepspeed') csv_monitor=CSVConfig(enabled=False, output_path='', job_name='DeepSpeedJobName') enabled=False\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,028] [INFO] [config.py:1013:print] nebula_config ................ {\n", + "(RayTrainWorker pid=31281) \"enabled\": false, \n", + "(RayTrainWorker pid=31281) \"persistent_storage_path\": null, \n", + "(RayTrainWorker pid=31281) \"persistent_time_interval\": 100, \n", + "(RayTrainWorker pid=31281) \"num_of_version_in_retention\": 2, \n", + "(RayTrainWorker pid=31281) \"enable_nebula_load\": true, \n", + "(RayTrainWorker pid=31281) \"load_path\": null\n", + "(RayTrainWorker pid=31281) }\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,028] [INFO] [config.py:1013:print] optimizer_legacy_fusion ...... False\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,028] [INFO] [config.py:1013:print] optimizer_name ............... adamw\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,028] [INFO] [config.py:1013:print] optimizer_params ............. {'lr': 2e-05, 'betas': [0.9, 0.999], 'eps': 1e-08}\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,028] [INFO] [config.py:1013:print] pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0}\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,028] [INFO] [config.py:1013:print] pld_enabled .................. False\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,028] [INFO] [config.py:1013:print] pld_params ................... False\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,028] [INFO] [config.py:1013:print] prescale_gradients ........... False\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,028] [INFO] [config.py:1013:print] scheduler_name ............... None\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,028] [INFO] [config.py:1013:print] scheduler_params ............. None\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,028] [INFO] [config.py:1013:print] sparse_attention ............. None\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,028] [INFO] [config.py:1013:print] sparse_gradients_enabled ..... False\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,028] [INFO] [config.py:1013:print] steps_per_print .............. 10\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,028] [INFO] [config.py:1013:print] train_batch_size ............. 256\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,028] [INFO] [config.py:1013:print] train_micro_batch_size_per_gpu 16\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,028] [INFO] [config.py:1013:print] use_node_local_storage ....... False\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,028] [INFO] [config.py:1013:print] wall_clock_breakdown ......... False\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,028] [INFO] [config.py:1013:print] world_size ................... 16\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,028] [INFO] [config.py:1013:print] zero_allow_untested_optimizer False\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,028] [INFO] [config.py:1013:print] zero_config .................. stage=3 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=16777216 allgather_partitions=True allgather_bucket_size=500,000,000 overlap_comm=True load_from_fp32_weights=True elastic_checkpoint=False offload_param=DeepSpeedZeroOffloadParamConfig(device='cpu', nvme_path=None, buffer_count=5, buffer_size=100,000,000, max_in_cpu=1,000,000,000, pin_memory=True) offload_optimizer=DeepSpeedZeroOffloadOptimizerConfig(device='cpu', nvme_path=None, buffer_count=4, pin_memory=True, pipeline=False, pipeline_read=False, pipeline_write=False, fast_init=False) sub_group_size=1,000,000,000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=15099494 param_persistence_threshold=40960 model_persistence_threshold=sys.maxsize max_live_parameters=1,000,000,000 max_reuse_distance=1,000,000,000 gather_16bit_weights_on_model_save=True stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=True\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,028] [INFO] [config.py:1013:print] zero_enabled ................. True\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,028] [INFO] [config.py:1013:print] zero_optimization_stage ...... 3\n", + "(RayTrainWorker pid=31281) [2023-03-06 16:38:25,029] [INFO] [config.py:998:print_user_config] json = {\n", + "(RayTrainWorker pid=31281) \"fp16\": {\n", + "(RayTrainWorker pid=31281) \"enabled\": true, \n", + "(RayTrainWorker pid=31281) \"initial_scale_power\": 8\n", + "(RayTrainWorker pid=31281) }, \n", + "(RayTrainWorker pid=31281) \"bf16\": {\n", + "(RayTrainWorker pid=31281) \"enabled\": false\n", + "(RayTrainWorker pid=31281) }, \n", + "(RayTrainWorker pid=31281) \"optimizer\": {\n", + "(RayTrainWorker pid=31281) \"type\": \"AdamW\", \n", + "(RayTrainWorker pid=31281) \"params\": {\n", + "(RayTrainWorker pid=31281) \"lr\": 2e-05, \n", + "(RayTrainWorker pid=31281) \"betas\": [0.9, 0.999], \n", + "(RayTrainWorker pid=31281) \"eps\": 1e-08\n", + "(RayTrainWorker pid=31281) }\n", + "(RayTrainWorker pid=31281) }, \n", + "(RayTrainWorker pid=31281) \"zero_optimization\": {\n", + "(RayTrainWorker pid=31281) \"stage\": 3, \n", + "(RayTrainWorker pid=31281) \"offload_optimizer\": {\n", + "(RayTrainWorker pid=31281) \"device\": \"cpu\", \n", + "(RayTrainWorker pid=31281) \"pin_memory\": true\n", + "(RayTrainWorker pid=31281) }, \n", + "(RayTrainWorker pid=31281) \"offload_param\": {\n", + "(RayTrainWorker pid=31281) \"device\": \"cpu\", \n", + "(RayTrainWorker pid=31281) \"pin_memory\": true\n", + "(RayTrainWorker pid=31281) }, \n", + "(RayTrainWorker pid=31281) \"overlap_comm\": true, \n", + "(RayTrainWorker pid=31281) \"contiguous_gradients\": true, \n", + "(RayTrainWorker pid=31281) \"reduce_bucket_size\": 1.677722e+07, \n", + "(RayTrainWorker pid=31281) \"stage3_prefetch_bucket_size\": 1.509949e+07, \n", + "(RayTrainWorker pid=31281) \"stage3_param_persistence_threshold\": 4.096000e+04, \n", + "(RayTrainWorker pid=31281) \"gather_16bit_weights_on_model_save\": true, \n", + "(RayTrainWorker pid=31281) \"round_robin_gradients\": true\n", + "(RayTrainWorker pid=31281) }, \n", + "(RayTrainWorker pid=31281) \"gradient_accumulation_steps\": 1, \n", + "(RayTrainWorker pid=31281) \"gradient_clipping\": 1.0, \n", + "(RayTrainWorker pid=31281) \"steps_per_print\": 10, \n", + "(RayTrainWorker pid=31281) \"train_batch_size\": 256, \n", + "(RayTrainWorker pid=31281) \"train_micro_batch_size_per_gpu\": 16, \n", + "(RayTrainWorker pid=31281) \"wall_clock_breakdown\": false\n", + "(RayTrainWorker pid=31281) }\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "(RayTrainWorker pid=31281) Model weights saved in output/checkpoint-85/pytorch_model.bin\n", + "(RayTrainWorker pid=31281) tokenizer config file saved in output/checkpoint-85/tokenizer_config.json\n", + "(RayTrainWorker pid=31281) Special tokens file saved in output/checkpoint-85/special_tokens_map.json\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(RayTrainWorker pid=31281) [2023-03-06 17:18:13,320] [INFO] [engine.py:3516:save_16bit_model] Saving model weights to output/checkpoint-85/pytorch_model.bin\n", + "(RayTrainWorker pid=31281) [2023-03-06 17:18:13,320] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving output/checkpoint-85/pytorch_model.bin...\n", + "(RayTrainWorker pid=31281) [2023-03-06 17:18:29,075] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved output/checkpoint-85/pytorch_model.bin.\n", + "(RayTrainWorker pid=31281) [2023-03-06 17:18:29,087] [INFO] [logging.py:75:log_dist] [Rank 0] [Torch] Checkpoint global_step85 is begin to save!\n", + "(RayTrainWorker pid=31281) [2023-03-06 17:18:29,109] [INFO] [logging.py:75:log_dist] [Rank 0] Saving model checkpoint: output/checkpoint-85/global_step85/zero_pp_rank_0_mp_rank_00_model_states.pt\n", + "(RayTrainWorker pid=31281) [2023-03-06 17:18:29,109] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving output/checkpoint-85/global_step85/zero_pp_rank_0_mp_rank_00_model_states.pt...\n", + "(RayTrainWorker pid=31281) [2023-03-06 17:18:37,982] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved output/checkpoint-85/global_step85/zero_pp_rank_0_mp_rank_00_optim_states.pt.\n", + "(RayTrainWorker pid=31281) [2023-03-06 17:18:37,984] [INFO] [engine.py:3407:_save_zero_checkpoint] zero checkpoint saved output/checkpoint-85/global_step85/zero_pp_rank_0_mp_rank_00_optim_states.pt\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "(RayTrainWorker pid=31281) \n", + "(RayTrainWorker pid=31281) \n", + "(RayTrainWorker pid=31281) Training completed. Do not forget to share your model on huggingface.co/models =)\n", + "(RayTrainWorker pid=31281) \n", + "(RayTrainWorker pid=31281) \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(RayTrainWorker pid=31281) [2023-03-06 17:18:38,143] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step85 is ready now!\n", + "(RayTrainWorker pid=31281) {'train_runtime': 2413.1243, 'train_samples_per_second': 0.559, 'train_steps_per_second': 0.035, 'train_loss': 0.32492108064539293, 'epoch': 1.0}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-03-06 17:18:41,018\tINFO tune.py:825 -- Total run time: 2591.59 seconds (2591.46 seconds for the tuning loop).\n" + ] + } + ], + "source": [ + "results = trainer.fit()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can use the returned {class}`~ray.air.Result` object to access metrics and the Ray AIR {class}`~ray.air.checkpoint.Checkpoint` associated with the last iteration." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "HuggingFaceCheckpoint(local_path=/home/ray/ray_results/HuggingFaceTrainer_2023-03-06_16-35-29/HuggingFaceTrainer_f623d_00000_0_2023-03-06_16-35-30/checkpoint_000000)" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "checkpoint = results.checkpoint\n", + "checkpoint" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Generate text from prompt\n", + "\n", + "We can use the {class}`~ray.train.huggingface.huggingface_predictor.HuggingFacePredictor` to generate predictions from our fine-tuned model.\n", + "\n", + "```{tip}\n", + "For large scale batch inference, consider configuring cloud checkpointing and then pass the cloud-backed {class}`~ray.air.checkpoint.Checkpoint` to {class}`~ray.train.batch_predictor.BatchPredictor`. More information [here](air-predictors).\n", + "```\n", + "\n", + "Because the {class}`~ray.train.huggingface.huggingface_predictor.HuggingFacePredictor` uses a 🤗 Transformers [`pipeline`](https://huggingface.co/docs/transformers/en/main_classes/pipelines) under the hood, we disable the tokenizer AIR Preprocessor we have used for training and let the `pipeline` to tokenize the data itself." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "checkpoint.set_preprocessor(None)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + ". We also set `device_map=\"auto\"` so that the model is automatically placed on the right device and set the `task` to `\"text-generation\"`. The `predict` method passes the arguments to a 🤗 Transformers `pipeline` call." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from ray.train.huggingface import HuggingFacePredictor\n", + "import pandas as pd\n", + "\n", + "prompts = pd.DataFrame([\"Romeo and Juliet\", \"Romeo\", \"Juliet\"], columns=[\"text\"])\n", + "\n", + "# Predict on the head node.\n", + "predictor = HuggingFacePredictor.from_checkpoint(\n", + " checkpoint=checkpoint,\n", + " task=\"text-generation\",\n", + " torch_dtype=torch.float16 if use_gpu else None,\n", + " device_map=\"auto\",\n", + " use_gpu=use_gpu,\n", + ")\n", + "prediction = predictor.predict(\n", + " prompts,\n", + " do_sample=True,\n", + " temperature=0.9,\n", + " min_length=32,\n", + " max_length=128,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
generated_text
0Romeo and Juliet, they are married: and it is ...
1Romeo, thou art Romeo and a Montague; for only...
2Juliet's name; but I do not sound an ear to na...
\n", + "
" + ], + "text/plain": [ + " generated_text\n", + "0 Romeo and Juliet, they are married: and it is ...\n", + "1 Romeo, thou art Romeo and a Montague; for only...\n", + "2 Juliet's name; but I do not sound an ear to na..." + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "prediction" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.16" + }, + "vscode": { + "interpreter": { + "hash": "3c0d54d489a08ae47a06eae2fd00ff032d6cddb527c382959b7b2575f6a8167f" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/doc/source/ray-air/examples/gptj_serving.ipynb b/doc/source/ray-air/examples/gptj_serving.ipynb new file mode 100644 index 0000000000000..fb84ec5d6a136 --- /dev/null +++ b/doc/source/ray-air/examples/gptj_serving.ipynb @@ -0,0 +1,247 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# GPT-J-6B Serving with Ray AIR\n", + "\n", + "In this example, we will showcase how to use the Ray AIR for **GPT-J serving (online inference)**. GPT-J is a GPT-2-like causal language model trained on the Pile dataset. This particular model has 6 billion parameters. For more information on GPT-J, click [here](https://huggingface.co/docs/transformers/model_doc/gptj).\n", + "\n", + "We will use Ray Serve for online inference and a pretrained model from Hugging Face hub. Note that you can easily adapt this example to use other similar models.\n", + "\n", + "It is highly recommended to read [Ray AIR Key Concepts](air-key-concepts) and [Ray Serve Key Concepts](serve-key-concepts) before starting this example.\n", + "\n", + "If you are interested in batch prediction (offline inference), see {doc}`/ray-air/examples/gptj_batch_prediction`.\n", + "\n", + "```{note}\n", + "In order to run this example, make sure your Ray cluster has access to at least one GPU with 16 or more GBs of memory. The amount of memory needed will depend on the model.\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "model_id = \"EleutherAI/gpt-j-6B\"\n", + "revision = \"float16\" # use float16 weights to fit in 16GB GPUs\n", + "prompt = (\n", + " \"In a shocking finding, scientists discovered a herd of unicorns living in a remote, \"\n", + " \"previously unexplored valley, in the Andes Mountains. Even more surprising to the \"\n", + " \"researchers was the fact that the unicorns spoke perfect English.\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import ray" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We define a {ref}`runtime environment ` to ensure that the Ray workers have access to all the necessary packages. You can omit the `runtime_env` argument if you have all of the packages already installed on each node in your cluster." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "ray.init(\n", + " runtime_env={\n", + " \"pip\": [\n", + " \"accelerate>=0.16.0\",\n", + " \"transformers>=4.26.0\",\n", + " \"torch\",\n", + " ]\n", + " }\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Setting up basic serving with Ray Serve is very similar to {doc}`batch inference with Ray Data `. First, we define a callable class that will serve as the [Serve deployment](serve-key-concepts-deployment). At runtime, a deployment consists of a number of *replicas*, which are individual copies of the class or function that are started in separate Ray Actors (processes). The number of replicas can be scaled up or down (or even autoscaled) to match the incoming request load.\n", + "\n", + "We make sure to set the deployment to use 1 GPU by setting `\"num_gpus\"` in `ray_actor_options`. We load the model in `__init__`, which will allow us to save time by initializing a model just once and then use it to handle multiple requests.\n", + "\n", + "```{tip}\n", + "If you want to use inter-node model parallelism, you can also increase `num_gpus`. As we have created the model with `device_map=\"auto\"`, it will be automatically placed on correct devices. Note that this requires nodes with multiple GPUs.\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "from ray import serve\n", + "from starlette.requests import Request\n", + "\n", + "\n", + "@serve.deployment(ray_actor_options={\"num_gpus\": 1})\n", + "class PredictDeployment:\n", + " def __init__(self, model_id: str, revision: str = None):\n", + " from transformers import AutoModelForCausalLM, AutoTokenizer\n", + " import torch\n", + "\n", + " self.model = AutoModelForCausalLM.from_pretrained(\n", + " model_id,\n", + " revision=revision,\n", + " torch_dtype=torch.float16,\n", + " low_cpu_mem_usage=True,\n", + " device_map=\"auto\", # automatically makes use of all GPUs available to the Actor\n", + " )\n", + " self.tokenizer = AutoTokenizer.from_pretrained(model_id)\n", + "\n", + " def generate(self, text: str) -> pd.DataFrame:\n", + " input_ids = self.tokenizer(text, return_tensors=\"pt\").input_ids.to(\n", + " self.model.device\n", + " )\n", + "\n", + " gen_tokens = self.model.generate(\n", + " input_ids,\n", + " do_sample=True,\n", + " temperature=0.9,\n", + " max_length=100,\n", + " )\n", + " return pd.DataFrame(\n", + " self.tokenizer.batch_decode(gen_tokens), columns=[\"responses\"]\n", + " )\n", + "\n", + " async def __call__(self, http_request: Request) -> str:\n", + " json_request: str = await http_request.json()\n", + " prompts = []\n", + " for prompt in json_request:\n", + " text = prompt[\"text\"]\n", + " if isinstance(text, list):\n", + " prompts.extend(text)\n", + " else:\n", + " prompts.append(text)\n", + " return self.generate(prompts)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can now `bind` the deployment with our arguments, and use {meth}`~ray.serve.run` to start it.\n", + "\n", + "```{note}\n", + "If you were running this script outside of a Jupyter notebook, the recommended way is to use the [`serve run` CLI command](serve-cli). In this case, you would remove the `serve.run(deployment)` line, and instead start the deployment by calling `serve run FILENAME:deployment`.\n", + "\n", + "For more information, see [Serve Development Workflow](serve-dev-workflow).\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "RayServeSyncHandle(deployment='PredictDeployment')" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "deployment = PredictDeployment.bind(model_id=model_id, revision=revision)\n", + "serve.run(deployment)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's try submitting a request to our deployment. We will use the same prompt as before, and send a POST request. The deployment will generate a response and return it." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "(ServeReplica:PredictDeployment pid=651, ip=10.0.8.161) The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n", + "(ServeReplica:PredictDeployment pid=651, ip=10.0.8.161) Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[{'responses': 'In a shocking finding, scientists discovered a herd of unicorns living in a remote, previously unexplored valley, in the Andes Mountains. Even more surprising to the researchers was the fact that the unicorns spoke perfect English.\\n\\nThe findings come from a recent expedition to the region of Cordillera del Divisor, in northern Peru. The region was previously known to have an unusually high number of native animals.\\n\\n\"Our team was conducting a population census of the region’'}]\n" + ] + } + ], + "source": [ + "import requests\n", + "\n", + "sample_input = {\"text\": prompt}\n", + "\n", + "output = requests.post(\"http://localhost:8000/\", json=[sample_input]).json()\n", + "print(output)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You may notice that we are not using an AIR {class}`Predictor ` here. This is because Predictors are mainly intended to be used with AIR {class}`Checkpoints `, which we don't for this example. See {ref}`air-predictors` for more information and usage examples." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.8.10 (default, Nov 14 2022, 12:59:47) \n[GCC 9.4.0]" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "3c0d54d489a08ae47a06eae2fd00ff032d6cddb527c382959b7b2575f6a8167f" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/doc/source/ray-air/examples/images/dreambooth_example.png b/doc/source/ray-air/examples/images/dreambooth_example.png new file mode 120000 index 0000000000000..3b65acc893ea7 --- /dev/null +++ b/doc/source/ray-air/examples/images/dreambooth_example.png @@ -0,0 +1 @@ +../../../../../python/ray/air/examples/dreambooth/images/dreambooth_example.png \ No newline at end of file diff --git a/doc/source/ray-air/examples/images/dreambooth_training.png b/doc/source/ray-air/examples/images/dreambooth_training.png new file mode 120000 index 0000000000000..8aa963b347ce8 --- /dev/null +++ b/doc/source/ray-air/examples/images/dreambooth_training.png @@ -0,0 +1 @@ +../../../../../python/ray/air/examples/dreambooth/images/dreambooth_training.png \ No newline at end of file diff --git a/doc/source/ray-air/examples/index.rst b/doc/source/ray-air/examples/index.rst index 2917e588b2c79..252364456b638 100644 --- a/doc/source/ray-air/examples/index.rst +++ b/doc/source/ray-air/examples/index.rst @@ -25,12 +25,16 @@ Text/NLP -------- - :doc:`/ray-air/examples/huggingface_text_classification`: How to use Ray AIR to run Hugging Face Transformers fine-tuning on a text classification task. +- :doc:`/ray-air/examples/gptj_deepspeed_fine_tuning`: How to use Ray AIR to run Hugging Face Transformers with DeepSpeed for fine-tuning a large model. - :doc:`/ray-air/examples/gptj_batch_prediction`: How to use Ray AIR to do batch prediction with the Hugging Face Transformers GPT-J model. +- :doc:`/ray-air/examples/gptj_serving`: How to use Ray AIR to do online serving with the Hugging Face Transformers GPT-J model. +- :doc:`/ray-air/examples/dreambooth_finetuning`: How to fine-tune a DreamBooth text-to-image model with your own images. Image/CV -------- - :doc:`/ray-air/examples/torch_image_example` +- :doc:`/ray-air/examples/torch_detection` - :doc:`/ray-air/examples/pytorch_resnet_batch_prediction` - :doc:`/ray-air/examples/stablediffusion_batch_prediction`: How to use Ray AIR to do batch prediction with the Stable Diffusion text-to-image model. diff --git a/doc/source/ray-air/examples/torch_detection.ipynb b/doc/source/ray-air/examples/torch_detection.ipynb new file mode 100644 index 0000000000000..7b8de7f853f48 --- /dev/null +++ b/doc/source/ray-air/examples/torch_detection.ipynb @@ -0,0 +1,1011 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "da5b9b7e", + "metadata": {}, + "source": [ + "# Fine-tuning a Torch object detection model\n", + "\n", + "This tutorial explains how to fine-tune `fasterrcnn_resnet50_fpn` using the [Ray AI Runtime](air) for parallel data ingest and training.\n", + "\n", + "Here's what you'll do:\n", + "1. Load raw images and [VOC-style](http://host.robots.ox.ac.uk/pascal/VOC/) annotations into a Dataset\n", + "2. Fine-tune `fasterrcnn_resnet50_fpn` (the backbone is pre-trained on ImageNet)\n", + "3. Evaluate the model's accuracy\n", + "\n", + "You should be familiar with [PyTorch](https://pytorch.org/) before starting the\n", + "tutorial. If you need a refresher, read PyTorch's\n", + "[training a classifier](https://pytorch.org/tutorials/beginner/blitz/cifar10_tutorial.html)\n", + "tutorial.\n", + "\n", + "## Before you begin" + ] + }, + { + "cell_type": "markdown", + "id": "e9a6d043", + "metadata": {}, + "source": [ + "* Install the [Ray AI Runtime](air)." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "2d3ae999", + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "!pip install 'ray[air]'" + ] + }, + { + "cell_type": "markdown", + "id": "9b3d4302", + "metadata": {}, + "source": [ + "* Install `torch`, `torchmetrics`, `torchvision`, and `xmltodict`." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "8251d9d9", + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "!pip install torch torchmetrics torchvision xmltodict" + ] + }, + { + "cell_type": "markdown", + "id": "65bf13b8", + "metadata": {}, + "source": [ + "## Create a `Dataset`\n", + "\n", + "You'll work with a subset of [Pascal VOC](http://host.robots.ox.ac.uk/pascal/VOC/) that contains cats and dogs (the full dataset has 20 classes)." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "10df1b5d", + "metadata": {}, + "outputs": [], + "source": [ + "CLASS_TO_LABEL = {\n", + " \"background\": 0,\n", + " \"cat\": 1,\n", + " \"dog\": 2,\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "5567a6d6", + "metadata": {}, + "source": [ + "The dataset contain two subdirectories: `JPEGImages` and `Annotations`. `JPEGImages` contains raw images, and\n", + "`Annotations` contains XML annotations.\n", + "\n", + "```\n", + "AnimalDetection\n", + "├── Annotations\n", + "│ ├── 2007_000063.xml\n", + "│ ├── 2007_000528.xml\n", + "│ └── ...\n", + "└── JPEGImages\n", + " ├── 2007_000063.jpg\n", + " ├── 2007_000528.jpg\n", + " └── ...\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "f821e93d", + "metadata": {}, + "source": [ + "### Define a custom datasource\n", + "\n", + "Each annotation describes the objects in an image.\n", + "\n", + "For example, view this image of a dog:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "a29845a5", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfQAAAF3CAIAAADckC6rAAEAAElEQVR4nHT9d5QkWXUnjj8TLiO9z6wsb7q862rvu6enp8czDGYwI7RCsMiCAOEWASsJhARCiF15CUlIICRAmPGup3u6p311V1dXdXmfVel9ho/33vePHPGTdvcXJ06dyMrIUxlRcT733s/93M+DP/zrbz75Cx8AlAeiYAPAEFxbXBYdgifoBwB4RT/CwGCKDZENHMVC8Wuf/e0//upXHN6widCyAT73x3/hCHcM7TtgUAMKgFkW1VWJsgO9HeMhYWdtKdqUuHBretviuVBcMSkvSKahOYjhocrJ3UN+jknAFgGzbcJhl2FaFFiKUvP6w0sLC5/82CdPnDj2iU//NgAUQ0QJgZjP5tIbq8uSgBAmxDZFJFjEhrxgEpNZpsjjWqkCELQRQgj4XG6HKFVK1fEDx37nq9/8/S/9bmsiSm2tvTnhcMuIl8qaksuXqrWKqSoChxEHIcQccir1uizha9duCC7f97//b3/69W+kl5Z6I5Gz+/Z4qGbWqpzTlU+X2yNdpVxRlJBNDQsQgJggSJplrSW3AUa9PV2pTMbCnEkJxwMbASR7jpw5PbOy0NrRcfnqzTNnzqa2NtPpnZOnjk1OTi4urR89enR+cRFzcHxk9Nlnf9bW2fHUU0/96/d/7Ha7QwHP9vZWU0tzNpsv5Uv9g8O6rt+8ef0dj71tfnFuM7V1+qGzG+vb6VRqd09vMrmZKufPPvrwytxGenvn6JED51572TTNJ5582wsvvCS7nB0dHefPn29vbu7q6vrJT589cuw4J8hXrl09e/b0zvbW8tzS2Mjo7dt3Yk3x/pGBH/zgB088/DDP8y+++tKT73znvfm5bDrTEkvcu3dvdHwsGAl/93vff9e73lUp5DfW1x46++Brr7zkdHsj0dhLr7x63/0PNMViL7/4wtjocLVaWdtM3n/2gdfOnec5dPb+Mz/41+8PDw01t7S8ev5ib/8ujNDt2zcffehMLp9/4/L1x9729qmbNwxDP7hv949+9KPR/rGunr5XXn89Ho93tDWff/3Vd7zjHYuri/PLa/sOHkitb5YKxTNnH/jpj38ScQU7OzouXHh9ZPd4pCV+4/bt4YHRcrGwMjs9Mjwc8sduT98d3zeqGuq1S9fGRkYwRleuXxsdG3N4XJcuXjl536lrb97wOOUjhw8+98xPnR7XocNHXrlw8cChI+VcYWFxbu+e0XR6h5g0FItvZ7KU0ngweG/ubkdPV1dv949/9NNHH310YeZecnvr5MmTL774YqK5pbu377mXXhwbGYp7ApcuvrF7926TkvmllYl9exfuzTFinzh+7Ny5cy63Z2Rs/MaNGwCAw4cP/+QnPxnsH3C73RdfP//AAw8Uy+X5+fn9Bw+tb25kMpn2UGju5fMTbe2SBRAxqa0jBDmORxyn67pp2n7JrRo6JwqGbYSD/mQy6fb6GWPEtAghLp9ftQxFU8PhsFnTqU0EhBkENa3uj4RKpVKpUvYHg4VCATPUHE0YVRUhxMuiYqqEEIiBBRkk1CzUnU4373WXlSoxdIHDjEGTmlASGIIYcoyxuqpwHCc73IvJ1Ed+53f3PPFQnVaBCATIQwCq1SpjzOVyIYQETqir9Xq97nK5qE08Hg9jjFJaKpU0pWap1fTmxtq9aa8gEL3mc7m3k6kf/uT56ZmkDoDDH2xqa378iTND/a1QK2Ez4+VqYQ/HYymTTDLDkhFExCSGTgFCHMaIb2lrzuykTFNHlEqi71/+4aX2iQfe8Wf/zKBIMcEcRwlDkKcUQAwUU+EQkzgZAMosC/KYEIIxBgABAABgAAAAMKWEYh4AgAEFgEAKAOAAgAAABkGtVrYsy+fzAUgRQoxRy7I4jqOUQggBAAAAhBCCmDIKGAcAAIAyxjDGlFKAmG3bCEMEEQJQN3Sex7Zt12o1t9vNAYYFoZjLZTIZdPv2bQChTU1CCWAEA9je3h6PNnlkt9PhUhUtk8oKkOcB/uaffPPtj7/tfe99r8MhA0R1i4Qk0NoUVxRF03VJEpRKBQPo8Xggh+eXlxAAbS3tTlE+sG/frq5OvV7jIFDUGgWMd8g6ASVFeet7A4YQAgBgjBFCfr8fQgghbGpqSiQStm1TShmlCGMGmCiKjdMYgxBCxhhjzGaUUto4xhhzHMcYs21arVZrtZooioyQp59+n+R2G7aFOWF1bWN5eTWb23E7HcFQoLOzs6enB/OczShhUFEUfyCws53/s7/8s4Db1ZpoqlRKBqMrmfTVqTtlg1BBpIz5fJ5sLsUgUQwV8hwn8BZhFiFb29terzcQClZqNUVTbdOipsUDhAD0BwK+gN8pOarlSmtz8+uvvz62e1ySpMnJydbWVodDrNRqJ06cWF1dTaVS9913385W8vr1m+FweHZ2NhgMmqZ568bN1kQThHBmZqavr68pFtvYWDtw4ICmaStLy05ZrpbKFEHeIaVSqZtXri0tLZRr1VqtFggEnE5nLpezbbu9vb1SqSiKEg6Hl5eXfT7f4ODg7OxsR0eHbVmXL19uamoyLFMz9NHxscnJyZ7OrkAw+Nprr/V0deuqOjd7TxTFYDBoWRaAzON0tTQ3GbpeKFcARLLLCRBXKlcGBgZ6evsRQpqmlctlkeM8Tpdl6LqixiJRCHA6nR4YGDAMwzAMQq25ubmBgQGv23371p2AL+hxuVYW5ltbW8vFYjqdfvjhh+/O36vUqgNDQ+l02u/zRYKRK1euPHT2YYRQentndHx8M7k1PT1936nTd2buSA7H7j0TG1ubGHFtzS3pdDoUCrm8vnQmwyBwueSbt281NTV7vd5iudzR0REKhQRBGBsZS6VSEOLe/l1OjzsQ8HX19PCcIMuyrutLSwuRWNTlcmma3tvbP7+0yCDgeVFV1YnxcYdDnltYsG0aj0S1an3v3r3lUgVCePLkyWw27/V6/R5vIVccGBmWPe7tTPrgocOBQCCbSo+NjS0vL1dLZY/HMz8/L8tyW1tbJpMpFovt7e3z8/O7d+92e73pdLqjo6Ner6uqOjExsZlMdnb19I0Ol1VVtQyHLHNYcLnchBBGiKYZLU2JYqUsCALASHY5C/mSx+PBGGuaBgCQJKmUy9ua0ZZo2VhbBxgZlqkYerlajcViuqo1bkihUIjH41VVUQwdijzjEKXUITgkSXJ53CIvcBwXDIcIpZVa1eFwcBwnSCLisMBLhmEgAG3b5nkeY2zbNrUJACAcCQKMVEVxcBJCCGPscrlCgZAoiDzHF8tFxhghRFEUSunKysr6+rpt26FQuLm5ORxvGtm956G3vTPR08u7/RqDo3v3f+WrX/7a1780OtRNtPLs1J0vf/FrX/z8F29N3wOC35Yim0Va1pE31OzyBSWPF/AS7/a6vD7J4eQEfn11IxQJS5KkWZZN7ff+wpOvv/bTv/8fn4TQxtQ2DQNhDBBAGAAIBF4Secm2LWLakBcJAZgTCWmALwEQUgoABAhjBAgChAEGAAQIAsT+A/uZ2+2WJAljjBGmlAIAREG0bZvjuAaycZiDEBJKEGzgIkAIQQgJIQghSinHcRBCm9iEEp7nMcKmaYqiyHFcVVEBYYFQpKOjC73y2suAgwwDgACHsKEqCEKB51eWl0r5wptvXPzW1/742MSB7Pq2i5c//usf+/IXfu+nP/wRgAAgygPQHo8phpnOFzCjAZcLE6JphgKoxeOtbBFDjjLmwSjgdDJV4TGQJAFwuGKYOhazdRUAAAGhgAEIAWMAUJ7nEUSMkLa2Nl3XFxYWBI7nMQcRYpQyxjwuN6XMtm0AgG1TBgGltPESYwwBphRomgEBZoxxCKt1JZvNptM7bc2Jffv25CsFwGFBdumGlc9nJm9d3dpYX1tZ5Xk+lmiiEAAOIZ7LZrPuoPS9732nVsucPLbf63MTAGSvb3Jr6+bGliq5NtPpeFNYlFBFKdQttWKpuUqlVKutbW35gwFvwG8YRrFWgRwWAHLzkgPzek3L5DNzi/O1UiURjiY3t+677+S1GzcqtWpne2c2nRVF/tDhA6+ev9DZ1ROJhBYX5uPxRLFQvnnrxvjEGCFke3Nr38QEh3C1Xj98+ODc3entja32zo5Lly8Gff7jh4/MTk3Ho7Fgc/zC9SvDg4Onjh0vl8sej4vnecJoLBZLJpOZTEbXDEl0dHR0YIxPnTolimI6ne7t7dnaWAeEvv3xt21sbCRamvuHh+7Ozp69/4xt2+ubG29/8snlxaXk+uZT73rX0tJSrCk+PDa8k9wqFXNu2VmpVAYHBzeS24tLqx3d3ZgXFMOOxKJvXrnc1Nzc1BTL5TPdXW0iBwWMIsFIJpPHmN/e3k6nU26X0+N0Bfz+7e3tYqEkS061ruV30jLHV/JFS7dC/vDszFwwGgnGo+lszim7Tc2s12rhYPDixYvZnVQoEFrf3PQFgi3NrTs7OyMT4+P7dueqZYfLadv28uKKLIkQMtU0HW6vPxxY2Vw9evRYOp0plMqnTt9349YkQiAcDn/77/7hyJFjTtl9/ea19s6W7Wy6ptSbW9uK5aqpayOjQzMLc5qhR4KhN9+42NHb097dtbCw0N/dt7K8pqrqI489try8ghmGBCwuLsfjca/bc+7cuSNHjtSqys56cmJsfDWVLOrqoRPHZ+7NejyesbHds3fvjQ6P+Xx+QGhvf1++VHzxlZcHhoc8Hs/W1tbY2NjFixer1Wo8kVheXsYYNzU1vfnmm6Ojo8VaZbtULFi66PfkKxWeF3XV8Pt9uq5H/OFsuuDxegFGNU2taboFqOhwEkZ9Ph9jEENOAEjGvJKvtEabVV3zR0K8yyG5nbquW7qRTWc8Hk9za0u2WPKHIzMri6aE6sw0TYtDXLVeq1arAhYsw7YQMBHJFnKUMY/XSxhFGAOMXC6XaZrUJtVyBUMEGUAQhgLBnXQKUDsSCQNGAGOmYYu8w7Rsw7AIYaLgUBXdtmilXNveSQOIQ6GIYViGYQLEuX0RORjB/nDH2N5DZ58ItPUuJpM3p24Ru/pHv/uJL33sA+8+PZ5wgY3F4h/+8fc/8bt/8ZNLO1XHgC4kiBhCzkBRs5jbZznkqk0NwghhlIJSsRKOJXyhQL6SQXLtUx9/6vIP/vrH//OzwLYEgQeAEmIZlKqWhRG2CQAYYVEyLQqhQC2EsUAYBQgAgBDmbZsSaiFGECOMMQIgAYwAylAjdwcAAIfDUa/XDdNACDHGGhjNGAMAIIgoo4wxCCEDjFKqKDXGWL1eh4hRZjfqmMYJjY8DgAhhTqcbIc7nDRoWNQxbdLhQsVjMbGwgnjNti1LqkGVCyOTk5D9/+x++9uU/OHniWCaT+cY3vtGcaH7/u975l9/61pkzp8+ePWtbJoehA4A9Q32yy1muKxgjDhDAGITQ5fEohmkySAFIbm7pFhEACwUDqlqvqYogiTZCtiDlKwr7j+ulAFJAIGQQMEIJQgBD9MSTb9/e3p6ZmSaEEEIgQoxQCKHD4bAsYpomQsi27UbwBwBRCgghkuwIBAKmaVJKdV2XJAljWC6XOAh/4zd/DQBQVurlusIJvEOSQj6vz+WAjCwuLhbyJQixLMuY59x+tyzLmxvZH/7wB4uL8+95z7uRwBcNxebR1aWVa/fmAm1tdxbnvEF3rCkiu2RN1QFkokP0h4OQ54rlEoVAcMiiJBPTwIRwFMXDEYyxqqqGpuez2aA/cPfOnf379xeLRdOwQ6GQ3++funOnu7s7nU4bhjEyMlKr1Q4ePNjd3b2zk6SUnr3/THJljWhGc0vT+uZGIpFwueS7d6a7O7sAZedeeuXIgf2ra8ubya2nP/ALMzMziqKcPHkylUkrmqpp2vnz53t7+xPNLdPT0xBCl9OdKxSz2XwkGuY4xPN8JBIRRfHixYuDg4OEkBs3bqi6Njs/5/F4HA7HM888E41GJ/bueeONNw4dOlRV6ivLq2Nju7PZvGmau3btmpubC4VCANALFy54vd5UevvevXvHjh2bm5tLp9P9/f2Xr14BDIXD4a2tLadTTiQSEMKunu5SrQI47HK7eUGQZdnpdIocv3/P3kAgEA5HNU0TeX5Xd49pW9vpVDQaHRoaCgaDwUDIpiAcjvbu6nM4nOVS1e3xeTyebDZbLJXOX7pYVepVpc5xQjqddjhE0zQr1XpdN8pKNRyLLC8vu11eBuHzL77cs2tXOpPb2tke2z0+O3ePAHLgwKGZmXuGYciyvLGxNjEx0dzasra2Njo8UiwW69XawMDA1tZWrlg4fvz45OTk4FB/NBq9ePHiwYMHLWJvbG0BANLZzOLi0ujI2PXJmx6PZ3h4eGlppVqt+3y+5557LpVKLSwspVKphx56KJPL6rre0t6WyxUqlcqpU6euXr3qcrnGxsaKxeKhQ4cCgcDy8nK1WjVta21jvbm5eW5ujkG4a3x4u1CoGbrkckPMY04olisAIUopxlizTc0y/f4goAxDTtM0QkixXBJlh0VsThQaCXW5XAYAlGrVWq3m9Xqr1SoEwOVy8TyPIfL5PRjDcDS0sLKARQ5CWC+Xo4EAj7n0dtLrdokiLzqESCSUy2d0XW+QDLZtE0I4jKltizzPGMEY64ZRrdciTTHA4Z3NTWLZEEKMMaGk8U0YY7Is+3w+t9sdiUS6u7u7Orvcbg9CKJlMVio1m1CIRKcv6ArH5Eh0cN+B4w8/vvvIEQOx81fecHnEX//IL335S5994sFDUZ+wsVL41p9971Of+sp3f/jaes4KdY37O0cU7ILOkDeS4GWvjTjGc2XDqJkm7/T4osFCNQ1g+fc+9cEf/OWf/vR/fx0YOtUUjCGHgcTzCAAOcxBhC9iWTVdW1uuqQSmwiK1aOgPItinHcRjxoIFljFBmE2AzYANAIQQQNqgbwHFctVp9i4FBCEFk27Zt2w1ABwA0UnUAQAP0GWMIosbtarA3GOHG7ymjDaoDAgQgRpjHmLcJQ6pu37hxC2MeQUwhMG0iStL3v/s9s67+/he+UCwXdh+auDEzRREp5jJPv+fdkUjk05/7bCGXkwCFxHZyWFEUm5JarWbZBs9jDKBtmqqqV02bYOjxuHgMm3yi0yUyDHmJ122DYUSwUFZMk1iQUgR4wCBFjGJmA9IISqIoDg0N9Pb3LczPN67HsqxGeSIIgklsQRAAAJTaBDBAKLOJZVm6blqWpel6NBJhhBJCdF3HANYqJds2Dh88sG/vgbqiiW6HQU2jqqXWsmvL67wAvF6vrpv4rZtPTdts3MHz5y/09vYOjQwdP3XENC0bAcTjm6sb527f5sKRrWIWYhT0BJ2cIAvY0OuAR4qpU2qrmlZVFQtQhBCPMLSIreoCx7kckt/v4xD2uz21UsXQtNbW1kwhxxCs1GvJra2ujg4AAOK5YrViEvvu3buQAYxQtVq9efNmcyQmIHzv3r1yvRaMR5HAtbW0mKpWKZdcLpeqqiLHY8Ly6UwoFiUYTs/caWlJMEYWlhYfeuThXC63tLQ0ODKazuVnZmaGhoYuXX4zl8s1CIFMLgswCoRDlUoFAdiSSKh1Rbet+eUlXVF39fSsrq+9duH84OjIdmon4PV1dXReuXqjp39Qt+xXXnmlt7c3l05RYh/YO1Eq5kxNdbvkV199NdHS7PT57szMHT15miLu1p3pkd1DlWphenb62H33XZ+63dTZwTud0/Pz7V2d8eamXD7lDbrLtVpyO93b39fb25tOp91uN2SQWtTn8T7//PMCL0Vi0XQ2AzGCEC/cWzi4/1A+l5ueujM8PLyTS4seuWegz6JEkqShgcHr164mmuIDQ8PpfMHlcxFmK9VarVIJh8PhcBhxojcYdHo8vrC/c1cXJ2BKgKGZQV+wXqtGIuHJW9djiSbdsEqFXGsipqhV3agLCAiMLczNBiOB6Xt3N5ObzbHYpTfeIIgG4qFCKR+LxTq6uzZT24IgKJpqA6iqqox5o1yLBUIel5vn+UqtduP2rUgizkliqVYRBCGRSKysrAwMDBBA5hbnMI/uzk4XSkWPzzswNCiKoqrWJUno7etpbm81CfFHo2vrSZOAklojHGJY4AU5X8i6PE4bISgIpWxehpyDE0zNpAQ43B6LUZ0R0eOpanrd1JHIO2SZRzjo9xVz2WAgoCiKqqqyLFPT8EoCs3W/W3Y5pFxqRxYQByxkU0xIWyKW2d6EjFimzojldcqmpoZ8AUVREEK6ojKbcAxCBniOI4wiDnf293ZNjAGO2YCUqxVNNRDiEOIAQA6HkzFoWUQUHAF/wOv1CoJAGdUN3SHLkiSVy+XUdjqfLxBCTNMUJKccDEmRWMvoxCMf+NCZ9zzta+u+PDWt6+rbHzz79S/8zi8+fGos5KzvlP/+e6/+1he/9amv/cNCAflbR7JVkipqKsHI6QMur8lz25WiBqEnHEeSu6ZUHEL5y59993e/8pULf/3niOMJYaZhYgCYTUxDY8CGgEmSkGhJWERTrDrHixBwVUXFHGIEUIoswJkAMwQRYIgRwGwACABvoXaDFnO5XMViEQIEAKSMCbwIIQIAcg00ppTDHGPM6XQCANxuN6EEAthAeQQxY4BShhBGEHMcr2n6W2GDFzEnII5HEKG7MzPMJpZt2raNMaSAfeTXPpLP59fW1mLxeLVWG50YX1pZ2bVrVzQanVuYf+/7npYlB2RMxKAt5g67PZViybBMUZIty2rUF06Pd3Fj0wTA7w+ICJ27ej2TSTmdTgghMUweQ8QLBgVVRQeQBwBCABBgEFDbtjnMGZpOKb1y5Uo+nz99+jRgjALG8TxkgBLi9fsYAJphAghtRm3btCyrUew0iCfbtuv1eizWJEmyYRiCIJi6AZjFY/iB9z9NVN0iFmOsXq1+6XOffNsjpyq5QjGTcTukoC9omxYgpizwuq43JUI/+sFP8vnixMREqVoBHMKcQBHkRDi9mb0+vwAdDhswVVUT8WaJFzwuuV6qAEJlWfa4nX6fD1jEKUset5PDgtvlDQR8gkNKZzPN7R1bqWRrexulNJvNjo2NmMTO5/PHjx8/d+6cLMutra2rq6vNzc0dHR0ejyfgDYTDQVmWg5Gw0+kcGBiIRqPXJ2+KsqOrq6taLBHTCofD69tbTo/b0o3VhaXU9s7KyopmGpZlcaIQi8Vu3LghiFJbZ9fU1FRXV5c3ELx67fqDDz64s7Nz7dq1ffv36LqKebR///6lpSUAQMDnj0Qi/bt6IYSqprW3tzvdLq/ft7S0ZJtWPpubmpoCAFQqlXAk1tPTY5qmz+drikUxxjzmvF6voiiJRGJnZ6dYLLa0tebyhbqq9Pb2Tk5ODg0N9fb2vvLayw6nc3V1vaYqXbt6bt+Zmp27t3vPRD6fT6VSu3fvnpychBi1d3bOzc3FI9G2ltaFe3M9PT3La6tziwsdXZ1b2zsbya19Bw5cefNNj9O1e/eeS5cu9fb3uX3e82+8sWffXtu219fXjxw6sL6+PjU9vXf/vlyxUKqU29vbt7a2CqVioqX54puXLIsIgnDu3LlKtaQoyo1bkx1dnYqiFEuleFNU0dRr165Fw5G1tTVN15uaE6lUqq2tTZIkXdd9Ph/EXLVe83m8LYlEpVYVREc4HM2k0pZlKdWabdvFYjFfLjncLgFxHtkZj0Q5jpNk2ev3pTLpbDZLCNlc3/B4XD8n1qvVaiP/VVUVYxwMBu/du8fz/PDw8PLaqk2IYVs37kzvP3Gsc2gwW63IXh/AnEWoYViRSERV64IgNBqVEEJNM9xOj2kYEuaVak0QBEVR3AEfFDjDMk3DgJSlk9uyIJZKJX8g4HDKhUKB57hKseIUJMyAg5OoRQ1LxwKvqqrL4SqXy5FIJJPJCLzkcDgEQTBtK5vPGbYlOiSXy2PbttfrbXDokiRVDc0fDQMJVyrFeDwe9Afy+XwymSyXy400tsEmW7ZFGYUAGobBGBNFkTHm8/n8fj/P84SQWrVsGEahXKwpKsSCwxt2BKKRtl17T9x//KFHnOGwzuxAwPebv/rf//TrX/6l9z481BndSSnPP3vhvU9//vO/+806cTnju7K2VLB5U3Dz7pDLH6tqtmaDQDTh9Pp3ijs8r33pE+/9q9//7Pf/4IuY6A4IbFVFGIuiaNgmAMAmpiTxnoBLVevlWoXjBASooeuMMQgBARAABAFEECAIIQCMEEqsBoHcoM4dkgMAoGqqqqqUUgYYY8yyrLpS13TNMAzLtiilCKEG2w4AIJQwxmxiN7qvjaS/ESoopZZFAIQMEMIoABR5w55rN69ATGxDFXjEAAWASk453tl2Y+auzejJk/fduD71Wx//FOPQ7bszmWw+nU673R7LsnRLxQAMtDaLhOYKRYoFQoimK6ZNkSBYWNwua3ViZCq5Iwf2YYwNTRch78AY2RQwRiFKV1QLSoQiDBFjJgRE4jib2A6HAwD6nve85/d///d9Pr+i1DDClFGIUIOWaTwEjVb1W+0F29Z1XVVVmxAIYa1WK5fLXq83EolYtskAqVcrlXLh2NHD/UP9hqpLWOR5fnl5+R//5u/+4k++9q63PaKVCpsrG9QgMZ8XWgaHeAh4YoO/+au/7Wjt/NB/+xCwAGTIMG2LMSCgu6s767nMTinPMCrVqx5PIOKPtkWbXJxo6zq1bKRZHl4K+Xy6aRAMC2rNALSm17sGB6YX5/aeOHJr4S7kuZb2ttfPveYQBULY3Zl7YxNjPI/v3Lnb19O7cG92J5Uc2z2xuLyMIero7rw2dYsI2FS0nfXNQ/sPIAZee+XVAwcOJBLNyZ3tkT27VWKtbqw/cP+Zpki0Xq4M9O0qlUqiKLa0tDQlmpuamnRd37f/YLFUwRjv2bPn7vRMMBi8//77n3nmmb6+PkkQf/zjHz/48EOKpi4uL3V1dd29PdXa0tLW3fnqG+djTU2PPPTw4vxiwOuvKXWfz9fZ3lYtFaduT+q6XqvVPB5Prli+OzeHBaFUrWm62dTSXC6XFaVm6Jqh1ivF/Ob6Gs855hdW8oVCW0trJZtnhgUtYquqQxRaWlqe+dkLYX806PPN3b0jimK2kL8xebO5tWX69u3bN66Pj4/PzMw0tTZFY7G1ldX2lta29vbXXn81FoupdWVzfeOBBx6YmZlZX19/4IEHbt68CSHs6u6YnZmLRCItLS0vv/yyTdjBgwefffbZYydOCLLz1TfeeNvbH7cso1oq33/iVC5dLJerI2PDS+vLGjEiifi5Ny52dXYHvB5Vqz/8+NvKdW07lR2f2H93YTFfrgwODJua1dc30NLcvrS46nJ4MOVy25mQL9SWaC0Xigf27qvki92dXV6/Z2F5KRgOud3uuqYODA1qhl4ul48ePaorqqlqY4PDyWTyypU3Dx06kMtlXC6Xx+PZ3t7u6OiIRsOTkzcGBwd5nl9aWtq1a1c+n6/X6wMjQ3fm5weO7N+o5ku6XlVUh8Mhiny9plKbOQWBEduGrGYYkuyo1Wpeya0WazISJcZR09LqikOUvG4P0U2BQY/Dqasax3Eev48wijGuVTViMLfk4aCjr6cPQryZKRBOEGRnsVoTZK9mAb8vbFnEIpBhTnDJ28Wc5HFRBAGHKcKqpRFmY4AhxJwouYN+gKFWV7I7qWI+Twlp9NsbNDQAAELGcxgAwACTRIlSahPTtHRVVzgBiw5BlHA+m6uWisVcPrm1VSqUOcgJUPD6o85AuG147Pjbnhw9eYrz+a7P3d2uZh94/NTvf/6jv/3BRzu9gg+Cl19f/MgnvvHFv/jBFvWT0K6sLpfqqFqlsWibyxlEvMfGTugOmiKONTs+9xtPnvurr394on/p4iucgIFl2wSKnIsBDnEchADawOf2+dwBDKGLRyKyNb1s2rrAAAYQAQQAAgz9XAbTOPgPyGbBQJgQ0kjSIYA8xwu8wBhDCImiiBBCGDBAAKSEWhhhCGGDjQEAQAAhwIAhSgFGvCg4GGO1eo1SGyMIAEVYcswtLpQLRZ/Pb+g6hJBSKrtd4/v3Y4HnOM7tdGWz2fe+/xduTN6+7+zpx5542+3bUw1UhRBiADpbYnatxmyo6pYgOyHGGGPNsiuqLXkdtbqq6QoBgFk2hogxRmyb5xDRTUpAvqraABGbAQAZYxAwG1CEYSMT1zRta2tjZ2sDY0wZBRABxiilTtktSXIjzr/FOlEKGRSwwHFcA+sdkhMhlMvlIIThcFhRaqVCNhTwVEr5D33wg0qhxiEcDEf+/d9/cv3qNWaRX3zve//hb//2s5/5ZCIWWp1fY4YpibJFCCdwzz77XD6dPXrwyN4Dh1RNl2SRMCDKTorAaiqrEHszt1OqlSuappq2W3Z5na5QKBT0Bxy84HN7dMuEPDYBrekqpdTt9Vy7dbNroO/1C+ePHz9eKJfm5uYGBwcLuTzPiaLsmJub9fh9e3dPzM3e8/v9brfz3LlzbR2dlmVdvnqlZ6jf6XFnMrnB/v5703crpeqBAweuX79e19Sh0bGr127UasqJ46deeuklj8t9/Oix2dlZ3VA3NjZWV1clSdrc3lHq2sbGBqWU2OzmzVuCIOi6ntzeOnr06O3bt7dTO0ePHn355ZcppcPDwzdu3BgaGsrn81NTU8dOHF9cXHzhhRcef/zxC5cu+v3+7u7ujfW1PRO7d4+OMcbC4XClrlSr1WNHj5eqNZfb3dPTu7KyEg6HW1tb6/Wq0+Vobm7STSMSiYiClNrZkXihJdEccLtFjDI7Ka/bbermwMBAI9mvqUqpkLd0o7O7azO50dbRfvr06bv3Zk8/cKZWq6Uzqba2tlR6p1DM7d67R1XVRCIRjYVv3ry5q7tn9+jY6vJywOdXVTWXyzW1NPO8UMhlTp087vf6r169fvbs2ZWVlWKx+NBDD128eDEUDhJKr1+/efLkyebm5qmpqdb2FsjhfLEwMjz6xhtvFIvF7u7ul19+MRaL+f2+a9euJBIJ2e2amZlpbW29fPmyYRh79+6dnp7GGPf19c3OzkqS1N3ROXn9xokTJwAAO9vb999/X6lUWFpd6erufumll0SB6+/vn7p12yFKQ0ODd+/eHRwcbGlpuXPnzsDAgKqqGxsbQ0NDKysrtm0/8MADCwsLuq739vbeu3fP4XCk0ulSqWJCVrBUFPTNrq9JbrdhmpiDECNBEIrZTCjgU3XF6fNwkog4rNum7HKKPF8qFKPhCIZIURRFUVwuV11V/cEA5rlavZ7JZRmh1Ca2YQqcaJp2IpGo61p7bw9yiFuZjEYth8tp2cy2GKWAMej3BeqqjgU+GAold7YVTddNw+VyMQQpBAxCJPCc1zmwdxxAEIqGAqFgo6ro6elpbW2VREFV1UIhZ5qmbugIIgigZVsA0lKp1EhQHA6H7JSIZTsEkVo0FAoN9A01xePZbNayLEpsIDqQyy0EQoH2rr49+46cORtt79pIZwul/JlTx776pc9/8tffd2Ki1zbBq6/d+egn/ujLX//rWwspKdgt+NtuL2WSRaOmQckTEf1hHaGyUmppDfzeZz443OT4rfe9LXX9EmAWYKyRZCNAbVPlEBMh+Idvfa2eXIeIQmY7ZJFQEwIGKWAMMoYAABBgiBHECGNsGAb6j6aIZVlulxtjrGpqI54BAGRZFgWxEQkghG9R7Qg39DMMMA5zjZf/WT3J8zzP8y6XbNu2YWqEEAQFOV+qT169DQikFkUAY4SdLo9m69/7wb8sz8/27er0ez2qVk00h3v7uj76Wx+7PTONRZExBhnCAOzqioocVauqyDkbahNKqSBIJuLvLqX9Xn9HtJ1qOs+YQ+QppFiUTItgRiVBKJU1lQAscIRYECHSaDkD1LgwUeT9fv/29raiKAAACiiDAHPcf7QUEGOwwcLzmAMA2LYNKGuUMBhjQKjX621UuG0trYV81ut0ZzKpUyeOtTTHFEUr1eo6AS+99FJ3e8fVi1eyO+mQX/7m17/63X/822goVq1WNcuUvO6FxcWLL7823Df0sY9/tHNwUNEY5pyKrhEObubK0BPI1ytlo55RK3lDSRayDrer0QPkeV7RVOwQi1q9rNQs29ZVldqks7NzdnY2FgyvL60ImItEItvb221tbaFQKJfLHTlypFDIra+v9/X1SZIkCILP56lUSrquj4yMTM/MKqbe0tE+eftOS1NrLBK9dPnN9l29WBCvXL3a1dHplp337t07efK+ZHJ7YWHhwQcfbAhgT5w4cefOHZfLNTw8nE5nQ6GIqqo8z3d0dDicsqqqbre7u7s7EAqtrq+1NreYql4qlfr7++u6ARHq69l199ZUKBTq6d312oXzp07fhwCcn5vz+/23bt3KZrPHjh3b3t72+XyJROLll19uZJ0zMzMnTpwwTTOTyYwOj6R2MgTCY/ed3Nxc93qce0f3baxsSaKIMDRNPRIOMxNktzMhnzcaCxMEbMai0TiGSJIEm9Gapm6lk1BAt6YmJUno7+3b2dxIRKPBYJAQyxf0pTLp7Z2dYChALTI9NQ0os3SDUtuy7bn5pVQ6K3A4vZ00VEPAUj6fF0SOQ6iYL7R3dUKeiyTiw7vHFpeXNE07uP9AdjtlqXpTJJZJp3fv3t3R3XPhjTe6uzo2V5eoqXW1Ny8tzuWzKZdTPH/h1fb2Zo/XeXt6snegp1IvbaWTXb1dy+srq6srw8PDzz33XLVcHOnZNXnpCkGgq3/Xcy8+Nzo8YqjajSuX9+0eFwXu9u3bh44eWVtb8/v9Pp/PMIx0Oj0+Pm5Rkmht0Qx9O7UTDPp3dpIN6Lcsq6urq1otx5qbDAwmTh5RRZgzFc7pUAyTk0SdWC6Xq1QoOt0uwuxUNsU5xQo1a4gYGPhCgUKh0CiCAYJ1S3WFvTVqqsz2hgLVckXmBKhb0WCoVquYxNzOpzYLqQrV/M0xTzRUqJQN2+J53uFwSJLE87haLok8tnTD7ZARBcSwIEOWZXE8zwuCSW2F2KeffNvQyWMA2ohDgtMVijWFIxEIGaV2uVp0u51en9u09HRmp1wt1pRKXak2nlKOQ7IsIQBEng94fa2t7R3duzweHwPMIiZjVrFaqKn1UrVq2IxxEu/0uqNxb7x9aP99R848GesZXkrltvKZAwfHP/fxD//2Lz1x/0hLFIPb11a+8b9/+Cuf//qPry06OiaIvz2v0FLNEJwB0RWo2bhsmsBlf+ADD3zksSN/8skPAUA4CImuANvCEPCYVdcXf+HEgX/8nc/8ycd/HRATYKRpmiAINjUApMRmDZiCAFrEZoBRZgsixwBpaBx5nqeMiqJYq9V0Q2+QM43cvNFlRRARShBChJKGPhICaBObAYgR3+BkGh0LxiAAAAEoCgLPcQhCVDU1Atjs3RnAYIPTMUyDUfvJJ9756d/+1OS164iBiYm992bnm2NRj+x4/wd+4T3vff8//t3fUYsInMAACLqAAC1LN+uqgRCnKDUeYdO0fdH44tpWWTVtantFabSnRymVqG3phqHbBCEEAaoTUFSpDQDmIAWAAQgAsKnN8ajRUxUEIRAIKIrSiOSUUkoBA8Dtdjf0QD8vdgTMIQAbomkAAGU2z/PEZhjzlmnruu50iIBakXAQIfCedz+VzZZESQYc+t73/zUWi/k9XgTstZXlH//7vwEAvv3tbyuK4vZ6dGo7PM7vfuefZm5O7j908LG3PS5JDsuwRY7HPKcwsJhMdg30VZRqtV4FCDIItre3JUHUNM0yTAbATjZjUWoDhnmOMdaQ8SAGwuGwUq/zmEMINf6WotYkXshmszziTWIalpnNZmu1mmEYjFBBEi3LKhQK28nUVnrH4Xa5PO5ytRKMxlY317d2ticmJrSKYlTUeDhSqVQ8Xm97R+cbFy7xPB8Nha9du3bfffelt5NXr15tyEx5np+YmJicnEQI+YPBe/fu1Wo1U9czqTQAIJ5oMm17bmEeYGSaZqlU6WhrFzgun811tLXduHED81xrW1smm2uKJ0xiP/PMMy0tLcmNdU1RTxw/trq0aJnmwEDf8888G4lE2tra/u7vv93b3yc6nT/40Y/Hdo/W6/XJWzfGx0eXl5d5nu9q71pfX29qinV2tl+6fNmgZktHu2Ya8Xjc5/NVq9Vjx45l85lCpTw42B+Nhl2ys1wuIgQAgpVSafbu3a6uLoBBtpBta2sr5gu7urrHBofVuiLLssvlGh+f6Ojo8HjdxUJ+ZzMZ8Pm30ymXyxUMh+/OzhBCtraSaq2+vr5eLpcFQajVaqLoQAitra11dXWaplmtVkdHR5PJpNvtTsTjLpcrGo2Ojo5yAt/V1RWJRGRZRghZhIyOjrpcrqWlJYxxvDmxub21e894JBIhlt3T3dmaaK5UKg/cf8brcQf9vonx3fl8Vtc1wSEUCjlVUy5evOjz+W7duhWLxXRdX19fbwx/NOL9/v37ZVn2utzpVKpcKh09fOTm9WuBcCiZy4Q7W28sztUMgwGEBd4wNKfTSUzLUBViWC6HrGlag5w1TdMwDIiRpmkix0ciEUqpbpr1et3hcJQKxYasS5blfDYTjYUtZtSJ6gi70rXcpcnLBjA8PvfG+jrCDCJarZYlSbItgwPMKUvVarW1tbVSqYiiiHnBsEwGAUWYdzp7RkeAyFUq5WRyiwHIAGSMmZYOIFUUpVItqapar9cJIcVisUEo27YtCJwkSRziVF21CIE8BzBPTBtjQVPVerXm8rhsQuqaliuUFhZWVpY3dN0GnOAMBh2+gDfesvvwqQfe/q6+PfuWd3aWN9YGh3o/9/Hf+sInf33/SJdTBKvrxT/6xnd/87e/+MxrV4G3WW7qSdeJwiTOFzJ5jghQ0YsPntnXHpV+/MdfAkRxCJhnRm17+a+/8vkPv/P0nnbf//7ixxYuvvTFX3wKEN0pi5BagBDIqGEYlXINIUwBQwg1EvMG09DIuxvdUYxwKBTSdR0jTAihjDLAIICU0QamQ/BWCt9AvAaHA/7rBmGDSUEQQIx4DnOIYeoJuF956VnALMYsBmxB4HmEtUr1H/76292t7QBwCAr9uwZ3lrZkwWPYZGZ+7o1Xzk1dv8EAQgA4AehvbylVK+VKTYQ45PEBAGwAFcPgZZdTFubnlsyK6oHYg6EEIMY875AtBgwANIw3ynkLEAoIAhwECAEOIwwBbCgdFUXJl4qyLFu20Wg4NC4vEAgCgDAvWDblOIHj+IZISJZljuNMYjc6UY3zG+x8sVg0TK25OZFKb7/nve+ORgNVTXH5PNmc+vrFS5FE2LL1ro6OWCRy8/Y12e04cuxotpDnBEwxm5y5LTgdLrfb7/eaasXFQ9skNgFQluY3N6qG0tnVbquqVa2LmPe63bpuWJZl2zaC0CnJuqZxAi+65EhT3OX0VAqFRx5+OFvIj07ssWwDA3j02PFiuSLL0u7x0dnpWU0zPB53pV7de2D/2NjuSrkYjYUTiYRN6OF9B0aGhxOtLbuGBm7fm5E97sPHj3kDfgYARqiSK4Rc3pZYczad80ci29m0oigDff2iKPq9vtu3JkdGRsbHx6enp4JBf1t7y7PP/mx89xjP87du3YrH45Vy2ajUezu6Jm/fshEwqF1VlapS9/h9al3J53LlfMlSDRkLQX+gbmgWYr5wuFhVwqGo2+Xled42rUq5kNradMsyJBYENBAI5PP5zc3k+MSehaXlbDY7MTGxvL7hCQVG94zcvH29q72DmKxcqHV2dE/PTteM2qGTx2qanspkjh8/eWvqtmEYQ/0DVy5eGhgYGBoaunLlitfrXVldLpVKIxPjpm1ZljU6PDI9PSXL8okTJy5cuNDR0VHMl86//np3dzcDtFKrZvO5zc2kqWsdba3NTU1LSysDA/3pbJowO9GaKBbLzKQBr29nfd0liUqtfvvOtGpavCyrpq6aaqVe0RXVJXsJ4ATZzZBw+coNUXZni+XF5fV4rGVudrGYK/cPjiwsLK8urLl42VTNgYGhQCCQyWWbmpo0TVtdXXU4HEvzC+nNZD6dyuyk0qkUxnB9fb2hJDn/xoWhoaHBvv5sNtvb2+tyuRYWFnbt2rW+vl6r1bo6Os+fP59KpYaGhu7cuTMxMSHxwisvPH//8ZPVTH5pfuHo2dO7jx1aSW6HYnFVVUVRVFU9Eoq6kcBbtgixkxOcNhRN6hQlCKHb7aTMZowUclkHL+rVugNyZqUe9Pp8Pp8OCHaJ+/bv5gUQbgmfefz0/U8+fPyh+z7+hY+37WoGzGpORJNbK04Xz/GAWJrEQcCscqXiCfhN24I8zpVKWBQgh50+DxC4tr6eeFenoekcwg5OyqWz5WKJUAtQwhhpiscCAV+9XhUELpGId7a3OxwihAxCJooihJAw4pBcBGMLYAIA5DmlWpVFwet21mo1xTByhZJT9rU3d/R374KUVCp5wnSKbdnrxG63I9LSufvoibe/v3V8f57Au8tr8eb4//itX/+TL3zqqfv3dQXQzmrpW//rh7/0mT/45g/OrZUh8MSdLS11jOoWoIKUVLO/8OF3XH7mH5/5yqevf/8f/uxzn/j4ex588wd/9oEHdr/3od0evvirv/jw8oVXPvX4mUs//i6mNs9hRJFDcEiSXKorFgAYcow0hC4II4wRhogBSBmDjXlMn9eXy+cacM8YY4AhiCDACHKWRWyLaprBYYEyQNnPIR39x3DsWxsjADAIKAQAIUXTEEKFfB5YJs9xHOIYYLZtu93uj3zowzev37h58dLY0MjO5vZzP3sGAPqOJ9/1V3/zt0MjQ/sOH6CAGZblBKC3rdk2jXq9DigzFU23dAChaRGG8MpmcWSw3+OSI0HXA/uHRQx0XbdtyhijECBZzlYqBEDTshDAGCDCCATQtC2O4yzLCoVC8UTTZnKLMcYBhDkOYkQZkCSpAd8NiaRtmJChhnsAAwRj2HgmKKW2Rd/SkgKo1pVAIMAhHA0FH3v0wUK+Ijqcpg2+8pWvjO8eK1VLEMJyqYoQWFtf+W8f+EVbM5wOWXaIhUL+Z8/+2Ot2vu+9TzklibPMsGmGDUuklACayRYQ5hPxplq1XCjmd7IZmxJGodvrARDWq0o4GNE1BSHgkF35fL4l0fziM890tXdNXr8R9PmDweA/f+efOIg4Ds/MTN933/2GYayuro6Nj9y9O/vvP/pJf3///Pz8TiYdjUanbt/WlXpLovn5559v7+zAPPfXf/OXjRHTl19+eXxkOBgMXrlypaun+/bt21tbW0ePHLlz687SwqLb7a5Wq88888z6xuqhQ4fu3Llz69atgYGBlZWV1dXV+06cWlpaamlpCQSDhWLx1KlTjFJdUQ8eOOAQpWKxuP/ggURzc0dnJ4NgeuZuW3OLbVqFXH5kcMgytGKxyHHchQsXQqHQwMBQpVLxer21ar1eqXZ3dbhkpyAIpUIRUEYse2trKxqNViuVqampsbGxtbU1QKlNTMhYPB5va2vb2dnhOVFT1EuXLvUO9Ju2de3qjeHh4VKp8tprryUSLdPT07FYrKun+9/+/UcWo+FweGFhYWxszNT0l1965dHHHt/a3CwXSw8//PD8/DzHcaOjo2tra+Fw+NSpU9l0OhwO7987ceXKlfbOjmAkvL297fd4MUQcRE8++aSpG/l8/v7779/Z2ZFlORAKLy2t9PT0uN3uq1ev7tmzx7DMnUxm/8HD9WrVNM2unu4r1652dXVZljVzZ/rs/WcYY9vb20888cTdO9Nra2sTExOzs3OKovQO9t+6Mz0wMNDW3DI7Ozuxby9lbGVtdXB4SFXVfDb35BNvn5uZy+Vyu7q6C4VCsVgcHx+/devWwX37q+XK/Pz86OioaZo//OEPOzs7b1y9JnD40YcfeeWlFzvbO06duu8nzz4zMD5CRSGdz2uGyfMij7lKuSxgziGIpqEJHM8I5QBCCKm6XqyUG9Iyl8tFCJFFyeN02bbtEERdUQVJ2kxtzy3MW8Q0qdk12Hfrzq3NTDIQDe7kMgihaDQajsU3t3ecHnejXHaIkiRJtVoNYxwIBEzbKhaLPM9rlgll8ZF3vZNhvJPN7aQy5XLZsojD4YQMYIwFTiCUiKLY3NQcCUUkQdIMjRDCcZzT6SwUCmtra5VKrVKrNOp7g5gAQo5DlBDE8Y3h1b6BwUQs7vd4LVXjAdJVpVarVGtlVVNtAEWnW/QG3NHE6METD7/7fftPn9kuladm71qG+rGPfOjLv/O59z52uCXuSKeq//TPL332f3zjf/3Vv0zOJSPdo8DTnLckhXPUbOPjv/XhS6/88G/+5H+uT1984EDvlz/z3/tavUTPmXo2HOQ++NQJLTnz9U//6m++4+HX//W7QKlykMkilgUeAWhZJgKQMtpIzBlg/0feTRl1uVyVSqWRrb8F3gg1uq+CINTr9cZ08c/f/b83iBAAGEAIGEIut8/p8ia3d6av3RBlp66pxLIhhxlCum2trK2MDQ/ptbpLknwBNyBWejv5z9/5J3fIu53aQhC5eZ4HoK81YdRKulZTqjUs8BQwxgFRFHVFz+WLOgWMB4oFpte3CbVdTgdiAGOMMRYkSbdsCxCGMAMQAIQb+nyMAXxL60MICUVChq5bttWoZQAAkijJsowx5jgBIx4yBCizLasB9xyPG5U1hBhjTACTJBlCWK/XIcb1el0p5z/wvvc4nFI2V/r4xz50/6mTX/3qV9s7ujq7d7l9/lKhPHn9xshAf1MoTGuGSDBk4JVXnlPVYks09I2vfcNB6SmXcz8CAcvkKNhJ5wGW6nW1p6cnmc8ILplBzBDULNNmVBAEEXFeh1MpVQChgIDNtY2WeNPS3XseSV5ZWZmZmdk/sWdlcalSKvmDgdcvXBAE4b6TJ29cvipy/Pj4+LXJm0Njoy6X69qVq2O7x9bX18+//Op9hw+ntjbn783t37M3s7O9PD/30ANnk7m0yoyjp4+/+PILPZ0dowNDk9dvdHV1hcPhxfmFkaHhRx56mJiWx+0cGxvp6+vjOK61tdUlO19//fXjx48bxN4p5U8/fHb2zt3STqa/o/ve5NRob39fd8+r5193h/x3F+egJBw/e//MwpyHlwbbu//9+//S29Ndr5Y1XTl935lytXbl6rWO7l6bwWy+gDhhaXFFloRIwOeS+EQ0JAsCR0itUKrnKwNdA6nNlCjwuq5ASCRJWF9am7o+nQjFi+m0x+Vsb28tliqaaZmmPT+7IHD8gX37lGotFAibtpXOZs4+9KDD5bxzd3r37t3nXjmHMT558uTi4mJbW1tvX8+5c+c8Hg+CeGVlZe/eiXw++4Mf/KirsyeZTK6trT344IOzM/cW5xYP7N1XKucdLpGXxGeefX5gaDgajV+7cnn3+Gi5kE9tJnu7+2fuzuWKhZGRoReffUYW+EAo8PyLz3Ukmu1avVjMd+3qunbjqmXrkXDg1u0bkUQ43p544ZUXuns6AaEXXj032NvPGFxLpQ4cP7qVTGqa9sDDj9yZm4USv//wkYuXr7S2tDVF4tfevBwOhSKR2OLi4u7RMa/Xu7Cw0NKUuHPnjmmae/funZy8bZr2A/efmZub6+rqQgi9fumNif375hbmbty4vnt0AjAur1S3ilmny6tUNLdD1mpVCgHFMBAKJne2HH5XjepQ4sJNEcaYIAiEkGq9xklCzdS289looilfLAuCJCA+4PYbFinXVYClTKbk8wUmdu/VdKbpgBdkjhf90ebugdH17azTG+AEWddsDkBkU71adzucCEDbJhIvEEZ9TbFg3y4oiNF4U1NbS09fX3NLi8MhQ4xM01RVxdB0xiAAiDBmEcLzImOwVlPK5aogSD5foJAvaZoBLMYYEbGgqDXeIREICCMMAowxB6BlasQyGbGobQOGqQmBATPJzPr6Sjq/w/FYkmVOcjncoeb+wac+8pEz73o39geee+UVpV5+5OyJ7/z5H3303Wd2NzmQBl587u5HfuPPf+0z37ybpTgxqvJNC8lqSq298wNvf/Ltx379w287eXhQq2eq1WKpXnZF/A6/0LMr+M6H9jw02go27nz94x/6pQeP/esf/155dU7iAM8IZgwAACH4P5C5UZ1AAAkhDsnh9XqTySQEEADWgLtGB5FSGg6HG64M8OdBgv2XCMEYAxD8fEe1miIIgm3bd2dnAGWGYfC8YFkWZdQi9vDY8K1707G2pl/96G+8+xeeTm1tjg6N3puZ/dM//9Z2dgcwpiuGyEBHk8sh4EopzwCRJMnncpuqBiCVXE7F1JPZig0AFIFJiW3oyLIwIYARAphpU0JRpaqIWGKENtqhAFBKKYSI4zhimz6/3+FwOBwOgeMBQgChxpBXw+6jEdwaLVbGGM/zHIcbdjQNDTJjEDCkaZrD4SiVKrLgtAz7ztStveNjRw4fVKrquVdf+9af/umho0emp6evvnl1bHjk4OFD5WKpkEm/953vKOeLkuBgEOwfHw/wwg//8Kt3fvivv7j3wMGOtkf37W3lOQcB27lirq7wDokB0N/fXyiXMM/VajUAKADUKciIQI6QWDBczOVDoZDL5SqVStSyMUQdHV29PbsQQvsm9gR9QUbA8MhgR0fHhQsXDhw4RKm9sDh38ODBnZ2dWq02Nj66vLzcFI/t3zuh1pWhgcGWRJyDoL2ltbO9Y2FhTvY4bcCSOzv9Q4OGYczfmzt69GilVE7t7Dz22GMzMzO1enXfvj2vvvoqpbRWq9ydmVYUpampaWRkRJZlAEBza9tLL73kdrtFXmjoQ27dnCzlS93d3ZcuX25qagoGg+fPn/d6vbIsT9269dRTTy3MzzmdzqH+gfPnzoXD4d179k5OTra1te3evWd7ezsej+fzeUkQ21vblFrdNo321jZIaCwShRCrqjo8MtLU0mRYpiiKkUjM6/ZUiiWHKDJCEQPUJhDg/oFBj8+vKxoCmBDS3Nri9/sN20qlUrZhdnV11ev1RCKBMT85OanrerFUWlxcbGlpsSzL6XZ5PB5Fqbndzs7OrlwuzxjRLf3cudcHBgZ4zM3Pze3bt8/tdm5ubuzevXt+brFarfb29u5sJYP+UGdn5+rqalNTE8a4Xq/v27Nna2sLALB//37LstxudzAYLFcrw4MDoigmd7bbOjo2k1u5XKatrWV9YwNC2N8/MD09Y1MqCMKzzz9n27YoO27emkQIxxMtV29cb2trMwwDItTT07uzs6PWlXg83lA09fX1GYbBcVwoFKpUKuFwuCG3aIrFEUKqrrV0tEKBizY32Yz6g+FUvjh8cD9xOXK1miDLVaUeiIQhx1dVLZ3PBWORqqYABLPZbDaVDgaDlm07PW6Hw8EY83g8ssuZLeQ5gZdluVapirxUq6sclpbuLa/MLZ0+erI91lzPlqlqef1BividVPbO9GwwFK1UFWIDDgo85L1Ol1t26oomYAEBqKsaJeD0Y48CSSAASqLkcMgmsWybGKapKTrG2DTNer1eKpWq9SqCiFJqGIbACW63WxQcSl1zu7w93T2xSFTkcTGbsSwFQJsyk+c5BBGCnAAFDLAoiIRYgsNh2jaEuFCuVMqK0+nu7OyKhiKVasW2KYGI9/ic4ZgtyqHOrkff9/QTT39A8PmXN9dvXLt6bN/EJ37lw48/dKynO+ISwI3Ly5/53Fc/9YWv3VspOcJdFSbZktza0caAqVsqEnlXwMcQ1ojl8rkRz5oT4fsOjr77wUNPPbhPNrf/+U+/9MuPnvzjj/+KntrECBiGbptWYxAXvKVlhI0snjLKczxllOM4j8eTzqQZYxhhhJBlWQ1xZAP0ZFlWVOXnVlr/R/rfgHjGAGMMOWUpl8nyPDc3dw/YFsfzNqCY5zgAe7q6JY8rp9RMDFaSW1/4/P/40pe+BAAYn9g9OjKYSDQxxmRJRAxwAPTuai9X8rlSEVJoVxS/IGuaBkW8U8qLAU8BgJvLmy6Ph0fQwaCTQxgwCpgFqA1ApaxBxgPGIKAC5i1iQYwJJbZlcRzn8/pCgVDDC4wxBsFbbjo+v7+R2huGYZo2xwkY45+T7ByHJEnyeDy6aWiahjHmELYNEwA2NjzS0tSyPL/wqY/9Jg/B4vzq9777/Wq1OjExMTYwNHtndmc73T84UC8V3vnE2zheVAlw+Pw//t4P/uL9H770pS8fzG/3VNPvfOo9v/hvP3rHwROdANgA3NlYdgcD9bqKAIyFI7Ozs26vp1ouy4IoIQ6YNqIAIxCLNVXq9ea2Fn84tGuwf3V9rVapOh2ybhmpbNowrKmpqWgwkM2k6jWlVK5ACBOJeCKWsHQrEAiEQiGim9vrm4yxxaV5yza8bs+1S5cXZmf9Pq8gCDVVUXRlavo2xyHMoa5dXfdmZhFC4XD4+pWr3d2d1Wr15ZdfHhsbwRzMZDKHDh2qVEqmafr9/tdff93SdLNSd3JiR1enNxy0EbAgc4UDdV3T60pXc7tbcGwsLrc3NSMAc6Vy58DArenp1o6uzs7O27fvnD59ytB0XalN7B5fXpjPZ1P333ff7cnJtrY2t8d77cbN/v7BWKxpaWmpf3BANZTV9aWOns7rk5OLS6t9AyNzS8uAh01t8fml+XA0GggE5u8teBxur9M1O3evtb0tncum0unOnu7JqdvJnVQ4ELxzYxLYRBCEYrXSPzTIcVwgEBgdHVXUutPtZABk87m1tQ1RdNydmfb4Pb19Q7lCBXKI4/Dw8DCGHIOUMjZzZ4babFdXd9DraWlu2r9nb7lcxbzD5XHruj46OmzqmkNwqKqeKRSHR8dVVV1fX88U8qLTlS+WQ4GwpuihQDAcjyV3tolqWHVDrSu6aVQUJdbarFPb4/FpFaU93op5IVcsul0ep+Solcouh6u9o8OGzBsKCC5Zcjii0ajb7d3Y2GrQKYFAYHBwUJIkjuOGBgYdomTbdjAYLJfLHr/P7ffdW17UDXNXf/+bN691DvfniKVJYs5UqIOvEVrRza18xhcJQF6gAHEIB/0BqhlOUdrYWDMtvVQpmqZuaGq9UvY53cwmouzYTqck2VEqlQKBEKagzR+ZfOn8P37rr779J3/+2o+fjbp9MzNzV2/cojYDFBoWEXiZmVTCPLVsQohlWQ6Xk5dETdM4jjNsyxsJ7iQ3NUUxdAMByGPe1HSR4yGE1Wq9XldLpYpSqzNCNc2gBDgdbssmsuTyeQOxWFPAHyKUWZZFTCPsc0sc0JVaNrOjqDXTsCCFxAK2SVTNQJLEOOwOBfzhUHNLS2fvrkhTM0aiadhKra5odc1QdzLbmmnyssfhjzPZG+sdett/+9Aj7/2gr7nrxvTdbCH32NvO/N7vfPRzv/a+Pd1hWKaT11Y+94Vvfvr3/tfNxYLGhW3BV1bgZl7jPFHo9IpevyjIlbKKBH8g0UklMdYWeOC+gd/64Nlfftu+kRC9+cN/+ui7H1+7O+mQ+YbiQ9c10zRsYv/n7JsBBiE0TdPv8zdoBgCAbds8L0KIGYO6riOEMMI8zyPEQYj/n/wMhKzhc4BK6VIi0RJIJF48fwFAKPDYtHQMELHs1tbWxc2Nl8+fowBM37nlcbqOHj1WrlVbWhIYca+/9noj/TdUnQegq7V5e3OzUqlgzIu8wEPEcVi39V0D/TVNvfTmjVIu55OdMsYYEMyoKHACgk6O4xjUDFO37UbkIZTwWGzogd4Ca8AaRDxjDL8VmohtWbLsalyeIAiiyDNITdOEEIqiJPAiAKhUKmGeC4fDsiybpmmapm1apqaLgmARohrqcG/PIw8e1yzw7Msvj42Pv3HhQjq90xSN9ff3V8uVZ3/2U6ckHjt5wqpUA5Vqjw3S128c6ezpdkpP/9Wfh06f+pMP/PJGKmsACDiQq1frpqmbFrBoPBge6OtfX13z+v2ZTM4iNuJwQ/Nr2IboEFOpVDgUun796pPvfAfP83ML85FIxDCMQrX87qeeunTpkqZpo6Pjt2/fBgC4XK5XXnmF47j5+flbt26FwoH2rs4btyYPHDycSqXy+fzb3/727u7uu3fvegN+zHO6aew7eGBnZ0dT1JnpuwyCcrm6ubkZbYrXK3UM0cGDB9PpdK2qjI6Ozs7O9vcPlsvlc+fOjY9NEJNsbax3dnQsLS0BHiOEFpaXIEKxREzXdQFxAV8w6A/17+rv6+kLhIKE0WA4Uq1Wy7WqRUgymSyVCgsLC+mdZFNTHGO8vr569OjRtbW15dWVI4eP3Z6aLhaLHR0dr1+44A36evt3JVPJ3r6+1vaO1y+cHx8fN01zbW1tbPf4dmqnUq6eOXNme3vbsqyRkZHXX389kUgAAC5dutTa2pzLZe/cnnrXk++GDCwvr0iS9Nxzz1Sr5Wg0+tJLLzlkJy+Im8mtwcHhRCKxsDA3MNCXSqUuX74ca4ojhCqVimmamUzK7XY3NzfX6/XGBNyFSxc3N9cXl+arpXI8Hp2fv1dXahABjuMIIXv27FlbWysUCpFw1Ofx+3z+TCaj1GrxeFRV1UKh2NnZadukubk1FApRAAOBUCAUfO211wOh4M5OsrUpHg2GVEP3BvxdHZ1zs/OKojQ1xWZnZiCE2Xzu+eefHxgY2tjYunLl2tDQkFrXbly7LgjC9evXTdN0u90XL150OBwdHW0zM9M9XZ2lfOHq1atnzpwplUqzs7OPPf742k5yM51+8O1vZ07HVrVARB6Koj8YWNvYdHncDfpiZyvZ0dEBAAgHwrLDJXAipUAUHQ7JWSwWJUnOZDLhaIQxJkiiN+B3Ot3RYKg11mwVFdnGIdGZ204F/KHR8d0chyggO+kU4wBAEHKYl0SEEM/zaq0GCMUYK4pmmjZAgtcXwDwvCYJNbU1XXW5Xo7x2Op2yLDc3N7e1tXm9XtkhQcQ0QzFMjTLaUA1SRgEAkKH0drpcqtaqdWrZDYktsQxIGcfxPMI8zwMALMYoQBwnSJJsEstiRNdNXTf9/mAmk9FNzaJkdXV9ZW3DpoxzeLDbayAx2jnwxNMf/PDHPzN04GC6mN/O7OzZN/qlz3/6i5/5laPDnW4BL9zd/NY3v/35L/zRz166pCC3v300b4oG8uhMhJwTck7RHawZVrg1YTIjk920zfx9R0c/9N7H3vvIUbdV+swvv/v5f/xLl4xcAsYINBwULMtqoDuCkFICABAFkTIaDUcQgKpS5zgEmNVwnZQkqREJeF5EEGGI/l/gTn/+E115/crFizd+9TNfmpxfWVleFTBHTQ0CijieMnTw+Mn+kaFadvv973q7Wqutrq5SQCq1aqK5M7mVtTUDQsY7MAUg5vWIlGhVzaZQJ8SgJo+gwJDX5eUoONDX2yqLbmIjQweAIg5hRpwclmzmFsRqvUp4ZDDCIAQQUcYQAwwwgBhAEAKAG9bGkCLAbEsHDf8EzHFYYoRBxkzbYJA6nA7CgGUyagEMsCzLlmUoWp0TscMpC5Kom5plG063bAACeWiotSeffEIMij98+cJ2aqe9pcmyrMWl+auXLh4/duTjn/zEhcuXLEsXi6VPDwy8x+/tDPnf+9Xf3/ul//nsZz73hx/+8Lt++YO//bd/fw8w6nWaxF7d2vGHm2TJXUjlYj5vcyyysZUUPJ6iVocyL3pkKCHkQOubK7FQcGZqqqOj7Xv/+j2nz+UL+M+98mosFnP5vD974bnDx49RSufvzT384EMcwqtLqwf27bcME1J46vR96ztJxTJO3Xf/M88815RoGRoZfu2N87pt9fUP3Lx5q6WlRRDF2bmZAwf26zUlEYknEi3t3T3BSNSy7NXVtXymkMvkbZtubW2rqt6aaJ26fcfj9h48ePDqpTcdorR3797bU1Mtba1el3t6amp0YKhUzFcqpZaWxK2p2/W6Gg5H37xyTdd1Q9HmZu6ODA2apr65udna1iw4JF/APzQ0VKpUkjvbokMq16qZfKa1vTPR2lZVlHAsKssyLwr7Dh+0GVjf2vT7/eubG7VabVdP1+zdmWgg0tfVv72dam5uDYTCF9+8HAj7bducn509eviIoWqiwB09eKCYzXS0tbznPe95/vnnOU4YGRmZm509cewoBOz6tauPPvpopVJDvHD/2Qd3MmnLMvoH+mbv3uvt2TW2e3Tm3l3JIQ8MDm1vJ/0Bn+wQ705PdXd1Fov57Wz64KFDDJFKpRAK+1aX5yd2j0aigbnZGUYpY2Rq6taDZ0+XK8WNjY3Wlo7k2pbP5d2ze/zK1Tebu1p1y7x55XpnS1uxWMwUcuMTe9c2tjRFf/jBszvbW26P0+EQq9Wyx+lyy+6LFy5NjO/meX5qaqq/v3dzfa2Uy33g6V+4duVqIt48PDAyfXvW5/HGo01rK+snjp10uVyTt24+/MiDosTfm5sdGh68OXkDAzCya+DauTc6m1t6u7t/9O8/kEUp5PRcvPDGyNEDS8U0cQpEwKZBo+FYqV42LJ0Q4vf7q6omyE5RdJQKZZfDZdvUsqlmWrzkoJQGfX4ecx6Ph+P5sqrWib20uVkqVwXCG3nVUgyf05va3i4UchYw/RG/4OTXUxuemL+iVwCCnIBNYmq64pREaFNdNxHktZLidAU4jiPMJpZtaGqlkoccQRg7ZJc/EHK5vAAA0zR1XcMQKLVquVjIZXdUpUKoRYhFCEEcbm3vQpyjWrMsgvyBaGdHt8PpVPSaqpVrWhkAgiHgIMIAMYooZQhARiyexw6npOqaLLsqpSqzSSwS3tXZaRm6qWs8J3KCk3f6LM4pxtrGzzx65NF3Ng+MbpSKy8n13oGO3/vCp373E7/50L6RJoe0sVL633/74q9//mv/619fyAGvI9oZaOqqGhjwTgoR4rmaqvmiCcET1BmfrlYEn3j4+PiTD+zfHWDf++InPvHIiVsv/VQUMeYRJYDjRc00LGpb1EQIMsAoI4AyatkuyaFVysxSAbAAMGHDNBgiCBCgDP7/oLwhn6HsLUth9PMdeVzea9dunL98BcjyvZlZYBMImM1MDAGDYN+Bg/VK9d/+5Z8AtXp39SHMx2KRkZGR3/i1j966cfv8a+cAJYZpcgDsGW5zYVjM5Ru20SIvSBwGlGbz5WKpRkxjYniIB9Tv9kqSxBDkOI4QCyEAMaqYesUwGM/bgBLA/p/t4J/PYom8AACwLEt2uDiOs21KCBFEngFKGOV5njEIIeY4DiFEiGVZBmOM4zhRFEVR3NnZ8QZ8kiQASnOp7QfvP93TP4BE8MILL+6d2JfJpLq72jjArr15qVjIjQ4Pf/cf/n64NeEolB4ZHvvAP/wjqKvf+OSn+f6BT//r9xNn7w9OjBx68NFCWanXreXkdkXXdMOSRSmfTsWiYZfsKJfLgMOZYp4iWCgVLcscHR2t1ZSezm5VUY4eOlwsFpeXF48cP6YZ+traSl9//+zsnN/v37dv3+U33wz6Q48+/Mji4uKuXbuam5svX748MDTidLsmb9954skns9n8y6+9GovHNUOfX1w6efLknam7hmFEQuHXXnutqakpHA7fuTtrU9K9qy+bK8RiTbt27arVavFYoq2l9e7du4ZhxGMxCOH8/PypU6eKxeLG1mY80VStViEDJ0+ezBey7S2tLlm+fO3qffefzhcLyytr4+PjV69eRRgcP3L0z//izyKRyNjY2Nz8vCAIkiStrq/1DfRzHJdMJtvb26fu3DEsU9O0hcXFYrlUqpRX1teS29vZXA4hxHEchFgUxUKh0NvTBSibnLzt8Xiy+dzS6srA0FCumJNk0ePzbm5u9vb2mpq+vr56aP8Br9vz05/+9OjxE07ZNX9v4czJ+5cXl5wOuamp6dKlS52dnVpdm7w+2dvdY1nW9vb2oSOH5xcX3nzzzUMHDmLMX79+fe/evel0ulKpnD59+u7sTCAQaGtr+8lPfzw2NtbZ07WystTZ1bG2vlIplUdGRtY3NkZGxuLx+N/93d+Fw8FEouXOnTt9u3odonDn9o2TJ49Pz9xNpnZ6e/uTm9uMseHh4ZdeefnYiaORSOTSpUv7JnZDSra2Nrq6OhihhVz+2OFjb7xxSZKkpkTsyqU3m5viIyMjkzeu9fb2mqaZLxaOHDs6P79YrVZHRkZeeOEFQRDaWtqmb08p1aooiuVyub+3Lx6JKeVq0B8MBAI2JUMDw8VC4fTp020d7bzHzfk8NxfmqqYBAOQB1zBh1+oKz3F1TWUIKpohy65qte5x+1xen+CQbcJ8wRDHcYZhVJS6ahlrW5uGacYSzYIgWXUtGggF/SGv2xMK+NPpHYCB0y1jDkiSsLq14gsFa0qVUooA8Ljc1LL8Pq8kSYVCIZvOAQIsyypXK7ZtYwiTyc2NjbVyudzglG1iWxaBEFNKazWFEGbbtqIolUqFUtqQbGKEGQSBUDDR3NKUaPO6/YZl2bYtiiLG0LZNRa3V63VD1xmjCIEGZ80I5TkeQS4YCEfCsZ7u3uamlkAgYNsWACCXy9XrdZsQAACWHLzbByW3J9q659j9Rx58vHv33uWd7dszU12dzV/6wic/8au//M4HD/V3hcsF9V/++ZXf+I3P/uGffntpR432HcCBtpmtfE4jFpaAIAfjbe5QzOY51TZNog4OtT/1tpNnD/eS1OKffO6j/+szHxUt1eUUdaUuimLDgcC2LcAYYwBhzBAEmAtGY9VaDUBICIEAMoAAg7ZNEUSANWSP/1Vz8183xAkMIfq9v/kbkeNff+0cwAjaDDNgWRZgdGdtc3Nl413vfg+AsLOvz4ZwY2UV2fR/fuGLA339kigCiAAAlkWCCIx0dtVLFa1WlyAHLYvZhEFYqNZFbwDK7h8+99x2JuvyuEvlOsDYhgxIvIKILmAd8pmKCgFHKIAQ0v8Advafdg5zAADTtmxKAISIwxQwj8/bWAcA/peNIfSWvX0D0ymlqqqapu1wOBkCkDKR43mIOACVcvUTH/0Yx9A//cMzhmpGAsF8ekdEoJZLU73++gvPLE/fOX742NxObrumL33xK1/97Bd//Vt/9sC3vpWDbCufATz92G9+RATA7/FWbHMlm+ZlAWHmdDr1utLRmgj4naqucKJACMGQk3jR6XTPL60mU4WwL1JIZRycsGfPnvXkZkWpt8Sbsskdn9uTzeU2Njbi8fhmcuuZ557VFKVWqciybFhWPpulNolFIslkUhCEeDyuaCqEMBaLTU5ODvX1y6K0tbZ++tQphuD80mLfru7UdnJ+4d6hIweXVpdEWRwZG703N+Pze8aGR9KplFOSEIYej4cC1rGr0x8K7t67p5DLLy2t5ItFn9+vGrqu68ePHDUMwx8O9Q70FnK5sZFRp9M5dffOyROn7i3MpzOZ973v6YV7C2pd27f/QCaXlyVHe2vbnenpoydOZtOZna3kyMiIUlc9bu/w8Gg6nbVt6vf7V5aWA4FAw3Q+HItaxBYdQkPyFA6HDcuUnW6H28M7HPlioaYqrR3tNmGpbMYy7MFdA1tbW6vra37ZPXvnrs/pcblcbq+nqamJx5yh6V63a3lxiZfE/QcP5ovFSCx25NDh2bszbqfrPU89tbC06PZ6mE2yqWxbe4duWpVS+eTxE3en780vLO89sD+dTkeCUZfTU65Wenp3ra6uqqr+znc9lc8Xb9683tocT+9sbW2uePzeCxcvtLa2DgwMlQplj8vb3dk1PT3d3dN+/cbVjY21d7/73a+88opFSTzR9LOf/czUDYcoXbx8sa2rze1266p25MgRjPHrr7/e0ta2tZ00iBGI+FY3l+OJplisaWNjrbW9JZPJKXWNMZrJZNS6aqgWYpARqiiax+dPFfKKYbjdbkjhxWvXwm3Na9tbvkgk2tGeq1V93oCp6CIWTdWURFFV1WgoXC6XgYighGW3q16vE9PyuT0Qo+T2drFSzhcL27lMVVdjTVGHLKa3tm3DFESsmoptkUql4g94YrFApVhRKtW+ri6fW65Wq+VqJRAINC4QMSCKIs/zgiD43B61rgAAMYROh1woFDKZjM/n83m8lNkIg7eU3RTyvCTLPpfTBwHv9YQ7OnqamtoEXiCEIAxsqlNqEWJatk6prZsqxrhQKGUyOcsilAJFUfKFbLlctCwdAEKoSYj1H5bCPAQYIY4x+B8jnczhcIgSX67kc/lUrpACzLJ1jeMl2ennnCE51D506P5Hf+G/Dxw5PLez9sPnf6BomUcfOPiBx04/cXC0TwYuDTzzzM3HP/Cl//bZP359sRAZPlpD/oX1YjJdUS3gCoUjiRYgiJpl7xRyQsR17MzBX3rnAwebvHd+8k+/9dh9S2++5uSRWq0SQhHgGIGIAMgQYYBxgkqJjTkL88VKDWDRIIBSgCDgMAI2a5DrP2dgyP+lrQQAIMyBluamP/j61x48ffrypTeATXieBwwJAocA62xt+cM/+Go41gQwP7ZvX1tX5/r6OkTgnU++48yZM0dPnSC6IkiixGMRgPZISKnVKAWIEhEjQixBEi1CAcen89mHH3l4V3cHj5HL6UAAEkZtRiHP6ZQCUa7ppkkph/7/6jdNywQA8Nx/srSn1OFwEEIEQWjkgA1NaMMKuNF0beTsjRMsy7IsQ6/XIIYepwwoQwgVS4V9I4PH9+/9yAff+0df/dq+sfG+zs721hZNrVdL2YP7xm/fuPLYe57c4PHVculuPvWZSxeEPfvSZVVzBbzt7QyjV199WRQEVdc1ynaqZZVaJUVhECPImbrRHI0iSjRFJYRyPL+5vlHI5Xf19Usul+RyliqVSqlMKbUpdXs9Xe0d9XKlra2tq6srmdyUHA7btg3DGBkavnPndj6fDQUCmUxqfXU1mdxMJpMch9ra2niez+ZzqWzK5XKViyVE2UBP78rSsiCJg0NDmqYFg8FwOHhnemp4eGg7tT0zN9Pa1pZKpZKbG/27elOplKIoHR0dl69d1nW9VCn/y7/8y9DAIIQwXywAjFKpVCqVUlV1dvZuOp3WdX1tbQ0AkEqlMM+putbR0QUhXltb279/fyQW3dnZaWtr0wy9UqkkEonZ2VmHUx4dHb127cbg4CAvCs89//zo6GjA5793d+bAgQPbW5sY41hT/Nz51/zBQG9/3/LqSjwe93q9c7P3HA7HwsLC+vr6yMjI8sLi5ubm+Pj47L35e/fmVFVdW12PRuP9g0O6bvqD4Vwu3zAvW1tbY4y53W7GSCq5vbKyAiGsVSrZbNawzEwm8+abbwIAIAOqqm5tbeUyWY/bK/GC0+kURUdXVw8FTDVUw7AAQEtLKzs7O8RmlUqlWq0ihEZGRnzBQCwRa2lrVevVaDjicbklSZIkqVgupVKp/v7ebDbr9/sCoeC1a9cGBwcjwdDm5uaBAwfCwSCPcTzRxPP8zJ3pxr+AAjY4OGjbdkdHmyxLW1sbhmE4HA5B4EqlUnd3dyAQqKtKd3fP6OiYJMo+X+D69ZuGZfcN9M8tLUpup8fnnb49dfbsGcaBH/z0xw899qjD5ywZqo5gMpdjEJum7XJ5BEGwLAtj7HDKbr/P6fXwguD3+4llr66uVpW6yQjASJAdbrebMdaYYIjHmwRB4EQBQlhXFZfLrSk1j8OZiMaNqra5vMpT4JadlFJdN10uj6YZDS9ywmxKbd1Q8+kUsEyR5xySIxGL9+zqTyRaPR5f0B/kMGdYmm6ohFqM2QAwnucj0ajH42EUWqZtGBbHcRCyarWcL6RNS+c4hDEmNgMMxWItPm9I12xF0TDimxOtsVjMso1qvQwgNQyD53jGICHEJqThpgsAaLhjIQR8Pp/H4+F5jjFarpQMwyhks7W6rhtUcgWh5IEu/+ihk+/57x959Kn3OkKByTu3ktvrI/09n/3Yr5zYt7sp5AoGpIsXb//2//jKh37t0zmFS/TupXJsaae6tlNWLSi5vAajnFPKV6tOv7u1LX7m1P6HD434SfnjT7/tO9/4XZ9LhJqm1uucwBPCEMaGbVmAcBxnMMvj9SFeyhUKtm03HAtsy2KYAEAYtRlj8D9Ztvyf4I6ABYn14V/6YHd7x9LCspLPC6JDVVVCbdPQOA6JEm9aFgWwZhiZQlHTFADB4aOHJvbusW1Tty0KoUEsHoD7juw3TWNza4tYBoCElwUkcgyRdCbpcbl3tpMvv/yCz+sQIQG6BmwgQomZFFCIeblUrnOAQWphAABgBLC3+BnwlriH5wUGAGWUwxwDzLKtRt/GME2AECcIEGPWmFLFGCFkmmZDBY8QhzFumLQgCPV6BRiaX/Z4XR5PyEegEfNK73jw/scefuBP/vgbNy9fVau11tbWPQf2byU3BYHyDlLmrcSTD/3BjTf/bWcRuDGjiEI50ZS4cvnG8O49f/63f83JEhB5IItzW8mtShm63YW6olkWA6hcKPYkWqGml6vlaHPTnokJyzCn780melqvz052DHQFw6HF+YVYU1zTtLnZe+OjY6+++momk3nokYdXVpe9XvfeiT23pm63tLTEYrHFxcX+3r6enh5ZlgcHB3O53OLiotMlx2Ixn8/X3t7udjqz2ylimIVcQdW1TDHPGCGWcfPmTV3X88VCTakHwyHdUF0ulyRJl9642NPdTSl98cUXBwYGsrlcoVCYmJi4fvPGrl274vH4/OJCY/WMucWFPXv2CIKQzWaPHz9+a+p2vKmpp6fn1VdfxRiLmHvumedrmr68vnH5jcszU3dLtdpOISs7HGODw/V6vVitHD16VFEUjuN6e3unpqY8Hs/Zs2evX78+NjaGMbx+/fqeffumpqcXV5aPnzq+sLywurR49PDBarEUC4T2ju2+9MbFaDjS1d710gsv79+3rymRmFta2r//YLlcfvWNN2Ltbbempzieb21uWV9dQxyOJ2LbqR0KWHNTIruTcooSIzSdzYSjEYvY5Uqlo61dU1SX7GyKxU3d8vt8vmDglVdebU60VsuVa1eu9u7q13Qjm80ePnwYUMbzfGdn5/LSQjDod3qc9xbnbQBU3dB1s7mpaerWZGZnu7U9sZ3aVLQ6xtglOpyCA0K4vb1t6Ua5WHJ63NF4rFwra5oyODjIITwxvjuXy9mQIZ4DlC7em9NUValVDVUb6N1VzOfm5ubGxsYmb9xcXl3ZvXv3ranplbUNfzi8ntw8ft9p1TRu3L518r5T66urG2trp+8/+eOf/bhUzr3n6ad+9tLPKqby8NsfBQ4ur9eZS7QACEUjqmnwDkkxVbffk8qk8+VSMpNaXFzc2d4WOZ7nsWKoROKgyAsAhRzuelXdSKZswOqapmomA7im1DVDlx0eZgKomTF/UKQYUa4lmtAqtVwmS20iSw4OY8t+yyFLFEVq2YBBZtmAESxIlAFCKKWgruiNSECIWavnM9nNciVt2YptG/g/lhWFEFqWUS6XNE3VNCOTztVqimFYEGKMeUqoLDv9/kBra2csmgAAWRaRRFkSZaWuWZZVV9SGkTjGiFCbAdpwYbQsq1qr1mo1h8PhdDqdTmetXq/VVd2wdnZ2stmMrmkY84h3QjlA5WjX7qNP/conf/G3Pjt44Mi99bWyWjn74KHf/cwvPn12Yn+bI8zAwmz6V3/9f378i9+8ei/nCPUJ7ta1ZClbqokej+z2OSRPqaaVIfR1xE/cf/CJ+/c/tLv1hb/+48+8/UEzs+lxiplCniBqEZPjOAQopIYAAQKcLLtC/pClqZVKHgCLcTZAjMCGsy4FtEFVQPR/GRKgzaWlSND7z9/5x+985zsYw5m5ewBxnMAjBERR0HS1MSdm27YgyYePHllYXgKEWJaVTCVfO3fOAtSktoR5EYB40OfzeSxKLMsSBEHVFIOaJqBQEEWHBAA6c+asyHOiZcjEdmHONAwBI8SAQYBBoKrq6D+o9v8sD2oc2MRuDOYSSgghPMdDAGVZDoVCDQamQcLwPN9I3hvi98b283ddLheEAEBQLpamb09ywGgOuUs7a08+cvqP/vAr03dv790z1hSLLi4v1ZT60NCQz+MP+by37t0Ze/C+atBxbmXn+WefgW6Pbhgf/qWPnD15cvbenCcQtgHTLVMxTIdbTuaLnMtt2oAQptZVj9NDDKOjpUWp1ZVa3VANDnKdXe1vvnnR4/O8/sYFRauPj4/fm5mNhyON4dLDhw+Xy+UXX3zx0KFDPMfdvj157MQJzTSy2ezI0PDi/MLWxubevXvPnz8fDob2Tey5PXnL6XRijC9duhTw+Q/s27+0sHxw3/5isbi5udHW1uZwiEePHpFlmecxY6ThklitVjs6Ovbt21epVHw+XyzRpGumbdvd3d3Ly8vt7e3rmxv1ej0WiWYymYaHeEO44vf7L12+fPbs2VqtdunSpaeffjqTSa2srDzxxBOLi4vd3d3Hjh2LRqPBYLCtrW1xYTmdTvt8vvX19YWFuWKxOL+0aFHS1dVVrVaXF5e8bk9ya0vTtN7+ftM0Q5FgPB5PJpPDw8N+v//q1av9vX2yLN+bnT15/ESxWE4mk0cPH375hRdtSuLx+J07d0KhyP79++/cvXv48FGPx3Pp4pvtHW2yLK9vbT7yyCO6rm9vbx86dGhhbr67u/vAgQOrq6ter/fIkSMvPPtce0trNBo3DKMlkdje3Fpf33j/009fevMNU9f37Nm3sboeioY8Pu/q8squXX2FQuHWrZsnjh6xDL1QKnX17Mrk8gjiRCxx49r1k8dPyA7p+o1rJ04eT7Q0V6vVoD9gGabH5T527FixWMxms5IkXb161e12x+Px5557LtYU39pOZjKZeDxeLJSnp6cPHz2yvLyEeW5wZDiTyai6cuz4kYWFpXe+893t7a3Xrl0ZGhrp6Oi6duVqX19fJpOpVuudu3oXlpeiocj48MjK8uqJEycMTa9Xqp1dPUgQIc919Pdt1yvbSl3n0NzqWizRLMmOzZ3thol8pVSKRKNur6fh4aNUa06Ho1AoOJ3OkM9PTMsruxCEhVKRQeh2uzHGPp+X47BlmKZhuESxWiiEw2EIYa1W6erqcjtd9Xqde4uttTDPUUA8Hnc6tQ0wzGczm2trpmm8NZhDgMvpwphXVbVcrtTr6lvltaE13MsbizpxHAcBDwDmecnj9nW0d7jd7sbKDT+3WEEIEdu2bLOha8pms8ViEWNs23RnZ2dra6uuVGu1yltrvRGrrpYr1ZKu66Io8pzI85JlUYwFzbSwwHd1d7S2JCgx6/W6IAiaaQPewSQvlQOell1P/MbHP/+NbzYND22VSjWl9vDZk1/73S9+8ld+4cyBXrcApm6vfP1P/vJTn/vyj1+46GvaJYXamTumYxflXLqNFUaLWp0C2hT1nTk28d/ffT9NL3zw4RMvfvfbcZ/bVOuGrmEIGqsG8QBBACixMGJej0vk+VIpjxCgwG6M90DUoLMooA2Z+H8FdzcvmDX1nU8+eW9uwesLvPjSywDial01LR1AIkkOBgBkQOBEHiFV0QXRcfX6NafHHQ6H9x08ZNmUQxxhJqBWzMf7fa7NzfVqtSqJsiAIECHqEAu6mi3XbMhPLqxsbabHWlsDzOKpLnGMGLqAMIOCYtKqZhGGGGAIAAwB+rmTDGMQAB5zxLYptTAC1CaZdDq5tcVxXDgcdrpdPM83kB1jDDBCPOd0OhuuQxBCzAkI8wwgiBAQeE1V29pa6vn08vVLheUZZJaDcf/jj51JpddWNufThe1jJ4+0trbenZ6fujnb1dTdHo77JTw21K9R8N0fPff3//yd+8+c+N7f/9Unf+XD3/rTbym6xfEObyAo8LxpGBubyVK5igDwCE5i2KZuWoTYzG6JRvVSlVEYj8cz21sHJ8b1WvXB02dC0cjSyvKRfQeqxbKiKIdPHDNNMxQMRiKR25O3ErH42OjozL3ZpkTC0PQ7t6fGRkZcLtczzzyzd+9extj1q9dGhoYtzTBN84EHHpi5N3vz5s3e3t65uTnbtkdGRqamphqyv+npKZ/PFw6GDE3v7u6ORCJzc3P1en19fV12OBt9PAhxrVLtau8gjIbD4dbmlmqpPDY8YhjGrVu3EEL1cqVerR47fuTK9WvLy8u9vb2vvvZKe3t7S0f7iy+/0NXWurO5dfvunb7Bgex2ylb0g4cPabap63pvV3c+n+/u6YpEIoVCIRKJCBxnGoZTlBrUp9fnaSz2Rpht2/bW+obH5Z6YmFhLrufKxcaKH5HmpmA0srq6umf3BLVJMpkcGxvTFHVueuaBE6eWF5dM0zx27NjtO1MYoo7WtpdeeHHPnj2BQODG5M0Dhw7u7Oy8efHSfSdPcQhfvXLl8bc/cevOVE2pd3R07OwkG6Xej3/6s717JxwOMZVM9fT0Tk9P67ra1tZ+9fIVb8Df29v7/PPPO51OSZJu3pqMRGKlUmVnJ33m9JkLr59PJje7uztv3bpVLBYppTduTCbizYzS8+fPD44MJ9pbl5YW3/b4Y+VcYerW7QcePPviKy8btjUwMHDt0mWR50+dPn31+vXx/Xtdfu96cjvS0tLV0z11587A0OBzLzxvmUZba+uNq9eyqfTQQP/GyoqE+YDPb1JWq6tmXc8l05vrW1vrW9FQdHbqLoacyDkWFlebu3sCu7qSRN+sVjN1ZW511SbMskitUo8Fw2FfgMPYJLZq6IRShyAKAMX9oZ31zWq12tTUFA54iapoai0cCYgSp2oVCFmlUiLUkHhUKhdC4UC2lOFdPBTQxta6Pxjw+4OKooi8gHleEASnx13XVFM3jFSa2jalNJ/Pq5pmmaTBhvM87/OGmuLtzc3dTU2d4VCLxx0AANi2SYDJIMWIlx3eoL856G8K+EOmrVNma3oNcyxfTKczW6pWgYhgjlFmmpYqiCgUDjhksVwuK4ri9/tbWhIup2xaqmVrilau1YsIAZ7HAABRFClFIu90OgItie72ju5orAkASogBIalWcppa1tRqTakTgDWKeW9EszAItZx46pce/fDHIrsm1rPa4mayv3/X7376o3/4uV97+FC3i4Htzfyf/f0L7/+NL/zdz67s6C4x0S+E24HstW0iYCGfK9kAOYOe3v62D7/3zNOnx779+d/66oee1jNJl4Oz7JquVSHkDIMypguY2LYOIHFIkt8XNhRTUwzAGAUIQMgwY5ACSCGjjNoNlH8L3O9M3nTKjmw+//t/8BXeIU/NzBFKnA4HhJAC27R1jsMAAF3VEATBQOD973//d/75u7du3AQEXDh/sV5XEaA8BIDYGIDerg5DU4hFqUUAZbZtY04oVRUAkciRmKjs73CqG3fn33yN01UJQI5SxgjAgHKoUKsxxJFGq6Axu8UYeGtmFzaWJkkmkxsbG5lMBkIYj8chgxYlXq/X5XIJgtCYaZJlmVJKKf3/iPrP6Mqy6zwXXmHnfXI+AA5yLIRCAaicOie2SDWbQWxmyrZIiZIlU+HKsq5kWR6WJflK15+DJPuHLSqZokixm81udqyuqq4cUMg5Ayfnnfdea30/TlN3D/zGwMAB1pp7znc+D8/zoii2Nnc/8hAiiDGuNhu8qiiyzLlWYWtl8d71uWtv/8zLzy88ujM81JvpTL/+o1e3tjYmRsdmpqZXFpdKuXxXe+qpp57o6e24fvvWL/3Ln88e7X3x86/80Z/+yf3bt/7Hn//Zb/7W7+RKVZ+sQAhM5h0Vs+FYuGk2RY7Xdd0FnmkbQZ+/o62jXK7uHR4KnAAZCPqCxWKxVmtomkZdz7VsiFGlWt3f3fP7/fF4XFJkx3H2Dw4i8ViqvS0ajUZD4VqtFo/Hn3rqqcPDw+Xl5VQqtb+9I2CuPZW+c/fu+PHjoirPLcyPjIx0ZTrv3Lw9PT1d15rFYvFrX/va4vwC9dj4+Pj+/r6u66PjE4VS8fSZM5Zl/eVf/hWEWOD4/Z1dXdeXl5f39/cP9/dVUbJt2/O8VDxRLVcgo4zQ9fX1aDzS398vCeKFc+cRgo1GbXp6en193bWtS5cuvPfeex3t7aqqPnx4f2pqyrWdxcXFc+fPzi8uYIxPTk0vLSxqjXrQ5zMMLRYOE9fbWF2LJ2Ku62qa1hokmIYhcLxuGOn2VCAUKFXKTa1eKBQ6Ojs7u7t6enqeePLxcrmoKFIqGV9ZXlQk0dD1o+zh+Ph4LBYjrjc0NLS9uWVYZkdHx41bN3lR6O7symdz4WBweGAwny8kU2ld1zc2No4fP04psSyrp6u7Uim1aoVyuRwNh/x+/97e3uTUCdPU6/X6+OTxnb29bDbX39t3sLsXCAQCgcDK6mp7puPYsTFDM8+fOd+o1TnEnz17tlAqcli4dPHi1taWKIoXzp1bX1/PdHV2dndtbWx2dHRgjHf2DyYmJhiha2sbYxPH5+bm88UyAOjmzZtHR0ctypAgCKIg+f3+9o60RxxBEGzHrNbKTV3b399v78zYto0A6OvpXVlZQQBeunDx/t27pm5MTp64Pz8PQ/5zLzw7cGqai4YcDtmA+gJ+VZJlSSKeZxtmQPUxQnmOo5QiAInjphNJz/NyuZze1MZGRz3b2dzcdCgRRBlAymGIIddoNGKxSK1RTbWlW8OtaCLe1AzLsTHGLdNesZhHCLUMvWIqlWxr6+7vT6ZSkiwjDhNCWsv3HOYEXuB5AULOc4ltO5Zlua5dqZQODvcM02hBADHmCSUcFhiFjDFd11sJukqlcnR0qOtaa5MTQ17gpIA/FI/HM5lMOBwURZ4w1+/3u67juJasiLquybIcCoUETkIIuS7lOIFSgBA2LJMCRghhgEbDoaPsoWOZ5XJxdWO11mjYhGI5oITaDCgqsY6LL7z89CdfGT196dH65t9//3sBn/wb//Kb//H3fvXpS9NtEby3r//lX37vK1/75d/413+0cajxgQ6T+MoaZVKwSYBFAcUkGOSfuDD5rZ99Wd+d+6UvfPyv/t//AGtlnywBl3AYQwYAYDyHIUQAAEI8RVUJBbplu8SlraPyn0wg7P8TrgIA0PyjBdt2RkfHyo1arl4vVmueZSui4NoWAhBxiAECIZQEATFwYmL87//+H7q7e4f7BwKq/9L5xxzLZa7LA4gZQABMjgwJkOWzOUBhSPVzFHAMcwD7/UGnuDVC52//j2+uvvtXAaZZxbyflwSEEQYAO4wnTc+0WiFIygBjruNwCAPKKqVyPptbXV6pV2sdbe2qrLSl04l4HAIAEZJlWTctQZICoVArzN66jQFGDMHWUgMWeIARhYAQwnNYNzSgBAEvEdcTGVWgd7Q+t3L/6lhP+g///e+9+/Y7x8dHT544vr44P3vn1sXHz3d0pu/evWs7lqDyksL5RBgKB/7wz/7iT/7sf/zNt/8XsLRvfPPnAS8CjGSFZwjUrVrT1TTPBDxUfAoncVjhAUZH2TzghFiyTZL9a6uboUj8/oNHmq4/9dRT9+/di8disViM47j+/v5mrb62strV1bW9v1eqlF3Lfv3V1w4PD1PtbS3HUEPXIIQXLlyYmjyRiici4XCj0TAMwyJusi3d3dtrGIZlWI9ffqxp6KlUWyqVnn3w6NT0KUDojWvXTxyf2ts/3N7bHZ88fu3GjXyx9I1vfEPTNL/iP3HiRLlY+qmPvRiLRInrKZLcqNdFUVR9Cs/zY2Njruvu7e0JHI8grFarW1sbSwsLoiCYpt7V2yWryoN79wf6+yq1iuXYo8eOXf/gyrGxkYHhgbfeeisejcUj0TfeeKMtlTB1fXdnp6eza293l1AvEol8+OGHkUhEFqUffO/7mUxGVdV7d+729/VlD492Dw9Gx8cyibRPEiVFvHnn5ubWRrlYKhYKGGOAYCAUjMfjCCFN00KhUGugurm5Wa/X0+m0oiit2p8xtru7u7m5WSwW9/b2HNuGDPh8ytziPC+JiWgsGU/U63WP0Wg8znFcJBSGgIqyCDEirkspNS0v09lTK1YdzUiGw23JRDAWKtQrFPOqL6g37K2VLZ7xRqNZbzYsx84XC4Zpu7bj2U6z1nw0O7e4upLOdGi1ekBUQv6ARwhDUPX7MpnM+tKqX/Jp1WbEHwwoqk8JdrRlqOf2dneJsrK0vBqMBG3P3jnYGRgdyVUKSlAeHulfXVoMx0LBSODenZsXL5zNdHX8w/e++9Rjj9l648btDy89fVkIyEWj5sqchmmdOQ3iUgIQwJSAWDRhGybPYET2QY+6HsWyrAT9vCR6jquIiiSq2VzRdFzIC7rHHMR7Lo2EwtSmkuCraTrkuVKhKiEVUw56KBgM2o5ne8TnD7b8n47jAEJ5UQCMAUEAlEIILdsyTZMwr1zJZXN7pl0HgHqeyxhtzcxs2y6XyxzHIQSKpWy9UaLMZsCFEHoexZiXRB/xoK6ZhLBwOJpIpFTVjyCHIAcAogxAgDlOaBFhPeI5jtOCOWOMq9VqKxfQ4pcwxj5SRCBGAYMcZACJkiIKSlM3JF4wTD0c9PV3d7QlwpbRgJ4LKZGg6FciQPBx0XTPzIWv/frvPPXpL1dseOXaDUdrfOWzn/iDf/3LP/PURE9IEi1w8/1H3/yl3/+3f/RX6wUgJEddXyKnu02bVJomCITyRq1nOPHlzz3+7ET67f/5x7/95S/f/M73MHVt24AMAYJdhwIGGYIMAxt4st8nyWKxXCCe25pJAoABQABxALGPwu+MoFdf/1GlWjcso69/UFB969vbc3cfYAYAJRRQ9NG+E6CQIsggYpcff6zRaDSbTcCh3d1dy7BN3WCeBxGCAEwM9yLXrFRqGPPAoxLiIAMQwlqtkUzGytlln6pXnaNEb8LwLMqAZRMIMGMMIHRQKlsMuh79J+il6zj37t3b2dkJhUItc8XR0ZGmaZubmxB9NB2mFLRIMgghWVZkWeawAACQJKnVascYyz95Wn83gEHAWCgWr9frxHMwA9B1zHJxcqD3Zz/zmfLezp//yR+9+t2/OT15bOr4sXff+vH+/v7ZkzOffunj7W0JQh1FlZrNGpa49kyquzP94gvP/ua/+W1ACA8BD5kqA92qN11T9vswxoam601NFqVqvZbp7gIA+GS/wPEXLlzKZQvPPvu8aZqvv/765cuXd3Z2KpVKe3v7ytKSbdtPPvnkw4cPPcie+9iLd2/fmT4xMzg4XKlUJqem0un06upqZ3dXs16/evWqT1EhZY7jnD59emV1tabpvoC/2WzyCB8dHjYbejAU0hvN/f19jJAsyj1dvaurq+l0mjF2/cObI8eOxWKxD65cOzk17bru5vrGzMnpN17/EaBsfHx8aWkpFAqkEklD05OJxOrqKqX05Z9+aXdrt3Vurq+vJxIJQeAO9vdb2q9oJFIqFCVF1i3j4cOHfb29pXLB89ypqcnD/b2m1rh08eKdO3cghGfOnNnc3JycnIxFoksLi6emZ8qlUrlQPHXq1Prq2tHR0cjISDGXFzn+2PDw8uICJV5HW/uDBw86u7skQdzb2/nkpz7Z0OqlaglycO9glxf5jo6OW7duybIYCAUNwwiGQ8T17ty5M9g/IItSsVzq6ukOhsOlauWZZ56pVisAUEVRDMvs7OyUJGlu9kFvbz/GeHllMRQKVMvlkD/Q0dExNzerqvLo6Oj84rLj0AvnLiZjyXg0dnBwcHR0NDA0sn90+Gh+caB3oFFtpOKJkeHRubm5tkxbOBx+8803T0xPEUKWl5cvX75s2tbu7u7ExMTc7GyLy7i4uChLSjAYrDUbExOTp0+fzRXygwPD9Xo9ny/29fXdv39/ZWXl1KlTe3u7kUi4o6Pj6tWrMyenRVG8/sHVifHxYrm0tLb67AvPb21tPXz48Ctf++rG2lo8Hu8fHHow+8hyncmZk2v7O0wV+6cnq57dOTLoi0cZz3mQRSOxVsCsXC4HAgGJF6ymjiiLBEOEEEs3OISHh4frzUapWkEcDAQCekMXOQ4hqJsGw5yiqpZltb6JZlqRSKR1OkuCAACiridJiqIoACPgui28ewtYVqlUHMfVNC2bzVbrZcYIY4RSyvNSOBwLh+M+NRSPJ7syXZFwCCHgupZpmq0qHiEUDoU7M92Zji6/P8RxomN7Lb/2weFBLpdjgCEI/il/10rQVyrVeq1JPCaK0tDQkN/vzxfytm0DSBhwAaAEUAYB5DBjkOfFSDDW3tbZ29MfDodbLX7iObVquVYqGrW6Y5gU8HIgjpWQCcSTT73w8td+7vnPfoEPhO/OPqxWKy9/4mNf/8oXP/lTT4dVXuDxO+/d+c3f/pPf++P/MruTj/ZPuFLC5COr2Yaa7iACYNh67smTv/CVT4ZZ8z/+2jevfO/bCk8Ro67r8bzgEmLZDoQcpYxQFwDQlkjzHGKMQYQYpQBCBtg/MWcYY+gXf+3X2/v6VEn5pV/6pUiqzfbY3L0HgFBEGSUEAYwAZBBQQBgjEDDL8QRRhhAARtsyHd09vR/11iEgDCT8IJOI1xr1atMgrgcpa1G+dNupemDZQmbvsDnU/tDOFzlQZwjJQcNixMMM8BaVKjrBnAQA2N7a2t/dy+dzMzPTrViIZVn5fN6yrHA43NnZQTynNUIRRBEg7FHgeB5ACAu8IImCKLfyXljgKQStdrwgCDwv+n0h03QBY7KqmJ6DONG0vaA/HFZ8tYPD6tbWJ8+f/Ref+YRdOviD3/+t1/7x78+fPXf+3Lkrb7+xPHv/N3/ll0I+NRyLAOj9wb/99U994uml1aXZ+YXT09P/9U/+33q+5OdhMiJFgj7dslwGEIAxNRDkJM4FquxDCEkILj+8Hw2GfvzGm12ZzOLCXFdn59TU1PL62ujxCb2pvfq97z/xxBPRaPzatQ8/+fKn/IHQ2++88/yzz+YODsvVSv/wYKlUOsplT506NT8/ryjKxMTEysoKhDDT3jE7O3v8+HHV51taX42nkhzCjWrNpyh7e3tbW1ujo6Ot5kAymYxFE5IkWbadakvbto0x7u/vX1paoh6JJxNvvfXWpQsXMGQ//OEPH3/yid2t7cP9vWg4UqvVYol4JpP54P2rp2ZO8jx/7dq1V155RdM027b7+vp2d3ZkQezu6REEAUOUaW9Pp9Obm5uOZXMIlQpFv99fr9ayB4cfe/4FVVVnHz0KhUK5XK5QKAwODm5sbCRj8e7OLtt0MpkuRfYBAFKJpE9V7t+8mYhE19bWFpYXopHQwc5OV1fm/Pnz//uv/0r0KSdOnNjZ2RmdHE+mE3t7O08+/oRm6MVK+elnn7EsC2N8/uy5ax9cvXTp0tnz55ZWloPh0MTx4//wnb8/PXMyGovsH+w98cQTtVpta2vz7IXzD+8/ONg9OH/m9Mb6mksYIWxh7tHJmelKpXLt2rXnn33WMIx79+6JvJDN5mPRhM8XePTg4cmp6Xgiurq6PHFiotqo375399j42L17D1ZXV5999tn3r35QbdTPnDlz7erV/p5e27LWNze/9LWvJpLJlZWVgOpzTOvu3bunz5y5ce9OuV7rHx66ee9OPJlQfOr84sL5ixdikej9+/enJ0/wCO/t7H7uU59ZXVzeXl0/O3164dFCMBodHBubm18QRAkL4o1bd/sHRyQl4BEwM31Gqxu72/tDg6OHxVJqcBAnIjfWlmA4EOlob5imHAxCUcg1a1LADzyiYqEjnAQ2zRbyoiK2JWOursUjUZHn/Cpnm41moxGPxiizKLQlRfYgazhNKji62XA8G2NYqlbiiYRt26Zu8RC3UhymaQJKgMC37gAIoc+nJGLJgD/e0d6f6egLB6MCz2EMEQcJJZRBWQ4G/HEe+zwCLduzLAthQKhNqOl6pkcsChwGCKGu59mOY0LEXNdmwA2FfZQ52dxuvVnWjAaAFGMsCmIq2RYKRYKBWFemPxFrYxQjgHgBO65hmA3b0SDwKPA4LHCAIwQAxmFeRkhybMaYaDus2tAgxAKHiG0Qq1HMHmxvb+8fHAIAA+EwE0VH8SXGpz7xjV9+5Zd+PTl4bDdfIJicOTP+f//Wzz97YbgnCQIquH5r9Vd+589/4Xf+nzcfHbH4WLxv5qgBijoQI4mC2ejsS37qYyc/98zk//i33/rjf/UNvZoXJdF2HY7jRV5CDMpIFCHmGECtbDtkBACAEYOAUoAxj/7puXL7zvz8Yk935tknn9Bt2wPwnTffABAKHE8phQBBgH+CqqSAscHBwXK18vZ77wIE/YEAL/KSIhPAAIKMMRmAkb6ufD5fKJaCwSBgTIA8z8mHhYqU7MvC1K0c2yWq4YvJkbhuWx4lmBNcAijjKZbLmmlTijGOxWI9PT0dmUylUtnZ2XEcxzTNRCIxMDjo8/k4jsMcxxhDHMaYR5CjFHgepRRAgARBaKX3WmW7KIqmabbW2DiOE0U54A8BADsyXRQKps0g4AUsVHIFntL2eAwRVwZkZrj/c5/4GDVqv/Evv/76d//6yQun2qPB/c11AVDTsJLJ5H/5k//86Zc+/uY7P+7p7zo7MfzX//WPv/HKx3/nW7/Ym4wHZQkhVGs0GYOFQkGWVcvxAIINrZ7LHfX2de3sbKVSqUIu61dUo6lZlkUoe/jwYTyaODY8cu/ug5av7tatW7bn+kPBo/1sMBjkeX57ezsUCcdikfX19cmJ4wcHB4uLi6PjY/lScXZ2dmhoaH19s9lszszMrK6tHeWyx44du3XrVsDv/+QnP3n37t1UKlWr1VpIyK2tLQihLIvFYjGbzeYLWUppuVYqFPMXLlzIZ490XT934fyd+/dOnjzZ39u3tbU1NTUliuKj+bkLFy7MP5ozTXNweOhHP3qzr68vGAzfuXPn9OnTGPNvvfXWsWPHms3m4d6hz+cLh8McJ7RAzZ0dHfForFQqbW1tuYQEAgFRkT1CVL9PVVVZlGu1WrFYbElaKrVqsVyp1+uGYQR8fkVRevv7JElyLbuvp3dxcXFlfXlibIy47tLSUltbKp/NlsvlYDDY4kFKvHDnzp1oNFoul4+Ojrp6um/cuDE/Pz82NsYYMwzr+Rc/try2un94cGxs9PbNW5FIJJFOXL9+ra2tY2RkZHZ2Nh6PU0rnF5e7OzsfPXwgCeLE2NibP3qdA+z06dPrG1uxWGJjY6NWrnR1di4vLWn1xomZE7NzDzkBRyKh69duHD9+fGBg4O7du5OTU6Is3bhx4+LFi5ZlEUJ8Pt8HH3ywsbHx9NNPA4AopZFI9P333+c4bnNrfXNzc3BwsNms63pzbGxibW2DMaJrjbm5eV03otHo9vb22Nj4yMhIpVRMxmPFcmlrZxsgaJo2oZRSmsvnl5ZXCoXC0sKCX/XHI1Hbtrt6emq2OXz6lO1TDozGlUf3y56dtRpNyMRwyGaEMOCYVi6bNTVdVVVd1zVNkyTp/uxDxphtGYoo+BS1WCyKEo8x9DyPuhRxsKFprYYqx3F+v79arYZCIVWSecRzAPOQS6baaFM/3N2rNxsYIAQgBIDneb8vqMg+nhMoo4Zl2I7pEadaLbfwfwAAxiBGvChKjLFqtWpZRq1Wy+fz5XLZMIxWRwUhJAgyzwsQwkqlUiwWEQKBQECSJFWREUIIIAYYY5DnRJ8vwAAklDEG640mAECWRde1m81auVIoFYrE9RBAPMdTl1CXMIgEUfI8gjEfCkQU2Uc8ZnuuadmCJCaTiY6OtK5rlBLXo6IaVIKxBoG+ZPvTn37l5a/+s7HTpx0Eq83qc88++bu/9a1PvfT0yEBElcHW5uGf/cW3v/krv/3X//AWUNvkxHCTBX1tfdu5EuDBpQvHf+bFx/cfffjLX/pUfuORyGxILI84HmWuSxDjEUTEdSGAHiCEuQQQCADG8CdsGQAAQKtb2ydOzrzy8sv/6T/8h9t37n3mlc/PLzwCuoYFnlAGGGIAMADZT15votHoz3/jm81ms1YuKaqAeEQAoAi6jCJIeAB60nHP8+q6ZlmWgAW9YSly2EXSdsWVM4/rwlTX0Mt9vY9H/AlgmTxybUfjeNHwKJZ92WKJQ8glnj8QAAgSShSf2j840NGZSbe1xRMJ13MRh3lBoIwCxFoNHElRMScAiFs4utYrmyTLoiTZjkMZa9XtAEKf3484iCWhpjXD8bRlIYxVjpea9YataxgBiznlZtWoVoM8H/fJj58e/61f+oJX2fnd3/zVGx+8/9j5k5//zKds3ZKFoADQ8Ynpp5579j//6R+9cGasSzBeODGAygdD6YSKcTqebDZ0SVZ9kfB+tYwDiuE5csAXbYsflfOpzrZjY0MCjwf7+yRB3Nvbm5qaamXzo+EIx3GyLCs+tVyrlstlnucpYC7x9vf3IYTNZhNCWCmWNtfXAQA9fd0P5h41Nc3n8+VzuWa9USmXDw4ORsfHAuHQzTu3L126pGna66+/funSpY3NTcuxT58+vX+w29WVCQWD2cOj8bHRgYG+YrF49sL5ZDotKYooS7pp+HwKpR5jTNO0YrEoilJrd2l4eHh5eVlSZJ/f32w2x8fHW7qPF55/cWN1w9DMmZmZhcWlyYnJlio2FosRQirFyszMqYWFBQjh5YsXKrWqqqp9w4MLK8tdvT2toWVL5JRIpGKx2N7BfiyRbOvMPFyY8/kDot+/tLZaKBUN3VRVP4f57t4eAICnW3q1bmo6AAAzCDwqCILjufu7e5CBw/0D27TaOto9wHYPDxgEzXrD0PRapb6xtn7v/n1BkTzKctk8x/FH+weaoU9OndB1vZAtQoYs0+7s6k6n08Visa+7TxTl7e3tF597tlmvPXz4MNPd9XB2lufF4+OTO+ubgHgIs5X1JUEVGAb5YuHll15aeLRQrzfPnz+/ubbOMRwOh1997bWDw2wkHMvni4lEyq8GWhPR1n7QzPT06NBgV1uHiDngua5tYQRM3WrU6slk8sy5M0e5guILTE6duHnvjmYaO7u7Io+HBvsUWYjHo1293cFIsLsrc/LkdLFS7Bvse+zyhd3tDQwpRFTgIXHtWCz27tWriYGegYunzZBSFFhdRId2Y69arJt6MBoGHMcknvPJkMKQP5Rq69AdBwmCPxjikcQh3nVdxadqhmXbbjwc5wGKhxLAhSonU5vUag3dMACCjUbDsRxiO7ZmWIbhOS4KBn2BQKPROMzu61qjlc0GgBLqesRtyf/q9XqjUXNc4/Bop1w9sp2GR3THNSEAsqwqcoASFAiEkol0LJqQJRUARAlwHQIA9FwiSUp7OtPd1duW6vQpIY4TIICAknq9VioUiedhxCOIGWGMMAQgz3G2aVXLFVM3KuWyY1sRX4CjoNloMAogz0Geo5A6ngsAkHiREE8QpHA8FU13pfqGoh2d/qACgWPo9XK5TB3X1nTPsgK+gBwMmxSpbT1jF5994qc/f/zcM4cFM3tUefzcuX//m7/2e7/09Ym2dByC6l75f/3vVz/7lX/13//m3d26bHJt7cfOmEqoDrzOTPSVF8738fqf/vJX/+zf/0blaJPnEcOIQI4y6FmEwzwElBGbh4B5NmAUUAoYY4C0vpDoU8+dPfcH//EP//Fv/25ra+fS5cf2c6XtzSXAcYQwBn+C7wIYAQgZ3V7bWl1ZyeVyHIc8z+M5TAGDAHEQiRAJAJw4NoIAq1QqlmWpsqIoStM0oCys7B41SLBn5AnMd3iez3IoL3KAuQKCnmXKgkwAaNq2RUnrFuEw15IJAAB4nveIZ9kWz/E/6aBBAAADwCWE53lJkkKhiM8XEASJMWBZFoQwGAy2+uytx/M8w7YI+2grSg6F4qmMw3CtoZWrFVEUGQTFehXykEPIMS3X1BFxjUrxwsnjn37x2Xr+4F//yje7UpHOVFKErCOZfPsH/3h+Yvjeu6/+6j9/5SsvP+81CyGZmxkfTkVDiFJR4nf3DynCGONKpSJIEmWs0qgbjlUsl27evJlua/vww5u1Rr2zs/uDDz6Ynp5RJGl+fnF0dJQCMDs7e+bMGZ+sZLPZkYkxy3G6u7tHhoabzebm5ub46NjJ6an2TCYaj01Mn+jo6YrGY7quHxsc4hFWVfXdd9+ljMUTieXlxbZkqre39/79+5OTk5zIb+1sTk1N1arVUj7X19tz7dq1arV6+vTJV1/9R0kWBEGYn3+U7mgjgK0sLfd2dR4eHmbzuba2tsPDw729g2KxWK3XCqVyJBJTFV+j0YgEI92Z7tXl5VAw4vf7S6XK0ODg+vr6wcHRp17+zML8UiQYGRoaevPNNwcHBxFCr776aldXF2HewtLixInx7//gHxkE09PTu7u7LbD4/YcPRkZHbdctFssz06fqmp7P53t6+mrVuizL/f39Dx8+pJQG1ODi4mIsHOnp6TnaP/ArKodQqVRSVZUxViqVLl++XK/XS9UKxjiTyfT2D4yMjIi8gBASRTEYDLa223TLJIREE4lmU2/UmzzPh8PhQCAEAKrVarIs+9SApumxWIxDuFwspRLJTKa9VCqNjI2qqrq7vTU4OHjh3Pn2dBvisD8YMG1rdPzY3t7euXPnBEEwTVsSREqppMhnzp0NBoOtMKUoiuFw+OgoZ5hmrdZwbA8DuLm+kS9kY9Gw3miODA8nEoliMd/T0+N53vzcwujoaKVW/dGP33zxxRcfzs23t2dGRkbef/9dRZH6B3ofPnw40NsnYO7N13906tSpcDj8l3/5l6dOnbIcZ+9wLxz08zy/tbnz/Mde1Fx3v1x8+atfKtjG2KVzE5fOs5BSBt5WvagLAEqCbpkcx1XLlfWNTcqAZZmmaVLP4xHP87haLRPCAv6QpjUkka8WKgHFjxkSeYFHGAAgCALP8xgiAXOxcAx6NB5PAgQpYMFQKOgPaPVG/ihrNOuAUQBaUnsuFIj41BAhDEIYiYRkWeR4gDkIIWOAIIgURUkl2wK+SGs5ESFEqUeoywDRmg2OxxhBQgkAyCPUsh3isYOD/XwhR6mHECoUCrncoaFrtm1BCBBCkqgkk2meF0VR7uvtb0t3KIrMY+i6TlOv1xsVyzUQhAhDgAChhMMcZYABKEqq7RECgONa9UYtkYhxHCqXy67rFgqF9ZWVSqUmq0FOChDOJwSSo9MXP/fPv3nh6RfLDX1hcXmgp/u3f/3X/uD//jefePpcIqRAQv/y269+85d/89/9wX9b2q2mj81I6V7k8wcCvhefOtcbkx+++9qvfvWz3/7//YGR3ZEwIY7BCRyhhIEWSgFgjAH1AHVblXsrbYh29w42Nzev/vDVX/z6z+1u79y8d4dXwdzKIoCQeIwyQBj8SNNHGWAgHU80K7UXn30eUMoBQjwbMOh6LvFc5toKAP1toZCsZvcPiO1Q5lhEI7xlQwuJWDccQ3cRwM1m04S0QWyIOAygymHmmC6gLoQNw8CYJ4Bark0YhRghjCljGHOCKNquAyHyKKGMQYham2aYF0zbcjyXUCrKUjAcDkbCLQVwa/esxSpR/D4AgEs8jLGta4CRQCJ+VK1JPiWeTjFRpIIg+WXHdcvlsmVZtWqzUKxoTbtRrlGjcXH62EvPXrp75c2Y4tm1I6Oa9Xv6508f/9Ynn++OS6bbQJLgD4UZ81QJQmrEokHIY0KYCjk/EgAAuXJRUQJBXxhQNjQ08uDhYu/giKT4crnc5UuXPnjv/Xq9/vSzz/zwhz9sNpvHp07cvn1bEcTJ8Ykb9+54CHied+/O3bGxscuXL9++ebNer5dKpTfffstldD97VKvVOts7Hj14GFR9PlkJh8P+YKC3v8/Q9J2tTQaookj7R/v+YGBkbNS2Tde149HY2vLKiePjCIJHjx6dv3gOIHaUPejs7FxZW200mz09PVsbm5KqDI2M7OzuThw/HgoEqUcGhkb8wWCxWEzE4+trGy1kq8BLsqwQj0GGlhZXBF5SVXX2wYPpEycsy9rd3R8bG+N5PpfLfekrXz7MZRmCwXD4wcOHfQP9B0dHaxvrrWZOLpcbGx1fX99wCZUUeXFuMRlJPHHxsevvf5Bpaw/4gteuXesfHFAkdXt7++Mvv1QztYODg5mZmcNCLlcojI2NPXjwIJaIB0LBD65djcSiiIFCLpdOJtdXVrd2tnlRshybQdDX22vqRktgRiEwTHOgd+ju7XvxZAJxOJsrHDs2hhE62j+IxeKKL1AplUdHxo6OcoZhHB0cOJaWiIVECfMC5nl8/fr1re0dQED+KK8oiiRJB9k9yzV4UVhYWOjs7kqlUs1mk+f5ZrV2uL/f2d21d7BfKZVSiWS1XJNl2e/3X79xq7M74/err/7wB32DfQ9nZ7e3t9vb0/Pzj3Z39y9deqxUKvGYG+ofufHhnVPTpzzP++u//utT06cCauDK2+8+duHi1sZmoVCamTl18/rNB3cffPUrX4OYy1dK4Uhsd/8w4PNfvHj5ypWrfT39g4PDr7322tSpkzVDe//uzdTxoZlPPkc7o4fM2C7lZdXnOE4qlcIiBzGMhYLUskResGwDUK8tnQwEAoZhUEBc18YIqIKg600GCM9j4nm6bniOF/IHHNPybEfg+IWFBUBpKBRq4fx8qqwqkuM4lm1ACEVBhBACgFUlEI2k0skuRQ7Ikh9DEQLcopw7rgVaKRBKKaUAUNezG81Ks1k1zHpTK2eze+Vq3iMOBAwhhBDmeTEYDEqS0GjWa/UyITaEDEAqKyKAFACAMEaID4fi0WgKI9lxiGkautVU/LwgAohcw2hU68VKpcRzPEKYAfiR9w54IobA8zgs+QPhhqZbju0wt1yreoym2jsU1a8bJgRQFhU5GOUDUU8K9c1ceOGVrz336S8WTff2o4elaunLn//kf/rtX/nMU6eG2niBsQ+uzH3t67/zs7/8bz9cOooOn0TxdleUTp6a/OwzF48FuA//8r/80c+/cvXv/oznLIAcgBFlHASS4zIEsQsJ4wAFlDFKGWGAIswJuVy+Wq36VGlidOjv/8//Ef3q1Rs3AACMUggAArQlCgGMAQbD4eDnv/SFU5cu+gIBBijPIQwxx3ECL2AImOvJELz80ot+WTzYPYSAd4HnMJfjOMAQhAgC7Hgu4xDDnMuAh3mXMo84gsBDjBuGWdcNApjneSIvYk6gALa6bx7xWrvIlNEW/YdQgnHr181R8hFDRhRFxGFKqRoIBiNhLIgUfuRs0uoNWRB5LNi2zRgFCCFB9ADQGs1ysUQY0E0LuAR4LsbYsm1BlhRJhgx4jhv2q8yxmKmdOzH2zNmp4/3pL3/qxV/5uS+fGemXbI2jXjqZADwu1koIkvZ00jR0geMNw7AcDzLgV2Xd1Hp6ey3L0jQtEopubm4PjgwvLi+JsjQzM/P222+fv3RRkKRXX3319LnTHZ2Z3d3dRDTW3t6+urKSybTbrpPNZls4xp2dvbGxY0tLS7zInb94YWFpMRgOJZPpudn5k9Mz4WBoaWnpxIkT9Xr97t37IyMjgUAIMdTb299oaNVqdW5ubm5ubnJ8IpFIBAIBraEX88XpyROrKytHB9mQP7S6uhqLJzhBdIn3xLPPVqvVvb29gYGB+/fvO47T1tF+687NweEhiMGj+UdnzpzZ3t5eW1uLxGN1rWE5dnt7mud5WZbD4XC9WmshBxqNRr1er9RqmUymUCh0ZzqDPv/+7m4ikRIEwe/3Cxx/cHAQDocjkci9e/d8Pl84HJybmzs2OpzPZ69evfqxF17Y2NjI5/PHp07kcrlisdjT2XPlypVMJnPs2PD8/HxXT3cwGn30aP7Tn/50tVrN5/NPPPHE0VGOUjoxMXH9+vXu7u50MrW0tJTJtPf391/54IOTM6cCqm9/Z3d6ZkrX9Ww2+9Qzzy4sLKysrwwPD3/nO9+xLGtoaPj9qx8IgtDUtfsP7509f85xnGg0nE6nr1+/7nlu/9DgxtZmZ3dXPB7P54vHj58grnf37t3e3p56vb65vvLcc898eO26ZVmnTp16+PDh6Ohob2/vlStXTpw4kUwmD/f34/G4bbu1Wu3lT710cHBQbzaeeeaZhaXFjo6OdFub49p9/b2KoiwvLyeTSZd4h4f7vd3dhqbzmJucnDo4ONzf3lUkdXZ2VhAkz6Obm9ttbR08z//oR2+Wa9VQKKTrul9VNze3r1251tPZu7ezY5nO5MmTtucl2tqmzp5dPToI93b6OtJ5zwp0tW9XiuHOdoN6VU0DGGl1LRoK+2SFUSiqvoapu8TxB1QBYVEQiOXYto1FAQlivdoIqD5RFPxBfy6XQxDXtGYoEq0Xi0AzIcKyrPr9YZ8/6AuEQpGIJAkesRhwCXEd126JHOhPsn2W7TQamq6bzaZeLldrtdpPgAQIQsxzoiyrCCHTsBBCsqr4fD6e5xkglHotUbTf7w+FIvF4vKOjI5PJJJMpRZEIcQGgrmdalu55NkSIeAQAiBEPIC5XK6ZpWpbpuk6zUTMNjeex41qaoTFAEYSAUQgYYIRSDyAMABcMRqKxRCQcSySTXV3dqs8niYphWLVqw6MEUOYyIIciQPaLoWT7wNgnPv/lL3z9F3onxm/PzxVKuY+/8My//tVvPv/Yyd72gE8A9+6u/er/9adf+sav3VrdVzL9QjwVbW9/5unLH7t8Wnarf/vf//Dff/OrO/P3MHNdy0QA8VhgADDECGDgJ/REAAB68RM/fWx8/P/84/di7ant9eVT08cpQovrm8BxQ6JAdA0DCgHzAKMIMwQI8mzXpIwQiCgAADAISGspygMAAgwA4KCVy+0ahmU7TBJ9HOSwywGHAUod5jjMBjyEjEHAaS51RdHETCMWwlAUxXJdI4ADDAIAKIAQcgxQykjrJ+Y5jlHKGKQUYIRawEsOIlVRJIEXBc51bdu2scBzouBSRgCAiOMlmed54BHguYgSnyR6xAGQRmIJy7KIbWDqeo7rGI6Wr7pNgzBPDfk4DjXrVQQoh6Dp2OVqxbNsp1b1ec6Xf+rZC8eHqN10qIdElXm4VCjZhu56luJTg4FwQA26JomEo4VSkfMrR5WSz69AwAyjEU/EmMd6u/s2ttbjqfja6vLS0kLf4EBda1LAOrszkiICDJLJZC5XqNVqfr9/e3MrlUj6Q8G79+53tGX8qq9YKU/NnICIPXz4YOLE5MrK6urq+vGJEw8ePCiXyy3rNAK4u6P7zp17oihziL9z857fFyqXahjgro6ufL44P784MDDYaGrEISsLaz5R9WyPg1x/z6Bjuel0O1KUH739Tndvz/Tk1JX33z975gwFbGNj46WXXrp3747HyKXHLr77ztuRaHhwsP/B7P1UW1JUxK2d7VOnZw72du7dun3+7LmrVz7Y2tpKphPbuzsEQpfSubkFZtt6tRryBWOhmM/nY4xZlh2NRg1DazbrJ45PhX0hW9dGhvttz/QF1OGhQdM0x8ZGU21J07LaMp2CKC0vL6djiY211asfXps5e2phaRnzwszJMz/4xx9CBtrTqdd+8GpHW7vjOHfv3z934UI2m3348OFLn/i4Y5uLS3M9fb0fXLtqWdYTjz1+88Mb0WjUF/C/+/574+Oj7enk1vbG+YsXmpq1vbt/9tyFze0tWZXOnDt97db1TE/GFwzMz88Pj41Kqvrh7TunL5xr6Fq9Wrt08fz68pLrusPDw0tLywGfemxk8Ic/+O6JE8dNU79z596pU2fWtzc2tjcef/zyw4cP19fXR0bHDw4O/H7VF1BX19biqSQEuNnQz5+9cHB0uL61Xq5WiuWS47kNralZhqxKyWS82axCyPL5fFu6o1SqBHy+WCTa0Jo29fLlCifJqUy74vMnUvGOdJvKcQFJUmVfMpZMRmOQ0GazaXpO3TDztZrHQKNpJBNtuWxhe/9oZHrmzPPPGQHlwdHeYj7L+VSPAOIBw7AR4mSfurK3T0VZDailQl7hBKPUkHmBEGJBpDmOX/UbTY1Qm2Ii+hQXQgPQcr1ml2q1tU3AMACIMZ5hkQHOI55hNi2zkc8fNupl1zEc2wCAQQaoRxBAkigLgug4rmXZhFAAoGXbHiEItVKOWOLVUCjZ3t6bTHb6/GGOlyFAhqk5rs6YaTsaABBhTpEDvCBDiFurarVapVYv245hms1Go2bbBmPMcTyMBVnydWUGJEG1DMdzaDKeaE+3hwJBBJmpaxBQz7NbTQyEMYQQAUw8igDHY9HvD6qK3yOUEEaZJ4p8pVau1MrlWtl2LY/YDBAsKcgfsnmVT3Wd/vinX/75X+mcObNVqJQKhafPn/3Gz/z0lz725LH2AOeB9a3qt37vf33h137/u7dXm5G0l0iPnDl58fKZC+NDtfWF3/vGP6uuLUoCMrUKJTYEDFIGAaOEQABbgUj0K9/85WQivb27U8jn/D6lp6fLJOzuw9nNpRUgCJC61HMIdTjIMYQcQBmkgshDjDHHIQ7Dn1TWDDCe41tz18mxIUurVav1pm57HtE1UxAkSgDgMcQI8ZgQAhlUZZ/jAd1wCKOMEUKIKEilat0FCGPMWGuRFmKEEWAY4Uajce/BvRYIjed4Simj1HVdCCDHcZqm1ev1FgnHtR1TN03TRAC01hZkWeYlUdMMxADHIc9zAPUSHR0OQ47jOIZRr9YEjAJ+FWMcjsVt18vmc5Ki4I9IZFwwEFJV1ScJ7ZGAW6sQo+EZTdM0DcumlDLHw4Aw17H0Zi57QFzPc12AoWGZlutAQYAQ2paBMOB53jRNQRAc4sWTsWNjI7VazbLNuflHDa1erJR3dvZac9R4PGrbdjQSN01b0wxVVTmBdxynWCyKopjP511CBoaHms3m5OTkyNBwPp8/fvw4AGBxcZ5DWOB4TWs8+fRTB0eHS0srX/vaP6vXm92dPSenTy0tLZmmeezYsQ9v3p6amkm1dRSLRcrg2bNnXddVFKXZ1Nc3t3XTsjxSLFcfPXr02MVLcw9nRVlQA+r62orfp9i2Xa3Wn376SQTgyvrK0NDA5uambmqZrs6333knlUqdOHH81q1br7zySjDo39nZGRwcLJVKhm0NDg4eHux1d2R6Orv29w8sw0zEYo1GowWJrVYqsUjUNE3HsttTac/zFFXmOK5YLMqqcpjLZo+OTNM0baeju5vnuNGh4ZGRkbt377Z3dGaPioVS5ezZs60PfWJs3DTNeDx++vTpudnZZCoxMTE2O/cwl8umUinP8zo6Ohhj29vbA/2DBwcHpmkODw82Gg3bcweGBxqNmt/v1zQDMpBMJjXT2NrfHh0/tn90yPP82PEJSoHtke7u7ls37wRUHwBgfn6+vTNDCDk8PBzqH5h/NOcR57OvfPbgcJ9SmkgkDg8Ph4eHMcft7e2NjgyrqlotF2empmqVqoBQrVopZHOiIPR291WKpVQqFQgE2tpSR0dHut6cnJycnZ8jgEmyWKtXfT5FM/S17c3R8fFQJLa3tzc8NNJiKJ05c2Z5YdE2DUboztY2pdTv9xfLhUg8hiEqF0uq308ZW1xeCoYiuVxhY2MDIaT6/aZlU4RwQIVBX2J4MDbUV/RsKomCLEmcJHB8pVIJBv2Gqem62dXRZRl2OBjRDIuXZAAA8wgPUcQfBJRYhuZ5riBLWJIs1+M8YlcbwDJd1wUIM8ABhDDHiaL4T8k9Sj3D0DStbhg6IZ7rOh7xRFGMxWId7R0d7R2hUMinKDzHAcA84lJCIEIAQI8QlxBGkedR3TQty2KUupbZqFaKxZJtEY9QDHnGYCs157quoZuGYdi2rWmNarVs2TpCzPMcRilgTJZ9qVR7KtUuyyoh1HUcBKBpaIaptfRJnudpmoYx73ouz4mMQUYhgpgB0Ap0UEpFiQ9HgjzPe9Ql1K03qtnsoa41CWVYUaVgtGx5OBQ//czHP/dz3zzz2HNQkgOh4Mnpyd/6V9/6F1/6eG8mrghgfaX6x3/ynW/+6u/87evvVRjfdXzm+NnzTz75ZG9b8v/53d8ClSNFRpA5lmtzSGAEcph3XAdCiCBCpydneMjtbB+sbu6MHp985UtfRqJk2e7a0jLwqEcI4hGGADDXA4QhABByPZdSQijxPOIRgiDkAPSIRyhxXRsBkIiq6XisWmsQykReUmVVt0wgcJpje5C5lsMDjhLQqDbiPn88EAjwqsxLzKMOoTXDtigDCDFGIGQQEAZYi+WPMf7jP/pPG5vbEALbdjHCGGGR5wAgtm1BiIPBCEZis645hsUBgAlBxGOuA4gnSZIaDCgBv25bpm35AqrpOtAXwIFQIBLlBD4S8gkiJwdVKRAwTWDbWPFFOEmFAAOKmAsciyCewyIGgGBIG8WKZznE9RjxAHMkzHjHQ5bpGbVkzC9wTBAhx6NwLGxYNsI8BcjnDwKIa416pV67fe/uiZnp+eUll7Degf56rfTlL70yPDzcbOg9PX0iJ+7ubnf2dDmOtbq6Ojx4rFws7+4dpNPpcq2WyxbDgfDCwlKxUkU8vzg7l9s/NAyjUqkUS3lB4Hp7um1Lt4wmQAwJeGT82PTpk2+/+05PT08kEllaWpiZmlYkeWdna2p68t0r70IMnnnhORfQ7f29oeHBjc31SCTS29tbzBcnRsertYZhO5VySRT4WDxsu9bDe/cFLPC8sLuzd+fOPYRQT2dXo9FIJpOEkJ2dnZGREc0y17e3+gb6b925jTG+eP7C0sLiYP9AMha/du1aX3//zTu3b968OX5sdH19fW9v7/z5c5tbG6IgjI6Ozj56EImEIpHId7/7XUrB0WF2cWl5bHxibX2D44TJYyf2tg48imR/yNBNQzP1hkYIrVRqfX0Dm+uruVzOdYlh2QxSyLH93T3ieBjgYrFoWoZhG8lUWvUFTd0ol8uI58rVSrVcC/pD1UqpLZXUTEP1BeoNjTHm90l9PV0bqxuhUMTvVz8a+XqEUOq6Ls9jo9kAjHR3ZRRJjkQihFLHdRkE4XBY15rpZCoWT16/eQtwXDSRrDU0XhLX19dlScoeHSzMzY8ODR3t7Tma5mha/mBfgjAajogczwhZXV4xdSPT3rG9uXXy5MmW8vDixYtra2vbu7upVOowe3RiempsauLq7Ru1Rn1s4vjB3n46lrh89uz3/u7vpsbGpsePN6u1gYEBANDD+YXTF8/OLT/KVwpjE6NbW1uRcHB6avLR/Xshn+/0yVNHuawkSUNDI0eHOdN2PQhNwHomx3RF2K6WRFWReM5uGtAlHPPaYzFIcKlQRQghDroAWLYbluTuRNwxNKNRl7EIHcZhCCChlCqiJGLuzocfAh5LggAgIox51HM8AqGoKBFFCXkEVmtapVat1WqNZs20dI5HGEIMIYKQeJ7nuoR4lm0YZl3TK6ZVb2rlSiVvWhqHMY85QKGAJUXy8UjRmna12iAeghC7tocgZIwCQFsq5lSqIx5PQyBwnNDW1pFKJVVV5ATC8R5EHoAeAIQxSgkjHgSM4ziBAiYpomFohqkVCoXd3V1dtxDCAidCgFy35c1gjLFW3YkxrtfrEMKWb4JST2vUOQ5lC4ebO2s7u5um2YyEIv5AxGKcK4SGLz375Ge+Nv30J5gvvLGzO9Tb+dv/8ud+4yufemG0eywo6ln7299+/0u/8u/+4G9+lBXDPecvfP5rX/Ss6q99/YsAmJijmOc9D0MouYRijqcMOJ6Hdre3X3v19YZlF+s1l7IfvPZqXdexKN67dw9AgBAyDIODEENAqcsB1EIZfORp5XiMsUtcx3Nady/P8wyAAASRoL9SqxZKNZ4XietxPE8B43geMChJCqUg5A+1p9JGo/nhlauHh4cMAEAZz4sUi6W6RgEEjGGAKKCUUsYgwtDv82cyGdM0GfsoLUMooYBRxnw+H0Ko0ajXazVIGY85z3EBhYQQSggAwPM8QKikKoGgjxCiaVq1rodjKSUQLlSrHOY5TlBVFSDICTxxqcRLoizVGnWXAYcyRQmIguxZXrVULpbyzUYNUSIjJEAiQGJqZUYNBOy2eHBooHN4oFNVeFHEptkMBv21RpNSBiGuVqv9vX1+1YcBfOmnP764MDc1NWXb9tLS0rnT57/3998vVSufeOmnHz58yAh98rHHH96/Dzk8fnxiZ3OjLZk6fnw8W8ifOXX68sVLh/sHU5Mnksnk7OzsuXPnwsFQNp8bGT3GIDBtq1ytJNMpThZFRaxWq4e57ObmZiwWq9Vqu7u7kUhkb2/PMLTx8XFd15944glBEN5443VJEiqV0qNHj6anp7e3N3d3d8+ePj0/9yiZSKRSqb29PUVRDvYPJUm6ePFiuVymLo1EIoIkVus1SZLKpVLuKIsR4jju6OjI8zwAQL5YaCX0N7a2p6amDc3I54tPPP3Uzs5OZ2dnb2/v4srysWPH/H7/7Zs3L168yABYXl4+duzY+vq653mf+MQntra2AoHA6dOn33n/3Ug8FgqF3nn7x49ffiwYDiyvLJ46c3p7e9vQtekTU8T1Dg72zp4+szg/Z9pOIpHYO9wjxB0dHd1YWx8cGNAaTcdzM5mMaTv5fH5oaIgQ0mjUk21pjxJBEEKh4L2H92KxGGNsYWGho6Nje3u7Xq+mUqnd3V1JkgYHB3/8xpuqquZzuXw+n8lkGILZfK6np2dpack0jJnp6UajEQgE4vF4rVYbmxh3HOfOrdtD/QOKojSatd7eXlEUPc+bmpoKBH0Qo4mJies3boTD4eOTkwTAdDpNAHv11VfPnDtbr9fn5uai0eidO3eCwWCL+dzb1X1saGR+fn6wf2hvb292dvbUmZOuRzY3tj7+4ie2t7e3tjbOnz1z+9bNbO6os7N7e3vXsb0nn3zytddeiyfjg8eG5lcWMp3t5UJxe23j8cce29/fv379+sDAACFke3v78uXLO3u7SijQOdhnYyTEgunhvrxR5wOqbls84kOKr1oqMwYdl0CETNsSRT4UDriOBSjLZDKiKCqiBAFljDAAbNsWOB5BsLO5Bjy3XC5BQDBkgDKREzyPISj61GAkEksm0x3tnR0dmXQ6FQoFAaAMuABSSl0IAcdzGKIW8NU0zUqlUqvVbNt0XEM3645rAkggpAAwVVWTiXRHe2+qvSsaiQeCQYSQYWjFYj6bPazX6wAwSZITiWQsluQ4kZCW5Y0aptbUavV61TC01rj1J8hJjDH2+4O27TqOEwwG+/oGkslks9k8OjoyraYgcAgBQghjlFCiaY3NzU1ZlhFCquLHmK/XmxwntN4OAwFfd1cGMNJo1BFEkuSXgjHC+zwl2DY0/txnv/ixz34mmk4XysWQ3/eFz37mM598uacrI0qoULX/99+8+ekv/PLv/dF/nl1Z/blf/MVIPPLr/+yrgLg8AK1QEYc5CAEEgOc4tL679xff/t89E2Nf++YvjoyN/qc//MOurh7MC3cf3Aeey3McZoACDzBXRjymhHouRggy4Lmu4zqMffQhAQYty1pdW97d3uYB6GxLO47baOq6ZkKIPUoARhhytu36QmEC0fvvf/DDH/zw2rvv8zwvhQIuhIIkUgIALx+UqvQjAj2DDCCEAIIIQMu2XnnlC/39g4RQgceO4yHEeRQBKHiE1msV4tgyjyCgLdwCBYAATBFmDHquTR0bEkeR5FgkKmHZ0R1KYTgUkyVfPJb2KyHigIauAUB5TC29RmwtFPapkaDo95eqjVK+7JoOcyjHcYGAT5YxsRtUr0nMCvu5ZNIfjyltbaFEXA2HJdXHl2sFXkCCIFiGbRqurhkc4taX1xzdigYDm6srHcn03IOHpmlPjE+9++6VcDjGI/G9964MDw/zPH/tg6tTx6dEWVpeWz51cqpSydVrlbOnZ2bv3l549KAtHd/aWNHKpU88/8L8o4eCLFy6dOlHP36zPdOVaGuvaVpF0wEvPpiddx2HGNbRwX44GNAaNQBoe3t7IpWUFFnX9XK5vLGxsbK0fPH8BccwRcwdGxn90etvTE5OdmXS733w9oWLZ1xbLxfyL3zsxcGhYUVSBSxfvX5jbGxMkoRCMd+eafMHA9t7u7Is9/f2hlR/yBeIBkMSx7e3tVFKA6FQJJbwBQOVapXjecRzm5ubM1MnLd1yiHfy1KlGoyHLcnt7e7lcTqQT5y+e39nZ6cxkGCO7u9tnz56u1KuVevmJJx6rViuBgHr67MyDh3f2D7bHx4+9+95bnZ1t/f29t2/fSiRjosS/885bn/nMZ/x+//zS4vmLFxuNxsMHDy5furS+sppKJLu7uxeXly3XGhwcvHXrViwaaU+1PXz4sKevWze1eqP5wvMfe/jwfqmYf+zyxbfeeiuVbEu3Z1bWV3p6umq1xs0Pb5w+fbrRaHi2Iwvi3bt3u7u7u/p633j7rZHRYxDCN179YSwQUnnhvXffmTp18v0P3qvXKv/8q1+5f+fOnQ8/nDo+cfX99xKpJMPgzv17py9ebNr2g6XFL/yzn13a3Z5dW5s+d/bG3bumZV1+/PEr164ODA0ODA1qhj45OSmK8sHuQW9Xtyor2cOjFvy5Uan5ZcmoNzmI/D7f3//Dd0ORcCQZL9VKmb4u3bZ29vYGBgai0ei1Dz64cO68bdsMs+mTU+VCvi0Rnzg28vDe3eOT46fOnJybna9XG73dfcvLqwOjo1IgML+6GmlL+uJhKRWC8fBWs9agNBiN80wKigFO5Hi/IPokyzIg8Yx6HTBkeQSKoi8ULFfyCAHEYct1wuGwZWjRoO9ofw/UjYCklAu5Yv6A5zDxqMApjCHPA4BhUZAEQWKAUUBd13Ecq16v5vNZ09QBpK5rMUAgxLLoT8YzXR39mUxPMpkO+gOyxPM85TgXAJMyEyICIISQpw6FENbrpUqlsLO7mc0d+PyyIGJdb7iuRSkBgAEGKQX1ul4s1JoNw7Ycx3E0rVEqFWq1ouuZEDEAGARYkXwdbd3t6S6fL0AIMU0j4PcHQ4rj6qVy1jCblLq2bTYaNUmSUqmU6xCRlyDA4UCsu7M/HIr39x0bH59KxVLAJQKC1DZss1Et5WzbAAjyih/4AlTxp0cmnvzs53/6Z78+cOGS1Nn+5Msf/6//63++9toPbr3zzl/8x3832dl+5Uf3v/Gt//St3//jcPdAvK3zd37lW/X9XURN1zM8z0KAAUpd10L9vd2vv/GjxUePJH+A5/lkKl2p1DgsrKyuGpUy5HnMcYyRVmSmVRS7rgMggxAKvGDolmmaK6srh4eHR0dHmUymuzPDAfDM45csy2pomuM4LUyjpRsix1NK/+zP/8e777yPIRodHX3uxecfe/pxLAo29VxCXEpdLBYbOgMIQkh/ktkEALiuK4sS5PBb77zDYdSi0HmEYoQ94nEYtyKPAACEoEcJJ/CEQYAwYNDzPAgxz/PMY55HHdNTBAU4Lkb0xMQIQrBpGQ2t2dQ1hJDl2ooqpNKxQNCvN5sHBwf5fF4SuVDYDxkVBE4SOeIaplbmoJOI+kJBMRL1h0L+SCIsKYLn2palA0g4DvM8ptTLdHQ1GpqkyJwohMLRjq7OaqPe0LRStTJ5YlqSpGaziSBXqzYqlUo6nTw6OpBluaMtY5q2Tw10dHQUirmRwaFqtfr22+92dnYmY3HLto8dOxaLRBYfzY6MjGCM7969+/jjj88vLuRLxfHjx2uNhmmag0NDlNLx8fEWXaB/cDAUCd9/+KBULVEIFtdWJFne3d0NRyOapjUaDQDA2sb64NDA5vpGNpttb2+/ceN6e7rticcf+9GPXt/Z2ZFldX119bHHHisUCj5FvXzx0tbWliBJk5OT5XK5Vqv5FHVpYaGtrQ0L/PLKWjAc3j88qDXqgKHDw+z29napVKKul8vlYrFYuVyee/Soo6ODEGJZ1t7e3tLS0oMHDyRJKhQKm5ublmVVatX29va1tbV79+6FAr69vb35xYUT05M85urVal9fz/7hgW3bH3v++fxRNhwIPvfcC++//z4h7qlTp27duj0yMtrV1XX1/SuDg4NHR0dXr15Np9OCIKyvr164cGF/fz+Xyz311FN3790BGMQSye9+/x9HRkaGhwcPdvdefumT+WKhWq0+/sTlDz+8JmDu+ec/dv369XA43KImTE9NLSwsVMuVkZGRrd0dAMDJkyd1XSeEvPjii4uLi8ePn2AeW1/fnJiY6O7u3tnZa/VVZFn+5Cc/+Vd/9ZcbGxvT09M//vEb09Mn+gf6rl77YHJyQhCEcrn85JNPVqtVhNBAX3+j0SCEjI2NHR0dqX5fLBYzTTOdTgeCPkao36cqooQQSiQSluvcvn1btwyAYFNvpNrS9+/f397ebkul67UqoCSXy62sryWTyb2d3ezhUTwaa1Rr+VxusK8/FosV8wWe55vN5uzCfCyVLtXqc0uLqe7O6aceqyEqJiMVx7AAJRznUeISr1ythKIRxKgkinVDq+nNfD6PMR4eHHIsy3VdSZIs3bAMkwGPus7uxrqgKEG/ChEqFPKu6xJKGAMtjRoACADIGAQMAQAsy3Ecr16vHx0d1WoVQlzPcyilHiGu4wIAERIA4zwPOLbb0Gp7B1tbu2uVasH1LM8zAaKIx7ZjBoMhCGFnpuvYsTEIoWUZhHq2bXqeZRhavVG1bRshxHECz8ut2DQAwLbNer1aKhWKxXy5XG45uRgArucxxjjMSZLgEQdzTJYFWeEpcwyzASDleMQAaYX9RVEBDHoeAQAKghwKhR3TJJ7HAMEY8zxnGHqLaHZ0dHSYPao0GowTgag4SIb+6OTlJ04991z35IQ/mWjrzAz2933x6z93ZX7u7//2f05N9n94feFP/+S/vfraG7pu/MV//2+7m2uyiD3XdlyHMg8hhGqF4m9+61e/9M+/blkOIaTWqFMAREUp5Oory2tA4BqmDiHEmPM8QiGklAocT4hXb1R3d/cXFxcxxj09PQih7q5eVZWp5/AAtMfDsiwVy+XWucwhLIqibdt+1Xf2zJlnn3nmiSeeiKXiFVfLNkoecQWOtzzHwcDlec1huucCCBljqEWv/AmGPhaLybIMAGu99fMYIeqJGEAAGYUeAx6AJvEIBJbntng4DAIGEWXIJtCjvGVRBPlgONo91G/mtjcX7/ACa1oNA1j+mN8fDFBKm1azWCkUCgVLN3nGFA5BallmzQEWEhgkliKQ7kykbyDZ1h0Lp8PBUMjn8yt+v+URxyUcJxiGyXMchsw2DU4QIM9hWbSAV/esQr2e7ukwmediRDjECThfOMx0tieSMYc4lmX4fYrAoWgyeff+g93tPURgpVqHvCBL/lg4kUi2uQx6AGeLFQhQpVTd3N6dW1gcHOxfWVlKp5OebW1vbkRDfubZ9UpxcLA/W8g6nnvmwrn7C48qWuPEmZnN/V3KocHR4Yapnb54rrM7c/Xmh31Dg5FEnOPh4GC/6pOJ6/X1Dkiijzju7IOHwUi4UC5KEjc8PLiwsODzBYrF8sbaxrkz5/V6Y31p5fKFi+Viaf/w4IUXX5xdmI8lE2Mnju9lD+PJdK1SLxxlT4xPhP0Bv6S0tXUQRgulUk9nVyQYEkUR87xmGIODg6lYIhGLDw4PKj7l5KmZM2fONBqNSqXa2zuQSKR4XlRVH+V52R8IKgHXdBp1A3HC7Nz8++9dSSbTpUL5rTd/nEgkfD7lvbff6s50b27smKYpKtL61sbY2Fh/38DB7n6pkAeQvfXWW/19A+FwdG15pbUztXuw35JgbG2sG1rz+vXrfr+/XK3cun37woULhqbvbe9cuHChhY0cGhoqlUodyXSzWpNFMeQPWLbNeFw39VKzduPWTb+sVIvlpm7t7B8dZPOAE3lRfu/K9Wg44lnuowcPRgeGetrTxYO9eiHPdH1rYT7AYa1SU0UpqPru3rilNZpbG5tbG5t6U2s2mxsbGx6jxUq5qjVs4hEO8pLYomiVSsWNjY3Ozs5ms9nT02OaVqFQEkW5Xq+OjY2l0+lms+mYliJJ+WzuYHdPM61ILL63t4coq5VKxVxeUpWDw2yukNX1ZjGfi8cikXikYeiSpNoeXT7aJRH1iBolnuSYVWWOywCg2KeGDg+zgXCk0Wx62POAJ8ti4Sgri76+7kEEmaU1fZISCoUAAJZrPZp92EpYR+MJfyio22atUbZdgwHaGkUChhgDlEKek4PBWDAYjkajoVDIde1SqZDPZ2u1CqWEFwQAMfUgYDzHKZIYUJWQJPlc163VKuVKVtPLjBqWWRFFCQAUDEYVJWBbnsDLsqzqul5vVCxbh8jDHAXAA4C2ohOq6m8F8yRJ4gXOcRyXeLwgMMYAgJQAnuN5jifUBZAy4EIIKPMoJTzPQcCo5yIAbdNyHUfgeUYpYwBjnlHAcSKEUJAEQl3EcRQAk3g1zTA9J5vLlcvFgE8FlLiOA5AgKAHBH4dK2JN9GuKYIDRd63/93be/9Qs/9xtf/8rh0fYXP/HCSFImFaeSK7769rvfef21X/j6v3jtO38nSQLH847rIozR22/+sLerM3eUredyrRuMOEQSFV7Gb7z7NmC0tTrkeA7Hca3VUF3Xr1y5UqlUurq6BgYGtEZzf3+feF6z2SSEyKLgEj2pCt2Zjma91mw2EQOea/McIp5DCBnq7QeE5rM517MJ8CzHwBARx8UCTyACWDA9WtdNBj/yK32EjwCQMhoIBBhjlmXxHEIIMMAgQsR2AACKoiCMGQQQQowRhEAQeAgY9QhCyCXMdZgoitFYzB9QgdXYuP3+h2//oJ7bDSgt+SUPIKkUCrau6U3NsiwOYVHgBR5Fwz7KTEGifhUrEkolYx3t6UBQ9QUVyCNZlTzPq9WqjVqt2WxW65plU46XHdvVahqPsGfarmU3Da1UrWxub5cbtb3s4eLKMvHY1avXKaWDff1zc3OyoqTT6Vu3bg0NjoTD0R/84PuPPXZJEoSNjY1oNLq8vFyrV9rb27/7vX8sViuBULhcqzZ04/yFi8l43K+o2UJeVdX29vTwyKBnmYoodLSlWwV7vdnwh4KLK8uyLA/09z948GBq8oTAc1tbWx0dHTs7Ozdu3/rZn/3q9t627Zj+kP+1H/4AIdTZ2bmytDowMFCt1svVSiwcSSRiuq4DQFvrSLIs7+3tzc8tWKYdCITu370XDof7+vruP7jb3d195cqV9fX1sbGxhYUFy7JGR0cfPHiQSqXa2joePXrUQrm1ZrCzs7Pb29u9vb2VSqVUKjHGrl+/LgjCwcHBhx9+GA5FAcSNZlOWldxhDiHU1dv3vVdfCwaDsXCk1mwODo889eTT6XS7qRvJZHJ0fKKtrY24Xm9vr2s7oUBQVpRQJJxKpXb2duvVRl9f39ix0bZUcnR0VJbler2qBvy2aYZCoZ6enoODgxMnTsQSScdz4/FoKBTq6+lNxOOlUikcDqfT6XK53MJg5UvFUrnsUdLW1razte04js/nKxaLrShUKpXq7e3VNM12nHgy0TT03f091e8bHx+PJeIMAISQ3+/raG+Px+OnZ056nteWSodCoUajoWlai0DrOE46mfL5fN3d3cT1CCGDg4OFQqnZbCqK+uDBA0lSenp6FhYWMpnMU089+eGH1xKJRFdXF4S4v79/bGwsn8/7fAplHkLgxIkpCHBHe/tjjz3hWraiKOfPXyxVK8PDwyNDw/fu3QsGg6dOncrlcn19PX3dXffu3ZMlCUKYyxX6RoapKp598UmUCj3Y36pBzxM4UZFL1VowFDNMUwmqlDHTtTRNU3zqxvYOxVBRFEHgWlBZ3TBC4eDR3i5wCULItG2O44JBfzAUYMAzrKbzUdcFQoAx4illAECBl8LhqKqqgiDFYrFMpjMSifA8ZpQAQCFiHnGI5wEAIeCTsbae7qFQKKJrZi6Xq9VLut5oNmuM0VZ56vOFOU5iFEUjMZ8v0Ko7CWlFpimEjDHW4uhxnAAAUJVAZ2d3R1tHS/zkei5EjLJWewCZpum6rmlaIi+34jG25VQqlUqlommaaZqWZUGEIIIQAQiZ6xqMUcd1AEIMIo6XItFkJJrkeLmrp3diYsLn8wX8arVaNgxDN4xmU+ew6PNFo8lUOJ4Ix+Kf/eIX/9Vv/V9PfeJjR9U84OHv/O6/+Z1//c2AKADLqBUKD+7d/fJXvvK5z31+/+BQFGXH9dD4+Eg0EtjaXFeCAUo9RZJ5BontQJ9049EDADnGmMsAw5xtO9mD7M7mFiHkzJkzyWTy4OCAEKLrhmVZfr+/FVh2bEvBHARgsDfTqJYsw2CMSbzgOS7GmLi2Y9vM8QRBAJ7LM0/hEQKU53nXdSilBAIXwlJdIwB7ntfSWLemz5QSSZI0TfNcGwDguA4DiBCGRYUBhjjcGpxiBDkMBR47tkldBzLKIRTyB2LJpIhQfWftxg/+9toP/2Zn7iZvVlOqhG0PedTWms1iQaAspARUQVF42S+KPhH5VWyRuhoW2zvjXZlUX0d7RyoZCkZ8kRjglYZu1xuG7Tq60cwdbjdrJcQre9nGzlHJ54/KgowciimNhsLhYLCvt3tmarJRKSPCnnv8SaJbk0OjRs3a2jx44ac+nivkt1Y3v/L5r7773gebu3tfeOVzt25e9/vlyRNjuVxOFPlMpm1hee5zX/zc2Injt2/f7uzsFP3qu1evKJI8OjyyvrYZCAX39/eXFhbj0Vj+KLuztW2bVqVWNQyDOLZeq8mcUC9VsEOyO3tmrXGsp39zda2rvWOot//q1atnzpzZ3d8vF4ovvfSSpuvFcunE8eOPZmcpBGMT45VyOR6NVmrVcqNmWVar2zY4NCSrSu/QQPdAXzAc6evrq5TKnu14ljU8MNiZbl9dWOpoS3ECv589uvj4Y7ML80eF/LPPPru+uqY1GqIi37l/L9PRyUF8++ZNy7KwyOeLhReee75Zb9QbzRMz0/fv3xdFMZVKXb9+va+vr1AozM7OTk9Pl1sIndHRlZWV9957z/O8YrF4cHAgq8rq+hqlNB6NLS8u+f3+ZrNJGSCMtkQuCMC1ldX5+flAKHjv0cNyo5ZOp4uFiq2ZPAAcYmsb67bjMYQ7OjONemVna8OzPb1pqH4fL/Ge7Qz2DyCEZEVJdrSZnrO1tzs6PmbqRqlY7Grv8Klqo9qIxxLvfnAllkp0ZNIH+zu93ZlUMrq9vRmK+pfXVk3H7OjufPDo0Y179yjHPVhe5lSfEos/WFpp7+mqNOrXb92cPnOq1fpIplNzc3OpVGp8dOze7XtdHZmArFYLpfHBEeaSq1evnzt7/ih7cPfu7ZOnZvb2du7cvtfb0z/3aP7+/fujo6N37tzCGCYSie/94B8FQUwmk9evXhseGM0fld565+2Tp04d5LI3bt18/MKlSCB4//7902dP3bxzM1/IvfTCC5VsNhoInTl1upArhiKRumc1MU2MDh85jsWzQrOGOFFSVA8ywzJFRRYEgSDQsO2G51YNAwsixwmtkhHxnCzL+f1DADD2qMTx1GPE9RhzMWaCwBHqeJ7baNYQAgwQiDgGGMIcYEhRguFwVJL8AHAtZbzjGrncTrWWw5xHmU2IhxDHgMBzvni0sy3doyohrWlwHGeYTV2vE+IS4jFKOU5U1RBCgigojCJCWMuxTBlxXZtSBwDqOA7muWgsFQzGEBYJhQAgQgiEFEEGoOd4hmFqlmU5NuWxBIGgykHqIUlSJEkRRTmdbm9rS/v9aj5/ZFk6A57tmryAASIO8bAgYiR4ABPGyWogne70BYK263nUtV1LUSSRh5A6hzub+cP9crXQaDQlTg1F2iGWgrHU6See+uK//OW2yYm1QpYTwS98+bNj8bBiaIlEXAoF33zj7e5M/3/4/T/ikYzaU3FRwMPDw4auSYps2yaHoaZpkOOX19e0Ws0vq4QQAKFlOaIoZjJdfr+/9Y9kWZZlWaqqjo6OhoIRzEGO5zDGrm2JAIwO9hPHLBWKnkMYYxyHWn4QQgjCADLCAYB0gzctHkLP83iAeAw9z2MQ5SpVCjjE8egn3PZWnwsj2N3Tubq6SqnH87zr2hAjBgChlALGcZwkSQhi13JNTRcQDPt9kWhQFSHVSiu3333v1b+59sZ37Mq24NaDEuM80zN017CsepPzGHM9QD1ICUKIF7BrN3kB+INCpjOV6W73hwKKJEiyACG0HFs3LdO2XYcQD1CCPY9zmeRxgfsrO+/cfWgA0cECw1wLCEWp10IQHxwcdHZkFFF6dP/hJz/2caOqhUOh9vb2d95/B2HYnk4tLSyOjY0Rxj68eePZZ5/lMNre3n788cdlWbZN86d+6mNLy4sLCwvPP//so0ePNK1x+vSp+UdzR/sHn/70p5u6mWxrTybTuVwhEU/5/aGFxaXLly8LgrC4uDg+Pi5wvGPZTz31VCIWB5RZljXQ118qFFt6wtu3b3d3d0cisQ+uXGtlM9ZXV/q6ewxLz2azvb2962ubit8XT6bn5+dVVTUM4+7du8FgcGlp6f79+4qivPfee8ViMRaLteRNmtbo7es+OjqKRqPBYHB3d3dmZqZcLr/2+uvnz58XRXFtbW3m5MlyudzW1jY9PV0qlXRdj8fjb7/9djqdDofDCwsLL7/88tHR0aNHj559/tlyudzd3d3X2729s5lIJZNt6StX3k+nkydnpnd2tqLRqOr3rW6s9vb2QggL+fzly5cXFhYamt5sNrd3dv2BYCqV2NrcDIVCmUzX3NzcydOn2tvbVtfXuju7KpWabmi9vb2yLB9kj05MTX344YddXV0T42MIwEwms7qxfv369fHx8WKxGAqFVEUpF4qU0qGhoQezs30D/YFAYGtrKx6Jtly4U1NThVLp8PBwZmr69u3blNKe7u7bN2/19XRblrWwsDB9aoYQsru3l0qnc8VSsVCamDw+N7/Y1dszPT195cqV3t5eSun29vapU6cQQvv7+xfPX+AxV6vVIpHIxsYGQtzIyOjBwQFjrKs7s76+PjMzE41Gr1+/Pj19MhaN339wr6u3CyF0eHj42GOPLS8v7+8dPv3kM/Pz87KqDA8fu3HrZjqdnpqaWl5eRgi1p9v29/aOj4+psrSyuPDpl17u7uyam5vzyYqh6Y4HJk+f1RmZvHAGBgMlx5RjIZsRQVEpQNVqnRBmmDbC2HSdqtao1BqW5QBKFUUizLMsy9NNkC9iQWwVv633coiY5zkYQ4HnTFMrlYuu60DICCGAMYQ5RhmjiFHo2B7HCa3ei2HqpXLWspuEWq5nfnRKUASBoKrhVKrD74vYtsdxnGEYtVrFNHWPOJSQ1hEvinIwGFaVAKPAthxK2E/aMmoikQz4QoxBQmnLBccYE3jOtm3dbLqu7XkOx3GRcESSJJ4TAYMAYFUNBALhRCKdSCQxxoR4lNJ4PKppDcs2IGK2q7ueq8g+QhhhDFLGQ8xDzCghjosxZpQCADRD397edhwnFotkc4c7G5uOYdqG7Wqm7Av7/FEKRcZJU5ce//I3f+HYzMlwMvnMM890pBL1YjERDCqC0NvZ/vu/97sff+FZ9M6rP2C29dQLzwAMApGww0jN0qHIY1483Cmsz84Bx0Ue9YgXDIViqSTkMIQwFA53dnV1d2UyHR2pdLqlKm31wTGHEIQYgJGeNoE61WKJEAYAsizL8zye53keI4Q8z/EJwlPHJxOQcwzTtm0RQRliDkNRlgu1pg4A/Qnkq3XEW7aBAehIp956601CPEqJwAsEEJe5AAAOcQAgXTNth4qyL5FqD4RjCLL8yqM7r3/n1g//8uDem3Jzq83v+DjTbOZyB7u1Wo0wiBBSZUVCgog5NSADgTT0KgV2MhNJd4RVhRd5DD0mcTLFQtOyi+WC4xqAeZVikbqOZbh7O8W5pexbN7e/f33lytL+xU9+9uNf+1mNw01GHcQoh0RZ0k1zZ3dfVVVZFAKq78T4xP/5m7/pak8T16o0CvFoqL+/V1alzZ01SpxoOEghuP7hjXqlWSvWr314ixMUnpPeeevd9nSqtztz/cMrp09P9/f1Vkrl0zMnRUWenZ2bmpqee7TA8eLE8SndsFVfqKd/4MrV6+n29v6h4e9+7/ucJBDIrt260bSMfLW8tLYKecxJgj8S6urp1gydAmjaXibTZVlOo1YPR0OVaoFDWJWVtfV1QZYZ4ObnFy9evlRtVBkjn/nsp7c3Nvs6u/u6e2ZnZ1tEF9N1eEU6yB1mS4VQOJxMJn2K7Nr24vxCrVrt7e6ORqN7RweO5/V0dlXK5WA4FIpGKrX6mXPnh/qHmvWmPxDc2N7OZrMB1be3ux2PReKp+OLygse8RqPeqFWHBvrzpXy5Xrn82PntrY1qufQzn/1MNn/UNJojI0MLy0uiKGPE3713+9ix4UgksntwOHP6zO7u7uHRwejosf39g1w2Pzoxvrm5UavVhweH5hcXOjo6Av7gWz9+O5VKpZPxxYW555577v79B0vLK8fGRt9868eyLD/x5NNvvvt2LJ2sFEvrSyuD3f2uZu9u7V6+/Pj9R3O1ptbemfEYDQQCk5OTt27dHujtb0t1XH3/2iuf/UI4EK2Va6PDo8Vckbmkr6tr/uFsV1tbWyK+NDt74cwpS2+IEJ48MSlxPHO9px97QhElyzB7urpXlpYVRanX67fu3IYC19HTVaxVJk5OEcgKhYKi+CKRSKlUikajBwcH+Xyxv39wZ2fP87yXP/0pSZIqlQpAsFqpz8ycikej77//fjyZDEYinCAEfP65R4+y2aN4MrW1s0MY9flUAUFIKM/zN2/eXN9c8/uU3OGBBIWIElmZW6OuF0zEdpr1rWYtb9fqRLdsV1ECEqcgwEuc6FiOwGPHMlVJxRT6RLlRq8iyDCFr1Jr72zsAMtu1KIKQFzDmAUMCjxFkLrGTiXgsGsoXDvOFI4wBgJBQwhiEiCMeFASVEijLaiyWCIVCkUio0axUqrlyJXeU3dG0OoCUMcYI4rAcCiUSyU5KICGktazUaFQ0vea4BmMeo5BRzHOy3xcNhhKSHJSVQDAYEySZMkQZbV0/GOEWwbBULrXsni1mmeM4uXyOUkoIqVarpmFQQhijlBLnJ/wrl3geJYRRXdcbjYZp2vt7h9mjnIAlDmIOIUYcCAgHGYbUsy1RFBXZl0ikBEXd3j+oaFpP38DJ6am2VAogUqwU64Vys6ZzTFKlsGOzUsM5dv7Jj33t6y/9/C/++r///fNT4/WtPexpPHb6e1L37nyARoeHIc9PjI4Bx9lbWkII8TxuaE1O4HkePLx/D2DMiAcoo4B4noc5BBFqoZEwxp7nMMA4jm+F31sUGowxBSwVAP1d7fVqTddNWZREURZlqWnojmlBRl1KIKMJHzg9PiQQKgsiYoh6HgKeIPE2RTXDJQB+hJIAgAImiRIFXiIZ/43f+PWW45wwDwGAICDUM4ymiEE6Fk4lo6qIGoc799569d3vfHvu2jte7Uh2G0Fk+5hhlI8q+X1i6AIEqiIxYgPmCjxyHcsniZgDsiwMD/f3D/Sk0jFJxJGw368qAZ+/UqkxxgghWOArdT1fbOgut3ZQ/eDu4u2lXRTK/NQXvvnln//tiXNPNhwu3JYJptNN03I8CiGzbdvn80uSnD08ajQahqY3a/Wezq6Dwz1NqxHiaZp25869YrnU19e3tb3hOJYoyvF4MhqNnjlzRte1w8NDQogiq4eHWU4QBkaGs8X8owcPy/nCwvJSo9FwHOcf/uEfRkfHNM24du0acWm1Wl1fWU9EY7vb28vLy5///OePjo5KpVJvb2++WOju7evp693a2fEFAoSQa1c/zLR3lgrFRqPR3dUT9AcopZFIZHD4WKatvVatDx07FggGOQ719PYur60qqppqS7/11lsdHR2SKB4dZM+dOyeKomGZ0Wh0Y2Ojvb19cHDgx2++GQwGG43G9vbmwGBfuVyOxSI9PT2KovgC/lY1ynFcPp/3PK9lqa/V6929Pe3t7V1dXRhjw7Bc11VkNZPpiiUS8Xg84PMZmmbaVqarq1qpixwPAHj11VcRQj6fb2VtdWhoyDRNSukTTzyRL+QURZmenr59+/b09HRrl1VRlLGxsfX1tampqUgofHh4ODZ2rFQpVmrV6VMn93Z2fYra0dFx5cqVkzOnVX/w+o0bZ86ckWX5jTfeGBke3dvbAxgFAoF79+75/X6M8dtvv93V2dnW1nZweNiaHJTL5eeff35ra8swjKefff7evQf1er2jo2NpaUWWpHQ6Xa1Wg/5ArVYrF0tDQ0Pzj+b8qm9nZ4dD2GhqtVrNMsy9vT1KaSgUMk1T07SJiQlRFEulUigUoZTalispCseLhmHUm7qq+DVNsw17aGgoHAgamt6sayuLK7VqHXI4GAmvrK1WajVd19vb213XdV23XC6LoihJkqIokVjM7/fbtn1saHhzfYO4nqIoy6srmmHWG41oIHS0t587yJ0Ym6SUVmrVVE9v34nJI63eZERnTLNd2afatg0xkhRZURSMec8jsqwqigIocxxH5HjP0LRqFVCGMcYQtvqoPOZbqFoImWnrtmt2dnQEQ37T0jW9SSlFGLdQYIxSCCGHOQCQT/WLospzMs/JkUgkmUyqPhnCVkuHQYQAxICweLwtmUwHg2EIoWEYjUatXq/W61XXdSFEEGEIEUacKIocJwCAAOMQ5BiDjDEIgO1YuXy2ZYLt6upSJJ/rEM+lrQtje2fz8Ggfoo8ou639H47jTMvEGEuC5DiOTw24DsllC6Zhd3R0tLe112tVTatT6kLmAuAxYgPiQQg5JECAOSx1Znonjk8PDoyEQlEAgOfYACNBEh3iNRqNWq2WPcjauicroVy1uV9tKon05Rd+6m+++72/+LM/HR3oy+8X9VploKsL/ei1V7/953/u1rXv/OCH4ydPWY2myouKKGiWZTBw+8EdQF1JkgijBDAKiEc827EhRgwCCAHGGAIIAGSMYYQpYAAgjHjAmAzA+EB3uVw2dBtjkbrU9lxRFEWO9yxbUsRirUQo4CHw8SKwXYyx7brEsz3XZrxYrOkYch91ZBhDCLmeCyjjMYKIIQwFniOEaM2GrjUQo6rEq36R1EsHD67d+P637/3ob42duThqxDiLNxue3rR14+DgCLgQexgYxIcFTDxTr3HYpkSPBKVkIhyPBuKJUDioqJKIIVJEybXserlULBYwotSyiO2Uyo1Kk2zn7Qebta06Fxm9+OLXf/XJz/9c2/BMomv4c5/7F889/5Io+3hehhAbTY3HnMxzWrUZCUTCvhACMBAILK2uhOLhvaNDyOG2tvZctjg+NjE6On6QPUql0xjjSrE0PDh0kDt6//p758+ejMf8Bwd7M6dO7u7t37r3gPcHl9e3goHIzPRJy3Z4SR4Y6IuEgobWtAw9HPAn4xGZR2dOTqWj8ZH+4cvnLqwsLGXaO3w+3+LicmdnJ0KoVqvHI7H1pdXs3tGFM2d3N7ZUUerr6f7wxnUMoSz5FxdXmw1zc3PXspyaVi83qpZtNLUqQGx9a3NlZa2nu/fw8HBpeTkSi2bzedXvxxjPPnw42D+AGasWS8ePjS3PzwFIp0+drNRrfYMDW3u7s3MPPcclrtc09PMXLqysrAiCEIvF1tbWunt7xibGr165JnBiPJ7cP8r29w0SD25v73NYnJ9dsG1PVYL1mk4JWl9Zy+YLyXRGln3HJ6bSqQyCgiSIlUolGo81tObD2dnW5HZnc2uof+Du3bt7e3sXL19KplJHuWwwGJx98FBv6CIn2o6l+NW2TKdhWF1dXa5tVculdDK1vrlFIJw6NdXQ6olo7LmnnvYsW8Iih4VIKtXe02USN5FORYKheCii15t60wiGIqrf5w8Fr127lkwmfQHfzv4WLyJe4mWf3D/U3zR0BoEvEOQEnheljs4uhLhyqQYBl0yk3/ngA4pgW2dmfnmxrb3d5/PduXOnt7e3UavPzc3FIlGjqS0vLKbiibWl5Wq53NHVUaxVdvf3VH9AkdXe7p6IP7i5tn7xzLlELLaxtunzBQzd8ggbO3G8Zmq5SiEcD+eOjhChsVBQkkVBEZNt6Ww2KwhCPBp7++134smUB9ijuYXzFy/J/oAgycl4vCOV5Km3ubbUrDcM09o7OjIp+ORXfrbouCSgVF2zqWuSJCGRs6lbrzdFUfYYMRyzXClKksQxLHG8a+p7m6sAYMw4RAF1bECZ6zkAgJaoumXUsRyL57EkSZIkOI5tO4ZHLQYpYR5CjFCXASQIfr8ai0Uy8ViXJIUwEiGAlDmEWrajua4OgAM45LkUAsnnCyeT7eFwFCFkWGZDax5mD3KFnKY3CSMAMNTST1MCAKSMtdCVhdJRLn8giCiVTAUCAQCQ61FJVFXVL4lKIpHo6Ojo6soEg35FkR3PNm0dAEqZxwuYQwIDUJZURfGpqn94+Fg61SEKMqCeJCBCbMvSPOiZtgY5jiIsy77WywSgCAIOA8wY8IjHGOYEWRQlLGDBz4kBXlKFaDQsCBJ1gcjJiqgYhlms1osmefqVL7955eY//vW3J/pGNuc20btX3r906dJTTz21uba5sbYpCYrn2pZlMQ75Asrs7Cw1NF7gbNNAgGGEOMxhjDnUaiVTiCADzHZsxhgDDAIIIGQQYsAQAH097YSQUrlqNHWMW7JziBiQJMmDDAm4oVmOSzVN4xBnux7mEY+R69oAC/lytZWOYhS2kgM8x0PEIGDMI7ZtlStlzzWDKh9URZ6ZufX5e298//0f/PX6w2tOcSfEOT5mknrZa9aqhZxerzMKYtGk6zEOcoogSAgiT49H1Fg8HEvG2jqSoYg/HAsHQ35CiMAhUzPr9aau24ocYAyUK1rTohuHlZuzW4u7ZVuMnnvuU5/9F7/yzKe/FO8brVPkIGx6wGUYIrEt053p6YaYCwbDgJFms9nd3duoatFIxKeouVzhpz7x8Y3dzcHRoUAouL6+/sILL1Sr1dv37l64eJFCsLGxMX1i8vvf/4dkMn7hwoW9/R3XdiLx2PXr10+fPn369OmVpaWZmRnPdZeWlvp6ewWOf//990+cONHQtGq1OjY21vIZtWw1RweH1XLF7/eHAsHcUXZ4eDiXLWysrcUi0fX19Ugkkk6nN9c3pqamMMa5XG50dPTwILu6snJsZHRlZTUSiYyOji4vL6dSKQKo4ziKqmYyGQJIe3s7YyyVSLdepDzP84eC6ba2RqMR9Pt9ilouF8+fP29q+urq8vGpyXyp0NTqU1NTG1ubgiSOHDv24x//eGRkpFQqLS0tnTt3bnVjfX5+/rnnntN1Xdf1n3rxEx98cK2trWNgYCCfzx8bHske5Wfn55Jt6XQyJYtKX1/f3sH+weHh0dFRNpsFAMiyCgDY29sLhIJtHe3hSDSfzzPGLMPked52nVqjbtpWQ9fa02kOIUEQgsHgzvauJMktYzKlVDMMQzdTbWlNa9RqFdO0q9UqQ/DR/OLq8kowGDQMQ9O0tkyH4zitZPf+/n48Ho/FYoeHh1jgW3AbAli9XrdtO5FImKb57rvvhkIhx3EqlYrP57NM2+/3t7dnFheXx8fHA6Hgyvr6888/f3R0dPfu3RMnTszPzxNGH3/88ffeew9jPDk5ubW11dPVnYwnFucXZmZmqEeuXLni9/st015eWunp6bv34OHu7u7UzIm5+dlW3H57excgHA5Hl5eXU6nE9KmZG7dvnDw1Xa1UDvcP2lJpWRG3d7ei0ahp2g/vPZw+MUUpzeYLn/qZz129cVPx+8Ynj1+/cct13bNnzxaLxWg0PjQ02t7e7kF2UKsEujJrxZzJAzkY4HjetKxKpSJg1KhWREnCPBcI+D3HpoRQx/PLkmXqwCWYAQQAz3GiIGLEAwAII4QwQhhjUBAkx/FaJZ0gcByHms2aptc5DhLqQgg5zLsuBYwnHgIAey4khDmOZ5pms1nXtJplaa5ruJbG8QgASgllFPp8wba2rva2rmgk3tvTm0gkFEXBCFNGXc8FAGCEEUSUkkKhcHh4KElSV6YrHApbttVqIbR8EhAgSVREQQ36wzwvEEJM22z5LcrVcgs27hGPMYYRzxjz+Xwc5k3LZIARz2uxix3P2dvb0wy9VCse5bIeIZRBCDlGP9q9J4RwWAAQUgY8z22VKQhBz3Mc4iGecynhRMFjlOM4yInY598/zFWKpcsvvvjq9etvvfdj1NQMD7Dvvf4aFgWzVJUlCWEMOezYXigY2VzfO9rfwRRIHE8cGyPkuS7GnAcoBQBhyCj1CBEFGbXC5oARQonHEIACAOMjQwzBalOD8P9P1X9G2ZZd5d34WmvnvU/OdU7ldCvenO/tnCW1JEASAmGMhV4RBBgwGIxt4DUvwQYMGASYKAMSQgK11Llb3X1zDnXrVt3K+dTJcZ+dw1rr/+E0HuNfo77VGLtOjRp7rrme+czfAxVeZgjCPnWw71DsuC7i2PXdTUVCgVjEJj5kEACAYyCLEMfLTdWwMAEfrtVSURB94mLPVdVmd5chFosokujW9jeuvHX5G3+zcfO7jLrfEyApCcRlSBzdsQ3dMnXDRgzP8wHHJKbhIl5yIWB4yHFeMiGNDGci8ZggK1BkgYgwgJbt+Zgalqdbtu0S1wflaqtc7uTLnffvrDdAbPqZ7/vo53/2e370S7mJGSAruus4mARCQSWiROOheCoNeD6RzfYPD3sEYwgIBMFIuFAo9eb6LMtCFAiS+Npbb84cnt3Z3ymUSmNjo2+/9XooKE/NTF67eSMVT01OTCwuLn7mBz6j6tr1G7d4RgoFY62WOjk5md/fu3n18mOnTpT3dlzX7u3LVQqFVq36xFNPX7t1S1KUydmD7124GE2mO7abL9cy/b2Gay88WjJN+8GDhxwnOKaVTCRikahnO325XgAApiSciOXLRTkUtF1vY31zYHhoanp6Z3trenKirbXvzd09e+pssVjyMWBFaW9vz/Ocw4cPbe9sDfT2QUobjYYkKdV6LRKNQhaqWrvebEqS1N/fv7GxFgwGz58/Xy4WVVXN5XIbGxs9PT26rt+7d+/8+fOlUsl13SNHjhQKhbGxA8eOnfjg4qVYIhmOxr/2ta89//yzjVpla2N9ZmrSsc1oLHjszIk79+/wCE6Nj73zzluDwwOJdKKlNlPpBPHdzbX1nlRa1dsUUVEJbGzv1Ku1XE/GMnWOYUU5uL27F03EJicn5ufnHdtlGGZnZ2fywJSpGgFBOHro4MraaiAUkoOhK1euHD9+tD+XXX20+vTTz2u67hD3iWefLlcrgiAk44lHDx72ZDLpdNpyneEDY8Vq2fO8TCZTK9cQYsfHxxcWFhiIpien9vcKiqg89/RzN69dP33ylN7RNtbWjx07ViqVbt68/tJLL2xtbVUqlSfOPzZ/f25yYuLQoUNzc3NjY2OKJF+/fv3FF180bWtubu78+fP5wr5P8LETx6/fvEEpPTI7Q23v3KmzuZ7eK1dvHjpyDIrc/cWHo7MTUGDuPngwODwaCoQePpifHj2wvb6+X9p99vmn7927NzU1cfrk8VIhH5SVRCJxb+4+hPDo0aNXr14XBXlgYOC9i5d+4N/+G0z99y+89+kf+Kzp+7fu3f/UZz5r2X5+rxAQZYw9IrAooowenZWS8UK1JociSigUDgR5AJKxaKvVFgTBtm0GAezYLIDhUGjl0RLgEOLZLivQ9x0ACYIs9imCLIIswQD7VBRkz8We5wFICHVDYVmSmUarpBk6hYhQwnEcAIBhWd8nHMchhmU4nuN5lucoBD72PM92PUPTa4RaiCEUEs/HmACWFWQ56GMKIUKQwYRgTDhWhIBVO51KtVyvVyVJGBocCgUijus5jtd1piIIPN/GxAcAYEIYxHsewAQSCgVeIgQ4tue5WG1rWscol8udTsfznX8N6aa8yLuui3iOExTdtHheEEXJczzi01xPFmPfdV0ACWQAIQQhhmU5x3cJQm1DM0ybUuhbHkMYQRKhwAAeUAG4xIYcpAxyfEfT1EA0aHNwu1IoVItDxw+jjUKBctxzLzzLcxAIjGEYjueyAs8xEFBMCLhx8xZgWUAo8VwACMOgD4ecgHVcB0Died6jpQXbNAn2iY8BACzLAkIgAL3xQDwWqTeqjVrN95yArLAQUQgAA0RBsC1Hd30XAEkRHNtAACLAeI7PMAwr8K7vYc9FCEFEMcYdrWMaNvbccFASGCBxtLj24MprX7v65jebO4th6CZkyHomdExHbxta23VMy7IIwZZrcRxDie+7JstAWeETyWgsGQrHApFQAHuuIkqBcAhyTKujmY4NEEt8Wq81MWZaqr1daN5fLeypfrh/+qOf+7FPfeFnzn/0+8J9w00HY0ESAgExEIwlE4l0KhyNKKEgQiiZTjMsCIQCnCwaru1ToBtWIpM2XNt2HM00fN8/ceLE/QdzBw8ejEajGxsb46Nj+/v7e7v73ZU8y7Li8Xi73U5l0qOj45Bh6/Xm6OhoOBzOZrOzs7PteiMRiY6Oj83NP8j19oZj0eW15ROnTlSrlc3NzaNHj+7u7qqaNjk9dX/+ASPwh48eWttYHZ84MDAwUCqVZEFMxRMdTU2mEqZpFitlluMURapUKsTzx8bGarUKy6GuVaMnmz119szCwoIiKulkpllrfs8nP1mv1995552jR49ubm76Hunt7V1ZW47EY5RSD+NoPEEhKlXKhBBCCCvwtVqtUqlEQiG13s5ls4IssTwXiyXK5XIkEgmFQsurK5BB8/NzOzs746MjhUJ+fu5eKp24d++eZRmTkweu37iGWMSLws7OzokTJza21ucePnjuhecb7Zau66dOnVpaWkQIfOZT33fn7u2Z2alUJvX+Bx9EIpHJyQOWaaTTKUJIs9kcGBxe29hYWVmZGJuwDLPTaQ8PD8/Nzed6sqlU6sKFC4+ff6yj6vl8fnJyemV52ehoJ08ef/PNN13XP3LkyOtvvJHL9TQajf29vcnJycJevlmrD/UP3LlzJ5lMUghKpdLIyIiqqpevXnnyySddz3///Q/6e/sszdje3n7xxRfffvvtdE9mYHDo4sWLmUxmaGjon//5nzPZdDqdXl5bPXbsWLvd9rE7Ojrq2DbGOJfJ7ezsSIIYi8VKlfLw8HC3y37xxecZDmUzPTNTU5RSQsiBibGtrQ2K6ODIYKFU3C8VE+lEvV6VJPHkseOrq6uAUEDg7m4+GAwuLi5evXHd9/1Go1EtVwb7BwRJ7FjG8OhIs9myLYfn+e9+97t6R1Mk+dKVy4hlswO5L//Zn8bjiaGh4YeLC5FIRFaUzb0dB5Aj5866irxcKNiE2J4r8LzeVgOi6Dg2gQAT0o01Fjm+Viy+9hd/bBY2IPQ4hrIQEB8jCFmGQYABALGIg5DBmPK8iFgGIpYCBABgGBSPxmVZNk3T8zwfuxSSroXfwz6lEALEs3JQjoiijAk1bMewHds2G816s910HIdlWQYxgCJKGAZxBAMKKEKIUmpaeqlcWF1dicVD6XQyGFQc17Fsq5vDTAjxfb9WrxHSXYn3GMR0J64AQJ6TKIUsI4ZCEZblIWRkOZBOpyPhCMMwH9ptgOd5Fs9zlEJRVJKJtOsQRQnG48lUIsNxHADQ8x3d1DzsQkQJwAT4PvVbnYbtWogBLIsIxgzD8DyPGAhZGI4EBFn0fa9LxAoGFcezAQuksOIjoFoa0iH8/E/+5BNnTv/cT3xRlBhOZgmCGGPsugwllAF37i8Ciro2YUB8iJDvehzkAYAQQgpAW+v88i//cr1ahMRGAHIMi32HYwADAA9ANhqrFfIiCzkIiNe9agFAfEvXFSXYcbEDQE8qHBIoSygHOIlXWCgATExDKxZ2TUNttxuGZTAsFwwERJGrb6/de+fb7//dn2xe+07Qyo8kxbiMWOoQ22QhgpDRdMv1CGI4D7uYeIrE+r4ui/5gbzSbUmJBTpEQYDwpqDBI5BmZQcC1zEZdBZTxPE/vtF3brlcaG9vFraLqSz2nPvq5j//kfzr9qX+n9I3VLKdj2z6gYiAYS6WD0VgslQyEQ4BBrCgDyLEItmvF0v6271mMwANOcD3Cy4F8pVRoluWwIkhisVwWBEkSZMeyB/v7k7FENJoIKbFKqREPJxRJtm1bkqT1tY29/XKzo+l6B2MPYLCysja/sNhotXTNJhjVa005EKx0WoRnOJ7f29lOJmK5TGJ16dHhQ7OyIq5urDIcqxn6ysrSoUOz5Wqp0qw+8fRTDx89bGvq6IHRcrEgiHwmk3rw4AHEJB2JuI5dr1Vd215cXJQDSiqTsU3nwb0HYwcmKpUKNu1zx06++Z3XkvH4Y2fOXr90xXE8AsH27u6zzz7tec7K6pLvY8Oyg6Fw//DwwspKIp1xXXd5cXl8ZJx6JBqMJiOJeq0ZSyZ4UajVGoemD/Wmc8lkUg4EcrmcqrZ4jgkpcl9fbmCgLx4Nj4wM7e7vBmMh03UMy+IA26y3enr7xqcm9vb2UvGEwIuXL1998vGnKCb3H8x98pOfrBRL21sbn/3sZze3tlRdHRjqf7BwL9ebGR8ZnZ97GApGRFGq1+uHDx0MR0LF4v7IyND6+vruzs7w4NCjhwuhoDI9MWnqRm9vL0LorbffOHbsiCAIq0urzz/3zOrKsiDxIwdGG41aMhlPJmL1WmV4oH9lZcn33YmpA+vrq7Ztnjv72MLyihKN9g8NLy0sxSPRarmytrH19LPP+QCU6vX+kRHTNE3T/PgnXw4EArVmI5PJbG1t6bq+vbPDS3ytVouGIv29/bqqKZKEPbdQyJeqlXKxtLG23uqooiLOP5rXLL3eqlPg8wLVjZZtdrDr7W7sxMKRRq2MGKwZ7Wqr4gMiyQrw6PrSquv4fYPD8VSPbrm27QIMIMXVcsnwnGhPD+IF1/XDiswSAjyYimcAANFEeGt37fmXnu4Y2u5+PjcwSBCDLWesb3B1dVUnfnBkSFf4YruVymYBZCRJYKDHc9DwHBtA0/V8n/i2HwCkdP/qX/7qz1792z8GahlQzCEEMYU+gAQwiEEAsAyklAIIKWQ9CgHiCWUopgBQBiJFliADLcdysd2tgwzTze9gAWUhEQJyPBrJSFIUIZFQlmEFWQpIokII9bEPQFc5hl20iW1bqtrU9GYiGT554jCE1PVsx7UYFrIc6hJNuiU1EAhZluO4zofebIABJIAiQgCCHIN4npPisVQm3RMKhhmGw4R2+2DbMQ1TYxCgwO+GhPKcGAnHIqEkx8qEQgAYtks/hsB1zI7WANTzXFNT6wFRCCsyhBRjTwrKFFFACUMANh290REZLiArIi8wEPm+DymAFPAswzEQAYqkaOTKjWu22RnqyyIW2q7tOJ7CCRGEgpbTg8Gj998HthlWZM93AKWAYo5jKaCu7zMMAyFKJ3tGR8e3dzYhxxFCCQFdvDtxHQGAz33ixTAxt+dvcr7OIdo9fCgmEsNDH2DMVQ0cD8iMYygsURigUAxNXcbWSDYeEDkWkGhIiAQ4hSM7C7ff/qe/Xbh1wW4XMgoaiMkh4Hlmy3MMBPxuWoplWdFwRJZl7LthRZZ5mIwpiagcDgks74sCiYfEWFAg1OvygAxN67RVUzcAZgmVai3n5oON6w83PCnRN3nqhc/86Ce/8O/7D57WUFAlghRPp3KDqWwukUwn0hlJUggBjuValqUZZrlU1UzD9uxAQM5m0r25HsQyHqGu41MfCxIfCMqlUkmRA729/ffn5memDy8urKysrEwfnL11+65uWkePHdM0rVKpDGSzt2/dGuwfkGV5fX09mUnyAnvp0oVEIiFIYqXWmD44u18qrqysTE9Plyrlar2eTMRq1WokoMQjYcdxWmprfHy8v7+fUjo2Mnz+/HnLNqWAEo5ELl+5cubc2WBIKeX3picnY5FouVDu4gzr9XpPT08qleobGBgbG9vd2Wu1WnqnQzG5ffNWMp7IpXtuX79x/vx5Qsjt27d7e3vD0WhH144eO/z+++8rktDb27uwsDAxMWF7/tWr18+fP79fKHieNzk5ubiwQAmMxWJvv/VuJBKpVOu1Wm1kZOSDDz5QVfXIkSOPHj0yDD3Xm93YWNd1zTD15UeLzXptdWWZUizLIkDQMAzLNBVZ1k3t0fKyoijtdjsQCBw8eLClNkVRlCXx6tWr0UQ8Eo69/vrrR48cCQaD9+funj592vW99fX1559/PplIWJY1ODhYrVYlQUymEpraSaUytuVWq9VsT8axTE3t9PT0tBvNSr126NDBUqmAMQ4EAuVC8cCBA8lUYm19lZd40zSXFx8FJHllaXmgr1+QRIZhMrmsLMt37t6SJKnaqEdiiZMnTjQbDSUYGB0d3dvbJx5Jp9OaplquAyFdWlqqVCqKomxsbHAcNz4+nkonNjc3OY7b38uvra0dPHgwEol4npdKpSChFJPp6em5uXumbR05fmJxcXF0dGRoaGBtZXVqYkIWpVKxODs9o6l6q9XCGCOGdrT25OSk52FZCAwNDMuyomlaLJFieUHgxOmpKb2jHj5yMBSNrG5tAAAEjtfbrZGBwYAka5qWTqcLhUIwGFRV1TA0AKimqaVCXu8Yt67fOnny1Pr2zvWF+09+8mWTRfvtJhZ4D0KGZTvtpsgLlFJeltVOR2S5IMNEEekPSUvXLv7Zr/zinVe+CswmIDaEGFCfeB7GGPtdcpTfzRUiADCIAwD5PunSvxECiiywLNDNTkdreb6LAKWEQAoAhIBABPmAEo7H0+lkLh5Ni4Lsdx0pDNtN2vOx67jmXn7Xts1kIp5OJgEgpm1SSjmOQwhRij3P43gGIoqJh4nHMDASDrEscl3bMDUAsOs6DMN0qTgY427pAwD52O+6423brVQqhmHU6/Vms9npdDRNMy2TdKEq/zrI7Z4Wvu8JAsdyiGWR5zm6rmUSad91ie8LAg8QMEydF7hWq4V9n2NZyzQb9Tr2fc/zBEGQJInjOJZlHcuGlAocjyzHTaXTbbVZa9YQLzC8JPOC3+pEDF8qtT8SEkcbNbC3CYFPCLaw5wPfhwQDzLLdP4lgAl/+xPcdmJzFGEBeIIihEH4YkkvBEwfSf/Dzn/++M5MR0sF2h2GgbZgyJwmI56EIqaw1rDSDhlIJarehVWc71QNB7sxQ+pnDk0O5XpHntMLm4nv/fOMf/7h45+04VHsiKBFCHIc9G3s+dH1KGdZ0bFVt+b7te5bv2Yj4IYGLBfhUUIzI7EA2gaDjA1sQITY7jGsHBa6j1tR2FUFi6xp1cLXcun5rsdzmsrPPnPzMj53/4Z8++tEfkvsOlQ2KpXAqM5BJ9gZDSUKR1jFt02nVG41qzWrrVsfUVZ0FMBYPAYR3K6VX33nnT//8f//e7/2e53mUYoljHU0PCtJQtjfICtViycd0cvrgnbtzk9MzrCjMLc7PHp2NpuPbezsnzhwPyPzO5tq/+YHvfzQ/LzDos9//6U672W43z505XSkV49HY6Ojo3bn7I6OjZ06fXlpYHO4fODAyurW2fnT2kNZW796+MzkzvZ3fezA/b2o6tl1bNx4tLNq2rbVVz3UzuWytVnN0syeWvHHtushyuXTPwoP53t5eISDnS0VeFuvt5s7e7mNPPiGK4sGDB2enZyRBTCXjW1sbANF6q6kZxoHpqXK9Pjg4eOTQ4WatPjszE5AVx7Iff/zxW9dvQEJnZmYePnw4NTkp8dzu7nYqlRJFcXt7+4knH9M0TVM72Wz20aNHyXQCY+/61WuDvX2GYVQqlcOHD/M8H41GT506ZVrG9PTkxMS4pqnZbCYSCgZlKRWPsQhJsjAw0GfbpmkZrms/eHDftA211W7WG65HEC86lhsNhT3H7u3t9VxcqzZVwzRtq1WvMZTW6829fFE3DVlSPM+zLGtoaMh13d3d3a4yxvO8pmmKoiSTSYxxo9FIJpOmblXLVY7jHMeBEEIEZmanh4eHe3JZ03bi8eTt23ej0XgwGGRZdnx0OB6KPXqwaJpmu93u6LooKwsPFgVGiMjy7uZmKBSIJuKFQmFkeLhWLqvNZjwe3d7e3t/LP/3Ek7ZhUkpjifiV69ds35uamV5bWxseHIKUri6vZDKZYCB84YNL4+NT6+vrpVLl4Oyx+/fmIeBOnTxz7doNhNjpqcOm6YiiNDY2duP6ncfOPxUMBg3D6O/vd113aWlpbGwsEAjcuHFjfGxsc219Y2PjySefbDabqqr29w8uLy+brn7+ycdW1zc4Vjx+5GStXAsHFctQZQ4xgCYSqede+OidG/dGRkbGJse//d03ph87vd5q5E3DYXmXUAQg4zkRUTAMnQIcDyosJY1a3bOduKLEEX7w5tf/6bd+Yf39VwDWAKKIQZBhEMtgjDmG68ZnA0Bd4AEGUQQRwzEM5xHPcAwASEAWZUG0NKPdqEPiAYQp9ADCABCMKcEQAIZQ6vk+w7AIQtdzdEP1PFvTmxDhdCYeCgc0Q9UMjVIqCJLvAd8DlEIIGZZFhPiEeJT6CFEAPdc3MLEFkeF46GOXEL+jtT3fgZAyzIdB3oR2LwcQISSKYjAY1jqGabi25Ru61XUEdYGUEABMfAgJAJhQlwKPUL87UEQIeZ5fqpQxJqZpFYsl1/UAhHv5fLG4v7a2YhhaKpWQZbl7ujiOgxDL87wsy7KssCzX6WhIkqROq006RsyjoXJzWDVHWupxxJyKJX/w6NGT6eyxVCp/8yZwLZaBkIUEgm76EiGEEJ9hWELI+MSBS1evQYYjANKuYYZS4ns8BMjRgF4+Ntb7wsmjw7kkxF40HIQQchxn26bEc61SmQMgFwzyjjmSDj91eOL8xMBQhFWM6sbctfde/fqN997UijspCfTFpJ6I4mod4rmOaWmGZbvEcd222vR9FzGQgSQgsrIAQxIbVNhYSBZ5KIssoW4wqMTjUcexMCFqp2PoTqfjOYBb2S2u5ut31/Imq5x98fte/tyXzr34/f2Hz+tCVGMDbDSVGRrhZcW1XF3ttBo1wzBs1zFt28W+7Xq8qCiBMC+Iy8tL3/zHf/y9//7/feubX8WeHgvyzz31WCodp8B3iMOILANRuVgBDBuNJ8PRyJVrl48fP3779m3f9wf7eldXV01Dm5qeuHbtGib+sRMnvvPatyemJ6LR6FtvvTE5OTk0NFhv1YdHhvZ39x7OPYhHY3t7e+VKZWpqqlIqLy8+On78+Ntvvz0xMfH0s89cvnbl0KFDEwfGAKWnT53yHde27ZGREd/1LMvyfT8QCHAM6/tkcnLa9fxisXjy5OlypRKPxzPZnjv37omKPHbgwHsfvBeJRB4uLm7tbA4OD1i2jVhmaGhofX09EounUmkI4c7OTrPZ3FhbrxRLjUaDZdlyoXj8+HHXdUv7hcOHD29ubm5vb48Oj3Q6nUazlsmm79y7m4zHM6nE3NzczMHZfGFfM/SgrLRardnp6bNnz968fSvX1yvywsLCwrmz503Tvn//fiaZqZTLEKJsT8/tW7eSyVQymbxw4UIgEIIQ3r5399jJUxhjwzCefPLJrc0dy7JOnz59785dhFAoFDIMa3BwOJlMLy2thIOhocHhcrk8PTvjuf7q6moul7Msq1qtnjx50jLtvr4+URTv3r3b29sbCoUuX748MzMTj8cLhUIXwtVstKamplZWVsLhsOd5V69ezWazhJBqtTozM3Pv3j3TtJ9/9rnrV29gF4+ODC0uLfSP9kuhwBtvvfXMc89ZljM/P3/29Jl2vb63vXPk2KkHC4tnz549dHh2bm4OAPDkk0+++/Y7x44di0Qijx49Gh0drdfrK2urzz333N7eXk8uNzE5Xq1WKSGPnTtXLBYTiaSqqjdu3HjxxY9wDHP75s2nn3xK5IUH9+dGBoeIj/d2dg8fPPT+++9vbGxEo9Ht3R2E0NTU1JUrVxzHeeKJJ27evJlKpUaGx1779mupVIZhmOXl5RMnTkiyfOPGjWeff16QxEsXL46PjVUqlWg4MjwwyDNsrVbTNO3U2TMsyxJCPv693/PujWvT585w8VjLcRlJCYYjEEKORQFRoJ7ru3aXNqOrHWCbQNeSAgvV1sVXv/l3v/1fH114HRAdURdiGxEPAg8QwkDEQZbibrMLfM+DFIiMKHIiwRgTwrIoEg1LkqDqqm6olBICKICwu3FKCAEUcSwHAXQ9t9VqaLrq+Y4si5ZlUIodxxIEIaAEGIbrgmUopRAw3cPbdV1KP7Sxdws3xzIY+whBhABigO87pmmYltYt012ELYQQY4wgQykIKIH+/qEDByZzub5cti8YCHWVGQig53sAEABIo1nTNJXjGIy9gBxiGK5Wa0DIuA4xDRcA1Nc3oCgKADAej6dSdnGq0wABAABJREFUqcHBQUmSbNvuutS6tiLXdbtLIQihYDAYCoUQYnkG05gHhzr+J3zmJzjxP4YjPxGPPabIg7J8+thp1/a3H8wBQqBvu67LAIgAYiHiEEIsdKnLspAizIoCAdDzPM/3EYQIQgQpoa4iS81mfWN1kWItF5EZT3NdByOCEY5EFQarvKct3rqreM5LZ04eGexN8tSr7WxcePX9f/zznXsfRKiWi0gRWbE87FBGM1xAOR4png19DAzL9FyLBT7PwqAsRAJSRBEllgRlpIgMRb4QCnRsx3BcAFCjXLcsp2npLccrVLRq07/xMF/25Mj0+Rd+7Bee++K/n3j6JTY9KCb7lHAqk+mTIxHd0eu1gtauGGrTsw3TUD3fFAQ2Eg8nkhlOCV64efufXn/1t37vf3zt61+tFvZ+/Ic+9/nPfvzxowfOHB5/5Ztf4VkMGF+nDgpImf7+pqpDUYikE9Vm+fxjZ0ytPZjNMph06s2EEurUm1ur6zzDsqKwvLUeiEXy1dKj1ZVEInH37t18YZ8A0FLbQSUwlOuDHs5ls7F08uKNa+MTByRJunz58vnHH9stFzf395575hmz3WlWaolobHNzs6m2jx8/funCxbGR0YmRsa2tHa1j2La7vLpSadTLtWpPrrfVaimKIooiA1FfX3+z2bx05fLQyPDKxkogGuwdGVxcX8EsFALyvfkHTz75dK1S3d7YfObpp9dXV0v7hUOHDm1vbzumNdQ/YGi6pnY4hqWUVioVnueHhoY0TRsZGc5kMvV6HWPPtk3f96PRaLlcHBkZSaVSnucNDwzevnVn7v6D0bHxSrm6vbUTDoQvXbpiW14uN7C1tScwoiwEF+YeHTt0wnf8Wql+6tS5/F4RscKXvvTTDxcepTK9M7MHv/mNfx4dHOAguHr16sHDRxBiL1+6Go1GS6WCpnYef/zxhqovPFo6ePDg9uaWLMvhcPjevXu5XE4QhCtXruRyOcdxMMYDAwNLS0vNZnNmZubWrVscxw0NDV29enlmZkaSpOvXbh2cPbK2urlfrEzOTF+/fp0XGEFkr12/fGhmeqh/4F++8conPvaJ/M723Ny9p196yuNIvrT7yU99cn5tKdqTGp+c2N7eHuodyKX77j54mBsaXd7cqDSaRw4djgRDFy5c4GSxWCsDDp08fqJda8iyLMvy0tKS5dg7e9u1er2/Jwc8b/nRYjQabrVaAwMDR44e/M53/gVCeurUiWvXrnAcc/jwzOraUlfPuXrt8vTURDKdaLUaiiK3NXVx8eGxY0fa7falS5eOHj1ar9cfPngQjUY7nY7jeJFI5J133jF1RxTlu/duixI3ONh35crlXKZ/oH/0vQ8uDw4OZxLRrY0Vw+ps7G6WSrX79x+efPwxgxKHFcq62bEt08dIkC2PmI12Mhz2KWl1VMqiWDzsOwYimPoCg6UAhZzWuPQvX/mn3/vP6vpdACzEEtc0eAZxAGHPFhmOQxyHWJ6D2Hd912coxyEZAYZSSoDPyXwkEhYk0bQt13Ux8TBwKPAQAggh27Fr9Vqj0ZBlOZVK8TxPKS2Xy+122/f9er1erpRd98Na+WEQ0L+q7Z7ntZrq7k7eMh1Dt1pttfsjz3NFgY9FI4oiMQz0fdfzHR97hGBKCcMwXW8lJoRBHKWAYVhKESEA+xQCBgDYrciargmCEAiEKIUiL2NCZTGUiGcIZiKReCQSTydzlECOE3zfl2W5t7dXluUu87z7aRmGYVlWkERRltRWu1quWIYpcDyaFfgDFDyTyX7P5MT5/sGpSHQAkE8++di//fl//4m//sv+H/txKCn7e3lAsCjIkFJAIPF9+K+YdYw9THE0GuV4Zr+4K3KCwLKEYEp9xCCKMSF4Znr67t27nWajN6j0BsUQtQVb5+y23y4OxoTHj44/cWp2OBcN2I39uavX3/rGpde+XlpbiHIkF+FYT3c7Hey5FCDb8WzTIS5pN1XfcW1LJ9iC1EumotGILPKIYwDDEpFnBYHzsOt4XrVeMxwHIalcUwkUTZcpd/yVklZzBVfOnPnIZz71xV849/EfjAxOc4kcjCT5SMz2cbvdKVUqlUrFtBxV1wlFrMAr4UD/0EBPNtluV997+43f+63f/F+//we7xcI3XvnOj3zh/zly/MSf/flfbGxsMRSozbpjGgdGRvb2tgDLQB6ptrGX36/WG62OtrjyKBhUvvvum4bWYRETFJX+3oFYNDo5MW0YFkVQVTuapkUTyZ5MTpbl4ZExAsDIyOjAwCDPCdFIJBwOKgHJ1DqyLA8MDFy5cmVwcDCdTpdr1XZHVRRlc3MTY9zb26vrOsfzg8NDi4uLH/nIR0ql0urq6ssf+Wir1ZKCytPPP9fuqMlMWjX0ZkeTJGV7a9ey7FgsFolEnn3heZZl09keD/u7+3vT01PVaqXerD311FNXr14d6B2IReLf+ua3nnzi6Uwmc+3atc997nPpdHpvb29gYKDValFKx8fH93f3CCE7OzsQwo2NDUoJz7OTk5PJZLLRaGSzGUEQKIKIY9vttuM48XhcluVWq1Wt1wKBQDgcFUXZcRwIEAAAY7q9uQkArNfrEDKO42xtbiuK4vv+t7/z2tETx5vt1u5u/oUXXmjUa7raGRkZWV1fLxXLo6PjV65cyfVkOcTcvnk729uXymQ3NzcHB/pYBum6/uSTT5bLZdd1e3p6arWabduRSGR5eXlkfKwvm7t3714X8Xj18pUf+P4f3NraCgcjIyMja2trmUymr69PVdWXX36ZQSzHcQfGJxYWFlqt1smTJ//lm//c1zcQj8dv3L61tbWVzfTMzc05jrO5s7mxvUUA1Tuaa5vBYNC0LYqgJEm6rjfrDQDA6dOnHc9ttJocxzWbTcQyoiLXarVcLhdLxJvNZjwRCwYUSnF/X44BALteUJFnZyabzXohvz86OhoJBRr1yuGDs8l4rFwoHjw4s7q6rOkdCoGiKKFQqKenh+M4URS7S22JROLIkSODg4PJeOLg9EyuJztxYCocDmPPDymBWqW6vbd19vw5VVU3NjZOnT23VyywLDsxPlaulA5Oz6STKYZh9vZ2IskokITe8QN128GCwAQChAJZDmCfqrqmRELE8wOCFFJ46huG2gKeLRLMO7bi236l8H/+8Ldf/ePfVVcf8gICjgV8m+d4z/EBRdgHhACO4ziehwRg14eAgYj1MfYxNjwbIKRICmKg49q+71LqG6ZerZUNwwiFgslkUpZlBnIswwOAhoZGWJa3LIfjuHg8LkmS77sAEoio5zuCIHCMIAkKpQghjmF4y3IpRRAihmExJizL+dh3PKvLcEfMh0u2XVYKgohQgjFlEOf7PsvwEDCu6xJCeJ61HdO0DM9zHMfheZ7jGF3vEEK6QX0AAIEXsz294VBUlmVMiOv6lECW5V3XQ4gVRZnjhO7lxPd9TdMajcbi4mKr1QqFQsFgkFLqeR76fDzx02MjH42GB3h05lPf8+zf/+35//EbYGIIdNrf/PSn//eP/DANh+6tb4BKHXABzwYQEwYBQIiHPYypwCoEY5kV2rUGcTwG+NgxICDdPWCAIAOZnmj62edf0jRNAThkav3QmxDIqZ7wvzl/8rGJvgRnVrbv3f/gm1f+6c8r99/n1X0FmizyLctoN2ueYyLqeZat67phGMT1iOsgCsIhORnlh/rifb0Zz3Gx77AM4XgaigSkcFCzbAw416MQcBwSypV2W8P5mrOw2Sxr4sTxj7zwuZ95+Wd+bfKFT/GJASqEKBKIhxvlQrOW1zo16HnYciBFoWC8r3cy3TNOJGVhc+sfv/FPv/SL//Grf/2/rer+CyeP/Icvfv6Ln/9iOJr+4hd+4it/+1Udg1ffv+QSYX+/8eYb301ncsVKPRyJEgIUSZZDyuiBUYnnYqFwIhYeGug/cuiQLIr3H8wRCu4vLFTb7adeeL7eaPekek8ePlnfq4gMHw1FL168PHZgotZovf/u+5lUT0fT9vbz4Uik0W7N3b579OChqYnJQn7/6KHDIsfHQxHi+5jSnf29Sq1aKJcq9ZqqaYFAYHV11fW9RCp56cLFmZkZyLFXb9/sG+h3fNzSO6fPn9svFRmGSaZTH1y8sLW97TgOxb7IMhD7xPHURnOwt28w17e4sJDNZm3PnZt/MHvk8Orq6vLa6ksvvXTlypWm2u7t7Z2fn89kMhzHPXr06OjRo+VyuYsCLpVKkUjE9/3FxcVMJpXJpLpZeptr6+VCsa+vb29vT5IknucZiIYHhxzHabUax48frderlOCpiQPVSjESCh4+eBD7brNeC4eCqXQ8FFb29/d4nl1aWuREbmRsuFarjQ6PDA8OYYxjsURPT65erT/99JPlSqnVrKdSqY2NDd00Zmam1tfXLMuKRqNzc/PhcDgajUIIz507Rymt1KrPPPMMy7Kqqp47d25h/qEkiE888dSbb76diGcqlYqmqRT4vMTVamUI4dWr11stVWtbnbYxNTHbrDf2djZHR4fbWodhxcnhiagQ6kvkwkIgooSCssIJfDwZK1fLpqnnMvH81nqnXnMss9VqjYyNHp49+HDugWmaruvOzc/NHDlouOZ2fufQoUMrS8uSIE5OTt67d0+UpUOHDz54MBcOh9tqc3nlkSQL/f39qqoihDzPcxxnd2+73W4PDAwEAvLk1IFAIJDpSfvYdV2X5/mdnZ1oNByLRba2tmq1GsMw7XabEFKr1TY3twGh1Me+6/q2O9g/ODA8Thmmf3AglUntl/azvblIIj6/8PCFp5/N7+wW8ru92R5IvLXV5exgnwExSiVLjuMBSBCDEW9hYLoOxhh4NKQEUolQpieQCFOrtccQl3FxlA/ilp6TxerKvW/82f+49rW/8Bt7AFGACS+IlCLE8BBxmEAKAGAQ6JpqEMMwEmR4gZMxIA5xWAZJAg8ItG0XYy8SCcZi4e4SJaUQAAQAEnhFEoLBQDid7EnG0zwrIoB4nmcQRZAAglnEAIAAYBUxnEn1DfSO9aQHE7GeaCjFszIgjO9iSLvsM4yxR4jf5RBYltFuN1vt5oc1EBDEAACI53mCwCGGFop7pdI+ywFeQBBhnmdYluF5zrIMyzYw8RzP8HybUM/17C7yRBAEQkhACXGc4LnE9zCCTEfV2i1V4EVeFCgECAHT1AkhiqJ0SV+oh+BAs5k2jU/89V8OvfRs49H8H/zRH/zuX/3NN95959inPv1jr761qBqvLuyt72wDhDiG7aLaIEIswyGGd31PZEWA8fkzZ4t7u4B4PAsRYlyMfQARYgGFgiInkj2KHGxVK4PJyLGhzDOHx470xZjW9urVd26+8c3rb/9La281hNwQ9KDVRq5lGR3L0Czd0NstU9OIb2PfFlgIfVviSCKlMILPsD4CbjQSJNhXJCEQlDmOa7XaxWLRx0A3HNtHPpSXt2u7NbvuCFLP6Mc+95Of/Zn/evTjnwsOTHqE1XWs+bDSUFsdLV8qW44HIJRlSRC5vv7+bCpnmf6d+4/+++//z9/8tV//nf/31778R/+LZ9mf+qmf/OiLTw2klEdzN/7hW68MTh3p7R8bGhwNR5PvfXCpozscH/jRL/z49MFDmVyv5TqAUK2tDvT2Ec8HAMiisLO13ZfLrm1sAISGx8bnFh8m0xnIoPffu9CTyZmmeePazcfPP1bY3a+WawfGxsvFCvXxY088Nb/wMJVKHTt27P79+5IkpWLxO9dvdvPj37vwQTgYqlWr1XIlICmm7ewViulMhmVZvdMRRTGdTssBpVKvhcPhra2tnfxeNBrd3NiKR6KxWOzVV18dHhmhCF67du382XORcHju7r1cNru3s+u7XiaVqhQKMs9XSmXD0CChjWptbGys0+mMHhgfHx+/e/duNypvv1Ts7e2tVqsMw0xNTd17MDc8PNzTm7t64+r58+c3NjZs2z5z5szb333X8b3Tp08Xi8XhwSEI4fr6ejabVVW10WgcPnx4dXW1q9u88847IyMjkUjo/tzdsbExXe8sLM4nk0kASLG4L8vy3MP5SCQsB+VqtaqqarVarVbK3cq1u7vbZawnk4ku1Y4AOjg4wLLItoxuvB/LsplMpvsydDo6xrRUKceicVEUG41Gq9Xq0lcOHTpEKXVdt/shZVmOJeKzMwcVRSGEJBIJ27ZFUYQQBoPBdrstiiKETBdxY5haTyots/zu5nY2layXy6lE8ujhI41Gq8uUX1l49NjZMxIvlQqF48ePX758eW1ldfLARLPZzOVyJ06cuHXrlhIMnjhx4t69e0cPH8G+f//evWeeeUZV1evXbvT29um67rruM08+deXyNb2jjYyMFItFwzBmZw+1mmquJ6vr+s3rt6LRKM/zGxsbIyMjLIvy+7uZTEbX9d3d3ccffxxjvLOzdfToUVVVVVU9cODAzu6WbnT6//X5lILV9bW21qk1G5h4O/mdu/fvDQ0NdSmekUjE0LSjhw95nr2xu8mHgwYD2gRvVcsmJXww6BOghMKe5/k+wa5nWkY8FoxGxIHeOPR1z1QZz86Ew9TUgggIjr529+pf/f5v3/jWV4FaA8SBwKfEJxR6FBBKAQIIoW7MqU98AJALCASQQQwmmGDMMawsyYqiMCx0XZcQX+AFlmEJpRAgCBAAUOBlz8cUQNfzXd8HADSbza6WTUg3IBtDyADKIsRRjLAHAGR9jwi8JIpKV+ym9MNdy67xURC5aDTKssiyLIx9QjGEAACAGKDpnU6nHQoHRkeGASCUUlEUd3f3qtWa5/mu6zebbVVVuyAdBBHDIAAox7EYY4RYCBmCgapqnY7ZaqmxWKKvb4BlWUVRMpnMwMBAf3///xWUBEFATDpVtx1smVs/+RNf/uHPfv0f/+GZf/djv/j6e5/5i78Z/tn/CIYm6NQRFYBbD+eA77CI2J5NIKEAAIAAYHiGp8RnKBns7fng/XeXH80DBDChBDAUCB5gXQAoywCEZDmQTKYnRsfiIa6wdPv+6/947ZW/by7eMXdWMgwRXNvSLU13HcNzDBNiFxE/KMmyIGqdNsejcECQORoPoGgAcKKR7g3YroUxxZ6VSoQYhmm2tEZLazU1U3OwB3TTm1vYnFstSj2TR1/87Pf93G88/sM/G5o9A/iIZ3q2ZTYajU7HLO1Xq+WKYRiIkyLpnmgigxm0md/+2tf/8Vd/9b/9mx/60S/93H8Ih4NeeX+QY585fso1XcO1C/V9o7P7cPH6xcVHpz726ZXNUnm/HpdDeqNVzO+HIpGLly85rksp9R2fAYwkyM16UxJE4tP9fGF0eOSVV17p7e/b3t/b2tt9/PHHm81mpVCcGB1zbNt2naGxkW+/8Vo0Hj8wOraxstqXycai0e3dnYnpqfmFh5cuXZqenhZYbn117fz58ysrKwDBF196aXl1RVc7Z06e0jva+Mjo0PBwpVLLZXqS8USjXrddB7KM5blnn3hMVdVoIBQNhgK82KzWXcsdGRmhlEIITxw77lp2f673pRdeXJybn52csixrZWl5eHCwWa+3ms3J8QOmbliG2aw3KqVyrVbDlLQ7qkdwNyxtYeFRIpESJLFcLk9OTj6Yn194tDgxNfXg4Xw8mejv7//nf/7GR1/+mE/w9Vs3P/OpT+/n8wjAs4+dn3s4LwjCgbHxt996q6+3l+WF3fz+iVMnd/N7d+7dHRoeLpfL/f39x44dW15elkRxZGSku7GFEGq32y+99BLxXd9zDx8+uLqxUazVJg8caDergaCUyiTv338gK4FEpufSlct9PalsMrm2tpbpyQqitLi45DiO67pyQBkcHFxeXsaAxmKx9fV1juMAg7a2trpBaPn8bigUwNiDEPo+mJ9/5LoYIW55eTmZjDMM5HgQi4Xa7Wa73RkZGXNd//Ch6dmZA9euXlECMsuBlZWl0bFhTVcvXbpy4uiJrdWtZCSRy/WtLm8owcjU9JF33vluF2PQ6XQ+/vGPW5bV6qinz56hlN6+fXtoaEjXdULIwMDA9va2JMqpVHbp0VowGMI+ff21d55+6gUA0Orq+sDAUDgUn7v/8NlnXtra3NZ1/cSJM5cvXa/X6wMDA/fv349EItls9v7cPdd1I5HIe+99V5blYDC4tLTUrRFzc/dOnToFKdjZ2RoeH90r7Hum/exjT+7m9wRFyeVyjmkNDvSZphkOh6vVqiAIY2Nj33nt1Wx/bzweFwNK3/iIy6Oa7xoAtM2Oh10GMDyn+B5hGFbkBce0JEmSFSEUYiXBNfVivboTlCUOIAEyou9Jduveu9/669/7Lxe+/peA6Ah6BBKPfpi+CbshG5RwiKGAMoDtAmhZxEHAINJ1GGIIKWKA53mmZWKCEezm+SFCAACIZTmMAccJHCs5NhYFSdf1drtdrVY7nU6XOgAAAPRDlBaglOU4AADGfretJoRQSgCghPjdb0w8SRZkSUQIWJZhmjqhuOt0DAaDkqhgQllGJBgJnDI4MAoo73tIlsKpZC6VyAm85DgeBQBC6HkeAIRjGZZhIYSiKFu2ywtSLjsgSSFKIaVQ0zQIYTwe830PIgAgFSXB810kjPXVFdbJZe9Vql9647Uv/fPXD/7bf+uEIqvtTlnTCATf94UfNwG4euceYCH2HYQAAZQAAgH1sY8gQggBBlKAfvbnf35wcLhLyuQYgQHAcRyIMCZWPCqHZdZrFxduv/fBt/7P+r2LenmTtdscNoIsFYgjUEIwNgxD11RAfUHgWQ45ls1CNDbcJ4tQEaEsMQwLHFfjOM93tZ5MSuB4tdHQNc0ybNfBtabhUGG3qt1d2mn78sz5F37oS7/87Bd+bvDMs0BOUCh2Ok7DsiqtRrVe833fcr1oMj0+OTUwPNHTO9xS9b/88z/75le/9sorr3xw6bLtuv39/TzPXrl4KSQpnCjlW2Z6eOar//y6Ioerxa3BgaQn8i1KRyYnPQ935ZfLly7092Yh8E6cONQ/0tfS2yIvYM9HPIcEDkI4MjK2Xyz9ux/9woOFh4qizEwcuHf7zlB/3+GDh9rtZiaTcl23VC1Fo1HEAEL93t6sZRsra8uCwOX399PptCRJa0vLnWbr9OnTS0tLsUQcssz7Fy8MDA+lejJrK6sHZ2cdy3YdR5bldqtl6YYsyxtr6zzP92Vz7777bjQa7c1m97Z2fNfrtNthOaA2mmq7nUqlGuUqJLRarrz1xhs9PT1ra2uCIDzzzDPXr91MxlPpdHp+fqG/tzcSCtUq1ZMnT1ar1Var8eKLL1qWValUBEGYnp1pNpuUUlGRC8XiY489lsvlHjx8+NwLzxuG0Ww2X3jhhavXr/uEHDp06Bvf+Mbo0HBAUfL5/DPPPRuJx5aWlkZGRvb39xVFmZqd2dzcnJ2dPXz46Pr65sDAgOfhO7fvnTx5kmCwvb175tTp0n6BUnDq+Km/+7u/kyRpaKD/vffeO3L8GAXgypUrJ4+f8H336rXLZ86cKRbLEMLR0eGtra1Ccf/YsWO3795BCJ0+fbpar6Uy6Z6enms3rk8fnF1ZWVleXj5+8kSr3a7VakNDQ4ZhBAKBdDqtap1IJBIIhTc3N6dnZx3bCwQCTz75pKqqsiKdOnXq5q0bQ0NDg4P9jxaXJycnN7c37s3dn56Z2cvvAwCmp6cfPLgPAJiZmfmHr/1joidtut7G1uaRY0dLhcLq8vInXv54Op12fY8Xhffff7/dbhsdLb+7J3BcJpPpdDrhaEQJBErlcigUKpfLGOPDR4+Ypmma1umz5x88eMhw/NDQ0I0bN9rt9tTUzGuvvRaJRTlOuD83d+bMuVg03m62uolmtVrt0KFDvCjs7+/39/f7vm+aZq1W2dzearRbh48eWVlZmZ6ezPX13rp7J5XKxGOxV175zuzsrCwKt67fGBzor9frqq45vhdPJEzTfOe73x0cGa7X67woVOtV07bEoLJfK2rEoQJSouFOp+O7HiGA5UVRkjXLdgkIRqOhSCiVjvYP9PAcaVTLPIKpUIAjjkTsOI9hp7J84/3/9R9/euHiG5zXCbKEwQ7xHAIQoBAARAHlAEKACAxPfJ8QwrAsJgRgyjA8xhQhpMgKw0BVVdWOSqgPAAGQYIwBRQiy2AfYB7IclCRFEgOhUCSZTEeiUY7nu25IADEmLkCUUqy2W4ZhdN04XctjF5FCSPc8AYT4lFLHswj1ZVnkOI4Q3/ddjLHnO7V6pdGsAQA4lsOEQsBke3KJRDIcCnMc57iOjzHP8wDQRqPRajUsyyKUAEBYhmMQ298/EI3EKAW2ZXsuFgRJUYKO49i2zXEcz/Ou65qmSSlFrTi4atV/+cbcdzAGAAFeqtULFuNmc+lMKokYILM8z6O5xXWrXguEZUPTfEwBwQBQhgEEYMezCYQewwUiKVEJA8oAAonrIscL8gwHHOSo1eW7j777zYfvfVVdvSJZBc6qenoLIkoQNX270VYJoAD7LKLxREgQmI7WZBgoi1JA5Flqs0RHyA5GJD6o2JQ6lg097Kq62e74tpff2ivtFauV9vZudTXfCQ0e/8iP/tJzX/rVQ5/8POwdB0jQO2a92ag3WrV6RdObvIDiyXQq25sbHPR4dOPRyp/+3T9+4Us/9yOf+/zNt98eikbCwUgynbY9I5ONmJ2WJEliJtuRE0528uWf+I+Xb216NisxTDwih1NBzTemDk0VayVG4KVQ4O7dWzvrj3JJmfpa32hvTW+yLCvLctuzGpaZ7e1rN9X+gcHLV69NHZhKhKNqudoTDnMI5fO7DM8Wy/vhiJJJxUdHh9qddrleyQxkG1ork+tZW1021DbGeHR45Pzps4lYfH1zo2MZLM8TQCenpxrNJi8IHvaXFhZDsjLY118ul2VZHh4cajQaH33xJWp71y5dHh4YbNTrGytrZ06c1HU9Eon0ZFLF3bxrWPntHUPtjI+MBmRlamJS5MTxsQlRUhaWl86ePbu9s1Or17tElHQyNTM1denChfHx0UQq+f7FC/2DfS21XalUjh49ynBss9lW2xrHcfVWkxOE6enpa9euGYYRDodrtYYoyhAy1VL12LETmqYhhDDGpVLJ930KQSwc6831EwgKxWIwHF1b26jU6n0DQ5VqQ9X0weHhxUfLSjAUi8Zv3bw7PXUwmUy///6F5557QRTkufkHL37kpQcP5iLR0LnHz73z3XcFQTo4e/j+/fvBoNJqNYrF4sDwUCqTnntw/4XnnyuWCisrS8ePH6/VaqVSaWBg4O7du4ePHhk9MD4/Pz8xMTEyMnL3/r1DRw5LsrC1tZXLDuzuFfP5fCqV2NhcESU2EgzcuHrt5PETxAPvvP3egfGpVqtlmuaxk4eXV5cs18sNDN249yCW6sllB7Y2Ng/OzJYLxb184dCxo0RgK1otNZBZ2lyemjiQDIf3dnZs06IIQpbxMO7t7Q0EgrZhGW0dUVRvNTuWYTm24zh9fX0sy3qel0gkAAAjI8OVamlgoM80TVEUe3p6EEK63hkZGfJ9n2GY/v7+vb2dRr1Wr9chhK5tC4KQTCYZhjEsc+zAuGYavCQODw/7vn/48EFd1xEL55bmfYj7sn2aqq+tb/f3DzYrteLe7rnTJwv5PYZhQqGQ7+N6vd47OHDy3Jn8fimX66tWy4beMZoaDxCLaKtTLaoNnXpSQGIBkGWlUK4ZmMZz/T4n1jo2lAI+YggEg/29uWREa5Ra5UJMESWGQt9mPT3KuHGqXf36X3/jt/5T4do7gNiIgYjhTQ8iJFIfAt/lKGaAw7MUIOoQn3IcRohQyCABAg4TzHFcNBpWFNF1LcPUIQQMA7sXVpblWZaHgBOEoBKIiFKQ40UAgI8dTD2AfEwdn9iEOgS6DA8YHhCAfeJBCAnprtOi/5ss1M2rwNhDCPjYdT2bZRAFeHtnkxBflgWeZ5utaq1RsSzD8z3PdwGgmPiEYIFnCfF2drb2CzvhiCJIvOs79Xq90Wxggn3sEwx8nwAEEctABmFCurtLAEDX9bSOjn3CMhzLcEhIRfi+eD4AbhvaYj4PGEYMRyOxuGboX/3KX3/82Rc/9omPcoHgdqGwu7sFXDsQUCjxKcAUuD52KcAcx3b5Z7bnQcRQ34PE5RgfQrO5vXDv0ptX332ltrUMjRprNILIVqgjEC8qK55leq7r+4ThhJaqYmwpCmOZKiZ2X38ml02ODvf3pOMBhedYGApIrmP4vmsbJnBwq9LQVYflg7slLV+z9puuw4bOvvSpH/gPv/rM538yNHaIsopqek3N3tkvaZZluy4hJNfXl+7JRhJJ23Vv3L7z67/zmz/7C7/4S7/0S7/5a//vhdfemB0abVVq9XKlUtjvz/Z878defuLMOYnjWy3VcvH/85M/vb658Yu/+MuriyvtWiscCDuWFZSZjfWF808/KQSDkhLY291XVV1UArFIlGVRT1+vYTs+wY7jeNjD1G81mq7tNBqtWq2hSLJr2xzLzsxObW1tAUTDkaCu65VKORgMXrtyyXedTDr57ttvBQKKJAnRaOTosSORSOThw8VKrd7p6K7rTk9MIsQ0Gg1IkWkYnY4uinJ3gA4QK4oixvjh4sLAwMDeXr5Rqz1x7rGHD+YTicSB0bH79+9PzUx1dP3u7TsfefElgeOAh48fPz43N4cQQgjVajWWZSGEkiQbhjk6OpbJ5tpaZ3p62nVdTdNefvnljY2NYrF46tSJpaUlAMDQ0NCrr746Ojra/dWTk5P7+/v7+0WWZVuqOjA4BAAyDGN8ZNTU9C6/gud5QRAikUi9Xu960nf29gzD6KqH4XA4nU7zPKfruumYPM/rujY0NJTP7waDwUg0XCoUdbXz1JOPN2t1jPHI2Pjc3NzYyFC1XC4Uy/2DwwzL7ezsda33DESHZg8uLa92dOPUqVNzc3N9fX1d2jDDMKZt1Vv1c+fOFYuFcrl8/PjxtbW1vb3dc+fOXbhwoVsfF5ceRWLRo8ePGYYxPT1NCNna3sxms/l8vguY7FIA94uFCxcuPPnkk8FgsLBfSiaThmFsbm8NjYzu7OwcOXR4enKq0Wj4GFMG+QR7jrO/v99sNqvVKi8Kpmnu7u4GZLlWqzmO0z80WC6XBZ4fHh7e29sDAMzOzl6/doPneVEWFpcWAuFQtVHXTZ2XWAjp3fv3Tp065fjObn5ncGRQ13VRFAeH+mu1Si6XO3LkiGPZmUwmFovduXMnGAweOnRofn6+tzebSCSWVlcCIUVSZMQyDM85nmeapiRJiUQimU64vteX6w3Iyt5efnJykhDcaNSTyUSxXNrdz3suHhoYnrv/4PDhgxwDB3py40MjiiJRFmJELNdWgjIBtKWqvCiVKtWm2qk1mj4AqqoxDCPxgmObAoemD4zFworaqEsMlwlHcrEo69mM3UkIhNT23vrqX379d369vngfuLqMCPBcFkEIEQCAYgwBAIB03Rwu9QilsCtVUNg1oiAGsBziOMZ1bduxfOz+644oIARQChnEQoAIIZhgAAAmnmnpS8sPTaujdhqVSqm7smTbZrdhB///X90WnmVZSmn35BAEgQASkAOTE5O2beu6Yds2QmwgEJAkCSHEsRzGmEEMIcSyLZ7jBwb7uoG6juOYpi1JUjQa7TosGQayLCKEsCzDsizL8oAi1/V13bQsJxqNJpPJcDjC8zwyW61cT1b1QLljLz6cBx5+8zvvfPozPzQ9O/vDX/zCB/cuBXtjXDJQ07xbNy8BlkBie64FqE+IzzMsBJASgn2f+FhgWEAxRNiu59fvXrjzztf3Vq4zdqk3wis8IJ4rcSxLIfYoi3jqEZ6ybtsUCUd94DkOx3ki72Z7Y0Mj2aDC8ozPAEcQGNOxRVnyXc9QVa1RobblNAyz6ZQq7Qu3lwqmOHbmkx/50V96+sd/ZfCJj4FQwrO8VqtRq1XUdgP7XkCRY7GYJEmaqW1sbV+8fPf3/uef/3+//bvfevW1b7zy6rWrN7buz50bHvy73/rNL37qe3XDeu/q1c986ntOTE9tzC185U//amR40jK9ZDDy1f/1P1N2w92YC5HW4tyNTN+Q7YORRBgZTRdQJZaq1+u/+zu/+6V//yvvXrgzduhUs2Nmc/0eIR6EHsV96SRyHVFghoYGQnLgyXNPtBtNSZICiei95cXeoYF0Or22tpbLZtPJ1N7O7oljxy1NV2uNp86eB67fqFSPHDlSqdVK1crxM6d2SwVOkaLhSKNa80x7MNfnW04yEtNabez5vCwRFi08Wkxm0qlsjxhUuv7iyclJx/GOHDmiadqDhYdjB8ZL9arj2WfOnLl182Y8GgsGlSvXr45OjgMWbe1sHzlyZO7evWqxJCB2d3OLY1hZlDbWt7b2dguVskPx5etXkukEy6KbN28ePnhIFqVr164NDw+vrax2p46bm9snjh47dfzErVu3jh8/HggEdnZ2MpnsvXtzwWB4cHBwaWU5nkyLkrK8vHz06NGdnR1VVadmpiGDdMPIZrM7uxuiIkRjQZYDQ8P9mqlanuESO5lO1JoV1zNtxxBEdmNjjRLfNLRHyytyIKhIcjqZjMfjhVKZENCTye7t76YzKUCI5+F4Kq0EQqurqxyDLMvqLvoDAHRdD4VCLMsmEonuK4oxFkWx1WodPXbEsizbdaKJKMMzN2/fDoajS0sr29u7HMvXq7Vmo7G3u51KxjVN1XW9r6+vv39wcXGJON7oQO9of28oKMUyCSSy8XhyZ2u3uLPjm7ZvO9gmIhXGhw6wnAQF4ekXn9/dz0cikZnJqXq5NjQw1NObm19cSGd7ksnk5uZmOpFUG+qjxcXB4aFyo6Zb+pNPPj4/P+84zukzx3d2NlgWnT17+tGjBYjozOzkO++8NT0zNTIydPXq5UNHDgqCcPv27aeeekpV1ZWVlVOnTnU6nVKp1NfXV6vVGs3muXPnfN8vl8tdkFwqlU2nera2NkIRhWGBYWj5/H610khn+1bXN2YmDvT3ZpZWFyamJ7FP93cLxAUzEzNba+upWHR7dbVUKIqBIGBZDkLquZplI0V0iK8ElWy21zCMUEhhGUJcxzNto6MJHO84VqfTjkQiQTHgm67TtnzDjSmKJLOY2BxDFYiFTumf/vC/vf6Hv04KS5BqAHiujzFFkJEpZRjACojxgYUQ7ZLJ/3XySV3XxRiziGVY2PUgOo7dajdc1waAEOJ3hXgKaFdv6RLOMcbZbNayLNd1w+GwIEgIsRwnAIC6T+4+/P/KMhhjx/EEQQIAqaoGAQMBQyiAgIlFMtmewUyqPxJOiEKga4F3PQ8hRjcM38eiKAOAGCQElWgoGIlG4+l0OhgIdn2WhGDPd23Hchz7w1sCRBwnRCPJZKInk86KooIpcDzP8XzEunYuEcMEKLHAH/zlX0wP9P/oj/7Em99+lRWkX/iVX+ob7NvY3vQokcPgyoVLwCcCJ1JKu2ga33dd12QQ5VnII5ehztb9qw8+ePXh7ffVylYqwsYDKMxB5Bq20eIQ9F1sGw4LBb2tm7pmm5rAMxCRUFg5cnRmoC+VSUeCCu+aGov8YJAXJUY3VAxouVZr1ho8ZAnmGqo3t5xfK+p+sOejP/hj3/8f/uvRT/yANDBFsKAbfqXabui25RFektOZrBIIWrZz687dm7fv/M1X/s8f/8mfTE1N/cM/fPXClasfXLoYD4cysegf/e7v/Z+//iuF4+bvLzzzse9xGOHb3/rO3/3FXw6nUodGxoKyUq/XTV19cPuqZFSjRD041nvl8kVWiQfEYBIYtJ2HAjd19nyhUh0ZGXnpY9+zsZWvV+vlSoVl+QMTU+V6LRAOAd+rFPPtdmvuwb3R8ZF33nlrcKDPsozrN2/Ozh4yDKNYLL7w/Ev1ej0UCj/xxBP37z/I5fqOHD52/+6cLEipZOaNN95IpzPRaPT9998fmziAOLbZVscPTCqKUqnUDoyNFYvlcDicyqR38/uhaCybze7t7lcbTVaU5hcXIvFYuV57uLRo2lY6k8n0ZovlMsuy4+PjHV3jBNa2TVEUh0aG79y50263k+n0/Pw8z4n9g8ONRmtq5uDW1s7y6tqByQnAIAJwJpeRFJnneYrJyOCQbdu9vb2zU9Ob6xsDAwMsYgxNy6TS167duHv37ssf+8Qbr77earVmZ2f39/ePHTkaVAK3bt164YUXHi4ubG9vT09PLz9aGhgYOPf4Y0sry9FUIhgMbm9vnz9/vt6obm1tnTx9Yv7hQ8Si4eHh27dvdzSV4zhBEGLxiKqq1VI5GYsCAILBYCwWK5SKpVKJEOB5niRJlFLPw+2W6ntkaWmZZXiWZX0PS5LiOA7i2Gg8ZtpWJpMZHh5+uLhQrVZDodCtu7dSqZQkSeVysdPpNBoNhmEajQbP8319fTzPEwyOHDw0NjJKKU2nU+l0iuPYoaEhz/N83ycA2K6rm0alUumo7Ugk3OqoO/sFRuA1Qw+Egt088WQ8HglGlxaW5XAwN9D/yqvfGRkd5UXh5s2bQ0NDy4uP7t67d/78+Uql8uDBg6mpqW7qliwFCoXCydOnFEV59dVXjx47luvrvXDx8sDwiKjImzvb8WTCsu319fWJiYnV9fXX33pz7MB4sVjUdT2Xy926dcs0zYGBgRs3bgiC0Ol0FhcXM5mMYRjLKyvhWHR7d2dh6dGZM2e0tprP58+cP7e5uVmv12dnZweHhwLBsBwIUAgsxy6VSo8/fr5Wr7iWc/jgoXg0Xq/Uh4eHms3msVOnNcMkEHmOzwKmWq5RCnXDRAjalsFQKvGCJPIiz0mSwLGsIAgY42g8BhA0DJ1haSoWDMispdYdU+MRTCZissTLHMBGYzwbau4s/M3v/fo7f/tl2izwHGAAII4LIQsIwJiwgGEARPBDQZxSwDCcwAssw2OKEUSe7yCEggGF5znT0hzXZFnQVVYopQiibkYFy/DBQDgcimczvelkThQCgCKCAcGIEggh052pdiX4/9uq27bdarUsy/I9omlap6N7HvYxBQBSCigAlALfx47ruK6Dsa9pHdu2PhzSEkIIBQDwnMizou8TH5PumaHpLQq8tlpvtWu2bZIufxhjACCgDKUIE0oIQZCVZRkho5EMCAIEDc1YbbVhKs4B8APf96l8pf7pH/qRcrn+ue///latGeDkuRsPWqt7SAi1WobWcSDkOYaXeB64RmNz/uGV1z/45z+vbd0OwnYmwsRDjGdrhqZjx9VUDRBiO5ZpOsRDpmYhhDCxoik5nQtm+sKJjNLRKkGZZSmGnhsNKiz0XVsrlvaqjWpHV13HN5rGnZsLC2tVnUsfeO6z3/Pzv/nMT/1a9NA5wsi2bWua1tA6EHFyIBJPZCKxtO3Sh0tr77x/8fV337s1N1eqtz73Iz/6la/83ac/+dGDkyOF8t7eztaZ6Yl7ly6+/Knve/fWXYMPHn7qhWj/gfV8q96y+rN9r//TV1fv3zzQP8Bz3OvvvHb42MSBgcTscA8EeDNfcLEgMFLcrg3L7v3lpQNnH4ehwNe//U3EUMvSd3fXRIHlIDg4M/tobUOUJLPTOTwzEwhLoVj4g0vvnTx/fGVzSe00J4ZHd9d2AoJsaOadO3dFSdnLF65evR5PpJRAkJflY6dOa6YFAPzMZ76/0+kovPjY2XOQUJ7nhybG55cWO7oWi8XW1tbi8Xg8nVrd3IjFYqamq43m0cOHDcMolkrpXLaldyyAhyYPMAK/XykFIuFgNEwxKRYKqVwaCly8Jy0E5I2NtbNnTyMEarVaMpMGHFNXW5n+3vXtLZfgwdGRO3P3m81mT0/PxsZGIhnf3dsJhgKGqa+trXY947lcjud527aj0ejW1tbQ0FA0Gl1fX//e7/1e27ZX1lbHx8cL+XyjVhsdHrt//4Esy6IsrS6t5nJ9hmFdvXr96InjxVKpUCjEYrHLl6+GgpHZg4e/+c1/6e8fnJiavntv7tzZx3oy2a2tndmZQ4X9EqLoqaee0TRDFuR4NLKwsCAGQv1Do6VSaWRouFAorG9uHDl4hOMkAuGBiQlDbdfLlbGxMbWjUUozmUylUslms8lk8u7du/F4vN5qrm9sHDt2TFVVAmg2m93Y2JiYmJAkybBMAigvChuba8Mjg9evX11dXR0eGXr06BH1MULMm2++GYvFBEFQO62hkWFBCVQbzaGx0aWVZUESxycOLG9u9Y6O8gF5dWvt9NkzPM97nvfpH/xsuVFZ39k4fPjw6srKzt7esdMnVVXNZDKRcHhzfaMvkz0wOtZut5OZdK6v1zCM0dHRWzduax3riSeeuX//vuM401MHK+UmIWB4aHRraycYDJ8980R+rxgORw4cmCyXqzwvY4zL5XIikegGs3zqU5+ilPb09Bw+eqRar3meNzMz43ne0NDA0aNHL168mMumRIF79613U6ns4MDIO999n5elSCqytPwwHA7avhdPpm9cvd2bzo2ODL/z7luyJJYrRc93Oo65WtiL5nIIAwny7ZYOAYsQEjmOBRRbFnSchBLAtqtpWiQSYnnGJa7tuR4FkGVYHgZDHGB0hDqppKjw2NcNr2OwgAaCEi8xHa0mCSTI26Xl23/92//lxje+AvQWYhHAFECWYSSKEcEU/qsIziAGAOhj4vsfohwhYLqTVUURwmGJYbHldHzsYOLhD+GRkFIEAIMgxzA8oQwhDEIcywgcK3IcjxD7r3oO6d4PIIQMw7GIT8RSgCLTsD0PG4ZFCGWYbhAHAhR1jbkcywo8x7DUtDqYOAxLdb1TKhU6HZUQTCgFAHq+zzIcw7AEUB+7lmU4jhUKBSKRkG60S6VSW20DACFgupZNzyOAIobhKIFIIE46Eg4FGYaXuGh8o1B5+qkzf/UXf/YP//C1EydO/uzP/uxf/MVfEB9zjAAp8923vwuwO5hLBwMCMDW1tLN069LV17+x8fAm0cq5CN+fDguMb6oN6HnAx4hQhgAWQM9xiO9B6ruewfEgGBb7RnKZbFwJi6GwQImZTAZtx/B9l2U513Ycw2nVO7pmdzR3t6zPrRY2K9bwoSee/fS/+8RP/afp577HDaQ1k6g2MClyCKAMEwyHbdsul8tXr99457vvf+vV167duq2a9v2HC3/4J1/+0//9p7/9G/8NAhAKBEKRcN/QMEBo/u4d19A/8Ynv+eDKdU4JvvvBhS//6Z+Pj4yaHVMU5ZnDB3/lV/8rz/OpVKquNjWjwwB3b3OVZ5Ht+O9dvp2KZ2St2SdCi5Do6EjPyNCDhYe6qaVSqfW17WQsld/eGh7sIyxomJrvY8e0pidm6vX60889e+fOnXRPanp2tloqJ+OJTkcPBEJjB8Y9H8uyfP6xx5VQqNZqX7pxM1+pJuIpjhNu3LjVbrchg+7O3SeEGLp17/YdQkir0X74YD6VyewXC+9fvHD48NFyuey73sSBqXt37o4Oj4yNjS0traQy2WAkOr+0ODo+hjFeeLQUisT28vuYgFa7Y/teQ+20NH1yeqZQKvsUsDzX0TUCAMfzrY4WS8R7ctlqoz42NuZj2jHM8fHx9ZX14cFh7JNSpf74E08bjruysZlIJTc3t5PpFIUMZFjLdeqttov95bV1ApGHaaFQYBhGkWTDtB3PZUVJDoRCkVi1WlXkIMdxa6sbGNNwOEopTGcy1WptaXH52OGTlUp1aWnl6InjDxcX9kvFxx577PU33+rJ9SqK8vabb4XD0U6nk9/JHz9+vFSpLCwt9/b2bmxsII7t7++/decuxwrJVGp5ZQkiGg6HL168PDp+wKO01qgfO3GipWpLKyujo+OlUml0dHRieuLGzducLPqYzD1YmJyYKpXLpVLp5OmzaxtbxWLxwMTE2vr60OiYGFRu3rp9+MhRVdfrjca5c4+tbK53TGN4ZOTqjeuUYXsGB2/cvEUw5Rj+4sXLA4OD65sbu7u7PM+/9e47e/v7qqFfuXYVMiCWiLXbzWg0OtA7sLe7rwRCluOZhk0xaDbb+/vF3b29tqpqhs6LQqVSyeVyGOP3L1wIhIKFYqnRVqOxREczbt26FU8mqvXG5es3Gi2NYcVmo92b69/a2iIQJpKpRqtpu24wFLp+8xamsNloW7ol8NLI6FB+fzef3/U8r5u2Wq3X0un0uXPnGrV6o9acmTl44/atWqNx9vy5erNdrtRdDwwNH2i3jNXV1d7e3Obm5ujoaLvTsV1HDIqiyIkczzFsOBpzfK/RbCOGDYUC0XDAc81Wvey5tqFprVZLkEVW4HiJt10LU99yLcRCB7uIA4LIBGRZYhgGY1trO2Y7HBAVgSWOLlBfxkaQWOt3rvzLl39/6f03ALGAZ0HfZSHkGZ50g9wgpIACAFiGY1kOQgZChmVZAIBPXApwV4cRBR5Bij3PMkzHdQghCCIIICEAUIQgDyHTLdAU0C7eC2PM83wXJGfbNsMwDGQwxYSSZCLZk8n15nqz2VwkEgEAcBzjYxciwjAIfQibxAInhMNhjuMARV06QjQS5TmeEEwB4VnOxx6mmEWsIAiZdLYb/qVpGscJqVQqFAqxDONjtysrQQgRYhFELMsjV9ej4WA2l7N937Y8SlEgEgYi9/d/9VfActLJhCzJQ4OjghhoanalVKzfvwpKy9vX3rr55teXr73Z2X2QkJ14AAcZj6e4o5qGgSHlfJdii/iWp7daerNBHA9SHAjBZEbo6VVyvTElInGyIHCMxLFBkXdNQzMMXlLaqtGo65V9tZJvF3bbpbLrCX1nXv6xz/yn/3Xw+78UO3jWh5LpYMhLQlAhkHStrm21tbq8dOfGzcW5+UBAfuXVb519/HQmm/6TP/3yN195xXHd/b2t9Yd32sW9v/6br65sFJRIjFOUm7ceLD5Y+oWf+dloQPn5n/nStYvf/cUvfUGmjt6sOp773Pf/wLevXrOJPzw8HE+k622NIMbzcS7RM5gbuHHnbiwWC2MSZ7j9Ws2T5NnDR4rF4vr66sc//skb1x5SX8jEEmPjQ4FseKW0JygBvWO/986FXKZva22bEmi6pKUZfYMDckB2MKm226Fw2PZcVTcIRDvFYtuxp48d2S7kZUF2NdsxjZGRYTYY4AIBghhJknjIZePZvtxgLtdXbzZy/bnD07Nba+vxaKIn07dXKHC8sLe7W9ovHJqZLZertXJlYmTsnTfe4Rg+k8zen1tM9vQCnt8vVwRJbqmdvXyhUGuYjr+zm9/c3PYBcH1vY2MjHgkvLj2iEB2cPri2tjE9M4sY7uHDxWeffb5Wba0sbx07evq7F65s5YusLC+urKZz2WK9eWf+QTydrrfa+UqJ4YWt3UK11kqmUxs72wAAQZTv3L8fCMcqjdZ2fp8gZju/b3k+YrhKsZJOZurN9vzCI1U3GJaXWGl3YzfX0ysIwvLKSige5gR+cWlpenZ2b3+/WC5NTEzk83nHw30Dgzdv3wkpIVkWN3c2R8aGKYFrm1sjYwfyxfx+IT8wPMCwfKlSe+yJJ6/dvEEY1gNwZ7+g63owGF5cWJqYmNB1XdO00QPjG5u7+VJ9dHxmbmElFkvwkvjaG68//uQTkOGu37p95MTJYq3e7BjheOre/KIPWccjq5tbsVTacNyFpeVQOFxtNlTdQFBs1rVOQ00lko9WF1OpxEBPTgBcrnewpqqQZ9a3NxCApfz+7u6uruvNesvUrUar3dGN/UKF5WTE8O2Onslk88XSbr5g+9h1/b3tLcTCROpDCOXq2trOzo5tu4jl2p0Ow3GWYw+Oju3tFhqNVqVSEWVpt1CkPL+1t+tDClju0eqapunJZGppaSWgKOsbG5VKZaC3T221V1aWdNuqq/rco0fF/TwLcH9v3/UrVyVJ0gzj26++2dc/Uq+o7Zah675u+XIoZNrWzOGZ2/fvGbbb399PLNXV67apshzyqC+EA6bvma5nOWYsGREVVpYFHgFF4BEAnu34vhsMyaLAsiwSJNnBkFeiLhAtD2ECgwExGpJyqZivafV8XsA0zMo8BiFelIAXYTy7tH73rX/8p9/6pf17lwAwIbEp9QilPqYQQEBot1/GhEDIUAhotzmHkFKIAIcgTwliGJbnWI6BrWa93W4S6gFACfW7c1oIKYLQsnVdVz1sAeQxDOxylSGEjuOZpt3V1rursN2RbHd8wzJMW214rkaI2WyVOloToe7qE8sgUZEj0UgiFIxCyBBKMPGwb1PgNdsVXWsB6lFKEGQhYOLRnoH+sUy6PxJO8JzYBdBD5BNgEWBRiBkW0S65BlsmtoxkLAiwy3EcL4rvXboAIP3j//n7H7zzzo//+Beffu7pvf28HAoRntsvFi69/Z1L3/5GZWsZ+aqM7KgMReQD24C25eqGZ9nAw8B17VbTMlSj0/BtLRYWo1EpFOKDAS4U4hGDBREi4HuOAQjlIGpVKs1qDRK0s1du6v5e1diuOTsN2jN5+uV/+6VP/vSvTJ573mYCLhAxrxiYMJxAPMvttDjf0mr5lfk7nU4n2ZN7+iMf+/hnPn3k6MlYIvHlL3/5zdff2N/d7c/0BANyOBgqVSu1VjvRmw1n0pVKhYXIw2BhaemjH3/5p37qJ1999dt//ud/eujwVLavp1Cr1Aw9NT7Wxm6mLycI3OTs4VpTE1lFYaWdla2h7NDDpeVAJEwdS4I4EpC2t7cGRyY8wvzdV//h1LkzHcvjpMjBIycERe4fHwkmI0Nj4w/mF44cO7FfLCWT6Ugk3myp+cK+4ViW68gBRZSVtc0t0zSx6924ccMwTVGSVtbXDN0qFIr1er2jqe2Ourm741NSqVRYhFzXq1Rr5XJ5L19wsK92Omtra6Zp1urNR2srHdOotZuqqlq6sbm53Wg0OI67d/duOpkkHtna2pmamskXinPzC6FQ6Pbt241GQ5Sly5evupgkMj3xdDoWj/u+zzLMxvZW//DQ5u7Oex980NfX//Dhw/39/XAk9vqbb2i6me3tfbS0IgoyYtmVtY1gMHj1+jXbc4PRyIP5+e4b1DH0QCi4m8+vra1lMz0PHjwoVsrhSGRlfY0XBcSyq+trsWhid3t7e3MnmUkvPFpkeS4QCam6tr2X50WJZfn8fnFvb39sfETXddM0o9HohUsXI7EYwws37t6OJOKIZRYWHk1MTG1vb29tbSWTyXfe+26z2Tx+7OTFixdFWYpGw+vr67Vmw7Str3396z6mW1tba5tbLMvu7O4tLCzkcrn333+/Wq329PRcv37dwzQUDC8sLtuWMze/sL6xMT09feXKlZ2dncefeOq1194AiMEErGxsSkrA9fHiynL/yND2zs7m5ubI6Gij0djb2QGEmqYlK4He/r56vZqMRSnAe3t7LbXTarUikahPsCRJzWbTc/10uqfZaJfLZYTQ3u6+j2k8mcgX9mvNBsOxjUZLVzseJq7rJhKpVkdr6R0f0tWN9XqrGY8nIWR28/sMxy2vrrCSQAAqlkuQ5SRZabVaFMKOYd57MN87OFAol0r16uDQSKnaWFnbGBwcvHfv3s72tiAI8/MLDMNksj3rmxsd05KV4G5+b2Vl5fr1q6dOnWq0moTSnmzv3bv3A6GQEggtLK3u5vd1XWdZtt1ujx2YCMfijWYrm82YekfkeMswHc8ORsJyKGx5vuPjrZ3tRl1lWd5zPOz5LEI8z0qS5Ps+hFAQ+EgkRAhxXV8UZZblEYdYnvF8y7bNdCoVDUaw7bEUsoREFFFhgATcIGMLropbxVe+8uW//+3/0th4CLHBEocjHvW9rq+828hjgimlhAJKIQMFBFlCIcYUUAQIAQhKgUAm0xMKhWzX83yPZRgACSUe8WzHMRGAkiLzvOhiwLMiy/KmaSPExuNxURRtx/R9H6EPDwNC/9X8TgkAoFQqtdttSoFlOfV6s93u2I4DAECQJQRg8iG2zHXdTqfTrDdEUQQAWJZlGIZt25jQLgWeEIoxxgT7vk8B9jzH910ACKXYMA3DMlzPRdlEylY7vfFAUAI+tgAHVcOZHhsLsSAXC/7FH//BD//wD/7cr/zi6v4OCir71fLh2YPRYIDnoCjylFLLsvWW7hoe8aFn2TLLmu2aXt7z9QaiRjTOJ9OKHAKI1xnOCik89AnLcK7rGu2m3miYmr67vedbXoBTSnu1zfVivmhwsdGDL/7A537rz2Y+83k0OOEAxgSUCUgexB3TQAhBy2ptb+4t3r7z7rde/Zs/ClDzwIEDTCjy9t07//n3/+ClT33v5SvX33j9naWHC0cnxgbikaGedDAYjmVHf/yX/vOjva3Z4zPUtXuTyf7+zLV7N15/783//ge/u1PY/au/+8qf/Z+v3FxdKtj66zeu/uYf/H4oEbt+80oym0KyQgFn163BSC4diEUDkc3d/AdXr2T6spA4o+lIZ3/vxRc+IQaTxUr1f/zR78uJ+K//zz/88V/6L//9j//M8bAoCPOPFjND/Qby5leXbdff2d4DAEwdnH3z3Xcc7PkE3L17P5PJcIh5eH8uGghg4l28eFFieYjJ9vb2yNjo3v5+rV2PxmP379+XeUFTOxv5XcNzEMcyPKebxk4h31LbkGEq7Wa13WxrnXanzQuCi/1WRzUta2NzU5bljY2NRqNBfHzj2nWB43uz2dVHSwO5vma1jiiIR6IrKyuqodfU1traWqPRqNVqpmkWquXFtRVOkUq1qqHrkILltVXMIlXrLC0t1Wq1fD5vqtpw/8DK0jIAQLPN7d2dWDTaqjX0Zrtda1TrNdt1WADzm9udttqoVZaXHtm2tbWztbi0GAoEy8WirRrxSOzO3H3MwP1yqdqqtTqq6djvXrzUMIyG2l5eWVldXV1+tJTf3dtYXWcRm8/nW4ZmU7qxs7u5s+143nvf/aCLVVpdWY9G4pVK7ebN24Ig7ezsPXjwcGd7r1Zt6LqpKMFQIOh7HgvR/v6+53mRWHRrZ5tBXH5v/+LFi7FwJCiJuzvblPgAAE7gKaV7e3v5fN4HZGVttVaptmoNw9BisYjjWBT7HUMvlUoMBYDS/N5eR1WTwZBRb+m63ui0b83ds7HHILD8aPHh5qoQCZbzBbVcg5iWikXLshLpzP25BQ8TjuNWlpZkWa63mo12gxFhvrC9vPxosH8A+sBQ27FY7OrNWwwv87KytbdrEz+ZTt25dbvTVvsG+guVkhBU8vv7Dx7Odx8yt/iwf2hwb29vY31rcnJ6bmGxUK8jhptfWqw06kIweOPW7XSqZ6Bv8NHCIi+JhuddvXlLUAKUgpWVNUyhoAR001rd2iAs2irtb+/vpLLpUr2SL+V7ezOhgCKybKtR1zRtbW1tZ7fQ7BgPV9YaHSMUCMuc5Fl2rVKtNVouAEhQMOYA5hsllRKut7e/6zJkGOi6LstxlmXZlh4LBXzTaJaK2DWI7/iEBKMxQQm4HpaUkCBIAoNYQvRGA2EbO20BeRFFFAFOBTi3vvv3f/jfXvmj39TWHwDoQEQo7SJZWAQBBJSFDAsZBnIAMA6mPqAAQcQwBAIf+x4hBCDISiwvEQoc1wHA87EJoCuIDMdx2EfEF3gmiDEbCSXDoTiCvG25XTeOj13d6BiWRqiP4IdkAgSZcDg2PDQZkBMcG+A5RZaDgUCAYRClxMcOQpRBqDvLlUSF52THxrblmoZt6JZt267rep5DKQaAAEAx9izLME2TZXiEWM/DpmmbumVoum0avuuwnmfLitzfEwc24CWCEYimw3JAuvfg3vnHn/jiF7+wtLX2ym/8RpgHxGgWtvVKcTskcYgg1zQB9RhAEWQYhKjn2rpmGZrnmgFE0omESX2WxQxDXccJBkTsEuB72LF9y4cQMhj4HlU1vdZQddNw/LoUTJ14+tmBI+dBNAs43qXQdAkviTxiAXV937Md03HsVrXU2dvDtmG7Vr1W1nXta//0dfDejQv3F/P1miiKUVEKR2ORSCTACyL1HNNqN1VBDtVa6tXrN5599hlA/LgSqqmVQDDw9nff/forrxDH+8ZXv8qLYtloUwYlEoma2vrdP/qDUCBIbCd8Idw7PEIh11A1jODw1ExJ64QjiQ8u3Xz6137CvXyj8mih3UF/9P772WyWF90bN29bli3Kku/7FGOjo0miGE0mg+HI/cVFQRAggaFQyHWse3duZXIZx/OKxeLxE6choaVicejokWg0qj3UTpyZ3dkuQEJ7e3u//Z1/iaUjlUqluF9wXa/VaDYaDU6SSvVqxzIKlTJBJKQEVFVttVqaZeyXSxgB23XXVlcd01IBCgQCmXR6Y2MjIAV4nu+GLj148GBpaanLkj1+/LhlWQFRyGazc4uPtre346EIixjPdgSWY0Q+kUgsr6636o0AL3IcpyhKqVTAAHvE94gXjgQ7ant1ZSmmKAR7u9vb8Xjs0aPFZDzqWaZnmZZlYYxt0yKuOzI2ul8u8TwrcJxpmrquQ0zbzVaz2XBtm+OY3d3tcDhcq1RDSoBhGIDgyNjo2sYqy7LpZCYZizumMzIy0uqommHF05m9wj4CTDqRLhQKkUjELJe6KEpAiOd5PM9Xq9VEIjEwMNBst23bTiaTtVptYKBP5KX87t7o4EhbbS0uLgaDwXKtjBCKhsP5vb2xodFmrb7iL41MjHmec+fOnXg0wSKmWCx2qV6WZbVbrVXHoZ6vah3dNAKSrKmd2zdu9vTmSvuFna0tloFL66snElGB45cfLj5x7mwoFDIMo1sLWo3m9voGL3L7hdLa+nownFDbHVkORGKRra0t29Rdz643a5FoSFGk1dVVVuB5Ubx15w7HiaVSeW15TQ7KyWisUKokk8liuVSpVAbEgZW1Vce2erO5/b18s9nMpJN3797leV7k+Xv37smhoKIohUJBYFhV1zY2NvoG+hceLebz+ampqXKltri8NDA0qKqq53kd3VAYFlOwtbfrUmx7juPakiRsbW+kkslMJl2tlXM9aVkWNV3d2doYHR0ljiexYjAcCfGSvV8znKYiCgyAEDKu6xvAb1hOlJdYgfFd0y2V08m4YWq2abq+L0i8JHOaprqcmYqHTVHAxKvWavFUTtdMhBDHcQxEiLIQUJajaqsTCsYlwNqu7zhmRFRUy0jwbEIJlNfnvv5nW8n+sSdf/r7I4BjiFIC9rmTRDZKjhCCGYRkWAggB9YgPEIAMQyl0MYEAsgyDOAZjhxDcpfgS4hPAsSyHIKIUAMh0WQiiwLge8jwHE9f3XVEUCSGu65q+iSCvKAoAgBIIKOJYhePlbmsPAO0i3REDMHW6G7MEAQTZUCgcjkRtxxJFUZKkLoWXUOJ3V3AZhud5y7I6nY4gCJ7nmabZzcmQJEmWZY7lkJDgdL+VTIaHcwpjOQdyA7ZmDAwNPvOJl//L7/zW7JGjJ2ZO33ntjZPZ8OdfPPPrP/MjjN8BvkUt3W03qGtC37aNtqE21UY1GlEiYXmgvycSVQB1RI6ICCi8nAjG3bYtAs7QOu1Ww6w33WansV/Z3sjv5Ottl40eOPHiF37pY7/8OwMvfRYkhwlBFvYYngkrQddwqtXCztbm1SsXN9aXXvnGP3z1b/9M5F3L7rz27uW//Pqb37764NLcytWbt6MBcaI3M5CMCjzTbNYKhcL65uat+4t35pdV1dna2nYMLRlSZA/ExVCn1eEkudZqc6yQTaYzPSkqs4mh7MGjhw4dms3EYgM9mb7B3kxfpm+wFxOvsLujam0aFrm+5FKjTMOhZDi9ulL5wz/5zt+/euvR+5cat68WHt3KphQGQE23HNevlGuFQqFWrea3d1jI2oRcvXnn0NhBu6ZxiElGY8sL8x976QVN01bW1r73ez95+/bNRDTy8osf2draisYTvf2DW1tbJ44cwY41P//gscfOVRv1wn5pdHCIOJ6qqrOzs41aNagoLiB75WI2mw0HguFwUFREVmCPnT4eCgV915UFMRmNKQKfjEaatfr5M2dDoYDjWIlEol6vDwwNDg0NmY7ZP9gXDAcajUYsFuMYNsSLfcl0QJEFQaCeHxClWrEcDYWHcn1aux2Px6LRaH5rZ3Z8AnsepfTA5ITreaqqTh2YYCDqy2QfP3sum0oPDw/X6/Uu1zQZT+iaKgckVmTLzWogFJwYmwAEhARloK8/HA2lcsl4Iuy4hsShmfEJ5JOwHDo0MyvxQjIRK+zvRQLhTCxRr1RTyaQUUAzb0XQzFAlzHDd1YEogKCzI/dmcIAiO4wwPD3cTlCKRSDIWj4bCiqJgjDVNm5iYYBgmGo36vh+UleeeelIQuEwq+dRTTwiCEAqFpqam2u326NBwX6734PTU5OTEsSNHuyi0k8ePOpYRCigjQ4MMhyLRUF8mGw9F+gYHNNMYGhgMyYplaI8/du7QzHRfXy6bzYZi0WA8lMokqOkkpDALGEszJY5NRiNyMGC49tmz5zCmgiwdPHYIAKooiijJ+8XS2cfOxmKRaqE01DsYkIOtjspInGppN+7cPf/4E4W9vNXuPHHuMa3Zdm1HEIT9Ssn1/XaztfJw8bmzjwdZYWt5lWdZiRcMTac+bjUanCB0/ykbq2siYjnIRJSwogRLpYrl+YFwRDeMQDiUSmWmx6cExLeaaq6337Ycx/MOHj5kmiZ03WwiUalUxsbGZJ7zHOvIwYMPHy1oujo5eUAKyRjgE4eOtErVWDCq6Xrb6sTSUUlgAxwLHZfBWIlGry+udCJhNRSweKhahmnTgBIFAESigY7W9IErhoRgRFH1Fi8glhFT8Sz1kGN5yAcKxwFqxRKyqCAK3EQy0mk27I6JXJ/xHBl4IQ4wrsHZem9I+f8R9Z9RsqZ3eS7+PM+bY+XYVdXVOYfdO+fJI2lAYmQwtgBjY44Bm2PAPsfmYINIskk2IolgkZRQziNN3ntmh9mpe+/OOVToyvnN+f+hB/5r1ff+UL1Wve/9u+7r9tPe8eaDz/3ef3/4xU8reyvA0REibJvwIOVCwsMQAB50HQQ84EKIcAhwF0AMIgLDKAwg4EDPwTECAgJCAkDCcZHl2I5jecBB8H3W/ISOJwiCYzmO4zCMUFX1RBt38pOgaZqma+9DlhiEELqe7XqOC0zTVqv1sqx2JamjKJLt6J7nncRHAACaohmGgwAzLdO0TMe1cAKRFO4Bpyd1WJaNRCKyrLbbXZKkWZYP+P0+USRwAgAP122LoijKs0IspSqOpcjJaOjBe7fjgThDgcnh/v/4b35oaqQvE+I5ywCGZDu6rBg07yehB00LI3EAHV+Ahw7BcYQDgaHJDIN5rslRrKprsqaZuqn2JC/g100PI7ieomxubWG0L5QcOHfhqdjMKSAEAMbYHqEZHgEhzvAIc+pSV++UTd3Y2FqnWCocjr/1xqv/51N/9PSVc3/4J3/aavc0T+CCSZ6ALgSaoTfqVV1Rbc/t9mRFUWiSIQByAHQQATHCNO1oLKTJisjzJM24EEKCIGnO8TxomgRNIArTddVRPEWSSQ/2FNmAtuu6PE4RiDAt1TC0o1ZdQ0AynB5G07wv3pe8t3akWrpPECjTwBmscLjb7UoInaBOLk0RDMNMTM0yLK8retDnp0kq3deXisf2dzenxsZZimVIdnJiutFohoI+Q1GKhaNkKgVw4vAof+HCBQQ8tdcdf/qaqisB0X/h3IXFxceZeNLHC4eHh319fRRDuwAwLIUALBePHUNPp9Oqba8uL/tF35YicwzLsmy71+V5PhAIrK6uTkxMnHhgpk48+71eNp3Z2t2JhMPnzp07OjpMnItxDE1gGEFStqlns5nt7W1REKROW+q2x8bGJEna3dv+gQ+8SBCE4PP5AuLO3i5NkaGgf3t9bSCbqdUq7VZjamrC0nRDV+fnZ1VVXX/l1WeffbpVq3a6rb5k2rYPJLkbDUXGJ8YIDK9WKslkMt2fKpWKBEHpsuKalp/lMdsN8aKPF3q9XtAfSKUylUppJZc/WcyIx2KKJNuGubOf/9D159ZV7ejo6PT5c9V6rVwu9yXTsVgMOK7f75dlGTjuM8888/bNm7VKxXEcvyB6jtNuNZYWF+Ox5MDAwOLDRyRNaIpUyOX6EslWq3Gwt8txXP4oF+9L+n2+R48eRQLB8+fP7+3uPvvcc7lcrlqtTo6P3717t1GtTU5O7u7uBs+enZ6cuvnW2x/4gZdomi6UCqFIsNJqHB0ePnP9WqveIHFsZGRE6vYS8b5oPGZaVrfXO3kgLRRyo8ODhqmXy2WCxDudjuM46XR/Ot1/5737OMQ5lh4cHIxF4oV8cWZmOhqNtBRJEIThgcEjeCRJCkVRIwODSk/SNY2haIHjaYwYSKei0Wiv1ehJQUNTTFUXGDYSCGrtnqnrxcMcQ1KO4wxmBw6ODgki6pgWCbHycckG0O8P1uvNYDDYqtW2N7dEjhdFMcBxg5l0u9n48LMvbq9tHOzvnj97+vZ7t3Ecnzt1annxSafTGRsbIzASxwnTsKtKxweBBzBVVePJOM8Gbtx/59HR0dVrp08N9bkW3C+3RAaPh/lGoxWKBFvdDkmSsqrQLO16rq4bnof8oUSz2bRNSzdkzs/JhoKTOMWzhqwFg35gwXa77UEXMhTluTiGQeRano5DEuOJTkd68u6rq8uP+6fPPvvPfozk/ABgjusBBACACGGeCz2IAPAgcCFwT0gYABD0wPsdKBxzXYAQxHHccyzLMBzbIggSx8gT1BIADwLkuDbwkMj7PeBIsmToluu6COH/ZCYgcQoADEIPAOi6NoSAxGme5xVFQQi5LjBNRfI0kqR5lifI92mffypVecA5MTnjOI5h2MnxieOEeDxOEpTj2v/YknVd10UY4DTJFDEoIrNV121HzR9WByO+n//IpU/+4k/++k//i2fnBiMs6DZKtWpFVVXL0hz35NVDJaEdFKlwmBUFDEN6r1u2jR5BuBiJEI71er1qqWAZXcuSWJouVxuS6u6Vu0vH1bHnf+Dl//QbL/7nT8QuvwS4PsMgZNnBMY6jBYphOqp899HjV19/+5Of/OQ/fOkLz33wg5Lu/ulffPav/vYr4fjIUVmSIMtF+32RhOl4ra5aa3UO8rl6o1WvdYpHJQLDI+Egi0HPMVieCcZCQohPDaQAxFnRDyjSxRFJMZblMBxvuwAgZGq6VG8p1Wb3qNQ+Kss9iSGJM5NT18+eY3Ba4HiawAmaurW4UtHNnoM29g9yrfrW0b6k9zpyu1A9LjfrtUan01MJgmBZNhqNDw+PpjJpnz/E8CEcZzEHUhBubK+Znrmzs3Hm9Pzu1i4GiZA/uvx4RRA4Re3WSrnTc7OtnpSvVCZn51ZXVzVdWjg9s72+mkokfIJ/5fHKSHa4ViofHxz84Ic+KCmKqmtTY6PlXCEWCs/OTIkcH4/FApxgK7rAsAhC0e+jOLbebimGLmuqbpmWZZVKpXS6T5G6CLjAtkxVmxufnB2b6FTrly9c5ESBEflAKIRBmEwmDceGOMQwSJKkLxQ8ODokSXJkcMh2HEmXhbDPw9D89IzWlcayg2PDI9VGfWx6kqZpqd0Zzg4szMz5OSESiSwsLMQS8WKt1pbkSDhYyR9FIoH0QKJeL0fDwbMLp6VWr9Xq9aUzmqYJHDuQTmndbjIaJRE6zh1l0n2tTnNte50P+DACVyV5Ynh0b3N7LDs4NTJGYBjLsjiOx2IxgsQQQlPjEyLPOo5F0URfMs6xNMNSS48emrqajMcW5ufanWY6nZ6YmEAIRcPBavl4MJtZOHWKYZhYLDY7OxuNRodGhiH0EsnY7PSkyPELp04l4lFDV8+fP+t6ttrrnju9UG/V09n09NQEdOzzF84uLMwTBPHyR37Ise1arXb27GmaoiiInZ6a73Q6kiq5wPH5BEH0ZzJZzwXZgf5QNNxuN2OhcMwfljttgWOHBgdInKhUKteeemZrb/8oX0yn+/e39qK+kNmT97a2R0eGCZrYPz7wCCwaj7XrjUw0wSFiYWomGgoH/X5e5DiBtT2Lpmm121M73XAw4Dk27oKR/kyA44BjR2NhXVevP3U1lYhTGPIsMxoKGoqcjMZwABmGCQaDELjpRDwUCoXD4dNz86lodCybJRFkAHb13IXjQtEyzB/+6MtHB/vpZN/c3Kn3bt396Ms/vF/IS4Zeb7QcC5A4S5G8EAjhPAcZut7u0pTAkbDeBX//9cVPfuGtrR7mhBI6SUu6bWjAM/CoP0EiUhB4wzUBcsUgQ9HA1FrJWCAcC1Ac25HlZk+WdNMjcD4gWq7leU7A5w/4BE3qubpOI4DZKmZIHDSDFB7yCWGRTYl4c/P+X3/851e+/bdALmOmigHgWJ7rYQ5EACLoQeg4hAcJhCMXOK7rAAgxAuDkSZjjeZ7rWjjmcSxF4K6mdm3HtG0TQs/17BN1AYaRlu26LibyoVAwIgg+juMCfl/A72NoGgDXdQ3XswHwMEQgQHoAo0gxGk4ztB9DtCiEYtFUwB8mSNp1PAiR53mOY71fhYU4jpEETqmKrio6hPAEnYQQWraJIcyyLAAAhAjDMIQ84KNps9P46AtPPX8pe2Zi8P/7xR//lf/4cx959nLaRwKlYXYbwJRZAlK441oaBkAoIPoDbCIZioYF4OqOpZmWxguM6OOQayeTsVqlfFwutbu9YDipW8BB9H5NerJbfbRZqal4ZubCuY/+BJMc03TCg6xHChgf4HzBwuHh4t13f/Pjv/zffuWX37l1/8HiBuOL1ZrqJ//kL/7+8198sPT47Pkrw9NzQjSlemS5ozRbnXq1VilVc0cF27RMQ+NEIZUZUFWdZ9hoIp7ODviDYYIiAQIAQdXQe4osa6qsKp1eV5Kkdrvd7XYbrWaj3XIcZ3BweHxk/K//6tOvvPL9ixcvP763uL+xCyGmaUYsHBoYGPiRj32s1pO3c7lyp1tutY4qJVlXAoFAu9vTNAOjyWA4FInH4vG4TxClbq9RqzcajVy+1O0qAABdUyLxiCS1oWebmlqtVOSu7Jhuvd60HTMajRzs7DA0WTguehgmCCJCyDZMudelaKJSqbRaLcuwTV2PRaLXrl1bXFx0XXd2djbg91++cNEy9WIuPz8962N55IFzZ88e7OwGfX6WZTVNGx4YJAhicnLyypUrnudNTUyKoshxnN/v73a709PTpVJJ07TBwexB7qC/vz8aCheLxdHRUQzDEIH3D2RJkoyEw+1W69y5c4JPVA2d53kMw3qSNDg80Go1nnnqmmtbWxvrZ8+dkWVZ6rTTfamdrU0CR47j7O/sphJxv+jr9XqJRCIeiszPzY6OjZimSdP0yMiIoemJRGJscqxUKg4OZjOZDEmSI2Oj7W7Hdp3x8fGgzy8IwkmTiOfZ/v7+nZ2dubk5iqJwhM3PzviDfs4nhkKBaCgs8CyOI5ZlLcOMRaKO49AkxdLU2MgISWCiKJq6GgoEQwH/3NycbdsszQT9gdzhUbvRDAWCnVY7nU6bjt3pdC5cunhS/JmentYVVVXVTCaztb5h6kY2m11+8mRmZobjuHql+vS16+srq7quJxKJarUaCoWefeaph/fux8KhSCBYrZQwAnXkTjASIijihIwGnttut/tSiaGhoXSyLxwM6Lo6Pjrcbtb7032n5+ZNXe/r65ucnOR54fr168ODQ5FQeGRoYDCbabUbg4ODqXRSluWFs2ds18JxxHK0qirJVF80GsYwLOj3RSMh2zGnZ6cEn0jgKBwMKF2JwvBzp8+YppnNZmVZpglydHiEIUhouxfPX7A0Q1PUTCrlE4RUNIo8IPW68VhE4FjgORgGSQwXeWFjbR1iyO/39zqdSCicOzx68N69i2fOfPfb3xweGej0OhOTY7qiurbjOUBTdIQQIlAs3dfp9CCgIM4wolBsWZ/51ruf//47RdntIl5HwmGpXa9LlgllSdM1ExCY6WgEg3DCVbW2pnZJHAoCT5GkpmknobPPLwp+ARAeTpKcT8RJQjdUDAGRZSypi1m6nyGQrZrtmojMAFCf3PzOl/7gNzbe/jbo1QloYq7lGDoCALgeghhwIXA8iDAMERBCB3gecFwAHNd2gYMhDADgeTaO44Ig4hjWbNVVTUYnchsPQoDhGPk+/eIAmmJoinZcx7Id17Mhcj1gep6taYphGgBAACCOkY7jcawQCkYYmjs5yQLPcV0bgPcrq+Afx+8wRFAUEwyGI5GY3x/EcdL1XAIncBx3PZsgSABAu93K5XKIwV0KGoQpoV7r5z764Y89c3U2HZPrxUY9ZyHTxiDDschyMEsXKS/E44OpRDIW9gU4gFua0UPQFXieJBiEU6XjitLt1AsF2zJ4no/E+1d3Ct996/Hff/32myvVjRa936ZGFz5Asul7rz+oHrdpnIWOvbuz+t3vf/cXfvE/3L75ypf+9s+6+YPcyurq442ejG3vtv77xz/5aHnzMHcwNJqpdSpHx6XNg6OWqjd6vePjY6ndwoDNERiDgf5UH83SgUgoEAw1u7Lpwo6kN5tdQ7ObrXahWGy125quK3K33ap3mg3bMtrtJsQAwBBOUzoGN/JHB5XSb/zu7/76b/2OptoL8+eGB8cikSjEUP64uL+/DyGkKKZcr6m2qdjGfjHXU9VcoRhgAyF/mGYpxs/Jurazs1PI5eWORJOMyPsioXA4HGZ5BiJEQkzp9gKiUMgfRaLRSCxWqlb6+vp0XT/KHVy6fGFjY8Pv9wu8T1VV13UzmYyhaTiOstmsqujhcPTcwplMKvnw8dLo5Ljr2fnDA1PT/aIPQDg+Pn7rxs3h7IBf9N2/+94PfuBDPElvLq+mYwmOohORqGeZleNiPBorFouGYURiMVXXR8fH1zbXoomoL+TfPTyIxKLtdrtSPH760pXt3V3F0E9s6aqqEgjLJPoqpbKHIX8oeFwskjjh8wsn7c3d7Z1uu/PiCy8cHh4ODAxcvnTp7t27MzMzBEF0O53TpxYWHz2S2q0Xn3qqni84rs0J/I233xX4QCgQfe/Ovdm5aVXrbe+sTU6Mrq0+yWYz/WNjW8X8zMWzE2dmS9USRREcRR7njggINFWt1GvJ/r5Ku7a1vx2Mhar1GslRGInK5TJDkaOjw7mjA4rEKQLb2doUOFZXZVnq2rZJ4djq8uN0us+2TVEUa7XahQsX/H5xemqiP5MK+X04xMPhaD5ftG03FAmXSiWeZ5PJeKfVHhsZFUXx4GBvfm6mLxFzbHN0dFgzVIxAQb+vkDt64blnTVNPp/scz86mU91WO5vpT8biIkslE5FgOMQIIsVxJEkB2woJ3MTYSLvZ0nqabTpv3nhzYHQIkvjqxvrzzz7Va7XK+WIiGHM0I92XiEYjB7kjSdOvP/vM4eF+wMePDgzpXWl2ZFxgmXfuvDt15hTGM4831obGh1fWVzRVPj03q3Q7qVQyPTTw3pOHiUwy09+vKcqHP/wDJInnjg6efer6/t4OiWPjE6MkicdikZHRoXq1fPbs6ampCQDcUzPTnm7MjI5cv3SlVDzGIAQAHB0dZYcGS/Xq0Pi4B4GkyPVmM+j3cwwfC0VK+fzVC2e2N1YG+uPra49j0YCuqxxFUrZrdXoYBABz672mZuvQhQQiCJKTdLB+2Pujz73ztXfWCgbBJAeahteWDM0AHB9qdyRZ1yDhYQx0oEngtmurjqryJEEBz+rJyAG1at3FXcRgJgE9mqD8AsZQDsB0wxYEgcYxV+sGaZzGALQMCnkJkXXqh/e/9fdf+h//z9HNbwBXpknP0TXPgwASAJIAkQDgnudBD0DPgZ4HXBchhEHMBR4AGIS4C052LdxwOITjmGkZJyLFk54qhNiJCvifyHoMw13XVbVuvVFyXB3DPcvWZKWr67rr2QgDHnAc13JcEyIbYY7rGQhzHNfwwAks75mWDQDyAAAAIoj9o43ypKLlQOAhiLq9bqvdhQiPxhLIsXUMuhyOEbaq1oqu3JQbRejIgkBwLI6gpSodgcOjIT7g50W/0Ok2D48OXM/mWCbo94V8otzu1MulbqvlE3yWgyrVLsB9hbr66S9+9x++f9cRsy//2//3E3/6mV/6zd+Pjc/vVKT44Klz115qKvBbr739D1/+Uuk4f35hmiWc/M76SCqSFqmBoI/HKQoRlWrzf/7e7+ePjoYGs7mD7fzhVrmUV1W5UCgeH5cxAidYiuP5bDbrF30+Qex2u6urqxTFKLKePy7Xm616vS51ur12xzQMyzRlSSIIwnNcjqaRB1iGggAYqtbr9WRV1SwT4ESpVl9cebK6vukijOK4QDQW78+cPndxbHzy1q07q8srDEYYigyAe/782Wx2cCg7AgCELpAVrVQqaYoSicRCkUgskfA8zzEtU5dVpdvrdFVFkWX5+Q+8uLWze/r0aYjAvft3n3numc3trWa7feXyta9/7ZtD2YH+vtR7t96dnZ1BEL7x2uvPPftC4bi8vrmxcOZ0q9Pe3t0KhEK2azXb7UAgcCKWA9AdHxtTFOXi5Uvtdvvo6Ghmanp7c8u1nfHRkUIu71g2S9GyLHMse4Jq1Gq1w8NDlmUbjQbLsq7rHh8fDw0NHe4fyLI8NDT05MmT4eFhBKFjWhNj4xROQAjHR0eh581MTLI04xfEc6fPWIYZCoUImvKHQydpPo4wXVENVXvmqac6vS7NMFNTk48ePUqlUqZh3L975wc+9FIgEFheXp6fn61VSpZlzc7O3rp9m6bp2anpvf2d4eHhvYP9hw8fhiKR1a21wnFhbm6umC9YlnXx4kVVVTOZ1MBAf7fbhRAuLCy8d/vOzMwUQPAwfzg/P6tpWuEod/r06VIh35eI+/3+40JufHx0bGyMZxmSJCcmxh7cux8OBbrd9tbW1ok3Zm97J9OX0hUVAYgBKAhCKp12IXCBl0gkuu1OLBLZ2t4IhUKxWGx/Z7c/mWJZlmJox3FWVlbi8Xi309nZ3o5Fog8ePUylUoVcvlGrz87O1mo1wzAmJiYs14klE3Kvp0jy3PS0pqoHe/un508xJEUQxJUrVxYXF+PRxKVLl5aXlxmWunbt2urq8sTEWDGf29/dzPQnRVF85ZVXfuqn/vXi4mKjVn/qylO5wyPkgVOnTr19822WpU+dmlvdWB+fHMsODBwc7L/00kvNZr3WrD3z3LPb29uCyI+MDN2+824oGolEo3fv3PnJH/+JbDZ7686d9EC/pKvHx8enz529ffcOSVPhYOgbX/uqT+Qx4H31K1+KRsOcyLXa7f5s5vbt244LdNsqVSvBcCgUCW/v7qb7M4Ig1Gq1arWaSSUBcC1DA8jjRcHv94ucSGGkJEkuACTLkAxtuKYDgeO5rCBARtQR9srdw7/+5htvLu3obJiODDRkUG70WD7o80carZ5qGhiBTEcncc/vZzDP5iicZQiKJESfUK6WIHZyG3VJmuAEgWFFiFE9WVVUKRLwE8DBbUOkMJHCXbmT8JEhyuGBfP+1r3/593718O6bOG7hQHdNFUDXsW3T1N/XCUAcAQxHGAKYbTsnHSXLBY6LIESuB07uJTiOIwRd13Vcy3Etz3NOFpEAQBgiXAciiGMYwXMiw3CKorTbbVlSHcf7R/XNyXQgQggYpmaauuc5tm26ru04lm3bmqbJsixJsmU6COIn66wnCTsEHgCubqrFUt5xHJ7n/T4/TdPItBzX8UxTh8AjMBfHbZzwcMwTaJz0jGSQnByKjGSDsbifCwYxTmCD/r7+PggwgRasnrb5+Mnh1qrUqii9tij6bZd6/Z3V3/nUlz7+x18LjF/67BuL/+VP/8HyZb779juMn5u5cCrXaX7tjbv/62+/9tt/8fdLh8WnXvygrWtvf+sr84N9x7kd15BiNBZGbvNgl4X2//79X1t+/E7YTyv1st5qAFUGRldpVEOMLxmJC2F/YqjPY0mXJDCKLhVKpqKTCG+1WrZt67quKF3oWqYh2YoqEjQJCQISqqzpuim1elpPtkxH13UComQkkQjFOsV6oVCwHLPTaUm6vHW0+9a920uba/ly5dHKWq3VS8T7nrp4+dr8qYvTUxfPzAoMKTd7ak+DCOXLxzjCRdaX8EdJhMuOWVW7juPZlsFxhKF0oWFYshpLJXdyuaHxSRtAHMGxkcG1zZX+oezk5PTm5tbIyKjnwVat+tTli6V8zu8XL5+/tLq82u0pAyNjB/kc5+M5v7i+vT48OiT6hVanKauSpEorKyt3b90WeW43d2ggjyCIXqczMzdLMJTlOZevX5Z0VTWNgcHBtfV1TmCvPXWVILFr1684ltFrNXmKsTWjP9HnaMb44LDIcEfFwtjcJIWhAMvHg+FSrhANR4aGhlbW1pLxGGaYzUIpm+lfW1sTWW5qfOLxynK902r1Op7naT1Z6UmKJJerVcXQ8+Xjx8tP0qlUt9V2HGdwePzuw4drG+sjI0Mbq8uXL50fmxjuKN2LFy+3Wh0M4gunzrg43lPkvkSsU6k4moE8sLW6HgoGMRJ748abU3PTDMdWjktTY2M8SXXrjQsLCytLi4VC7vSZM6VSKRgKDA4N3Hz7redfeE5WJNvUn3nq6WK+UK9UhgYHKJJQZOkDH3xRliVZkrLZ7PLycn8mMzo6svz4SX86o6ua64JgIFypVHRd7+tLSFLXtm3P80KhkGEYFEUFfP69nV2GYw9zOdd1R4dHdNMYGh4+zhcc256enn6w+Ihj2WQicf/R/Vgy4Xgwnyul02mp3WIgdnpyemnpcTgSO3f2wp1btweyaehZq+trAwODuqQV9/OXrlytNVu3HtyZmp98+Og9kkCD/enS0SGFgWQ8urKyNj09y3HCd7/3yvmLFwJBX6VYjAg+rdNTur2XPvDBfD5fazfPXDz/2ltvplIpPy88uvfeqbnZ5eXlQqko+IT93AErsCG/7+ZbbwLoBiKB3aM9MeizXevGO2/HMglZlY4LxauXr3S77UqtOjs/p5sGTpGhaEiVlXgkGgiHFFVPZ7KVWqPSrA+MjxTKpe393VAstra13VM1WdX8wbBmWjjNHB2XbIQcjBB8QZrmGq2mBWyHhLIla7Zm2IamKQBD8VToqAH+6puLf/7NG1+/v9YjfR4VqDc0TUF9iVEM4yTdolkGo5FpSayAhyI8L1CaJpEknozHDF318ZSna8C2WZJBAMMxiveJgMRLlZqPFxgCae0aaap+EmKeS+EEgUMK2W499+5X/vrGn32ivXob4SbwLM/VSRJDCFiO7QFk265leZ6LCIyCLgYAAB5EiDD/cdMa/qOhDEJwMrfnuBbC3teQOY4DAPI8CDwMAIpnQ35fLBJORaMJnxikKRZ46ORs67iO43gQYKqqS5ICADhZUNF1HULI8zzHcQRBuC7AMOJ9mw2A3V73uHRs23YqmfIHAhiB257jAg8hjHRdIAg+17VtU/YsjcBAOChA18gkw/19EZbDCAbj/SxOkSTHaKZqGFqzWqkWS08er+KkEEkNAyqweVT7w7/83HduPIyNnfqD//OFf//Lv7681/yLz37rx37q3//Iv/6p//PZz4ihcK5ce/3GHY8QcMY3vzDXlwzlDva/99UvY906LjcNqSP3OkEf25+K4FDHgP43f/0pXeuVirmDvT3PcZHnhnz+qdHxoOhLxvsQQo1mu1StrK5vNJpdSTFUWWFp0tJUCFzoOizDkBSuq4rAUrVys91sdVttradGguGvfe2rL334BweGBtPpdDQSQy5gCXpyZnJudvr6tSvjw0OGIiuKEgwGg34xHg1PTk7H4nFRFG1D5yiSAqjT6rbb3VKpUq6WTc/hfSJFkgSAtm1rht5otxwIIUZiOKnr+kA24xfFD33oQ+/dv4vR5P3FR5VqXdW143J5eHSk1qi32+1sdlDRjEqlkohE3rv1bjIR8/HC6urqzMxcJBJbW99cOHN6d3+v1qrPLZx69HjJtPTBwcHVtbXnnn5mYGCApsns4IBumZIsT8xOkywjSRJN0yzL7u/vBwKBVF/fysrKyMhIsVjM5/PXrl178OBBIBD44Ac/mMlkaJomSXJ2ZmqwPzs8PHj+0vlavR6NRikCl3qdkeHharW6vbd96vT83t6ebVoL83MrK094nscR9qUvfenixYuGaTIME49EE4kETdOZbP/m9lYyncIIfHx0zLOdaDgiCv5yo0bz3PTsbKvVyqSS1XJpe2urXq83m02KonqyxPLcyspKMBhMRmPD2YELC2c61Xp/X4qlaByia1eubGxs7O3tDQ4O7uzspPtSJE5sb23NTs+MDA8fF/OjYyOVSmV3d/fs2bPvvfcejuPnzp27c+eO49qnTp168OABhmF9fX137txxXXdubq5RrX30Iz9Uq9Vyudz09PTG1iZN05lMZnl5GcOI8fHxu3fvIoRisUipVDp5T+p2u36//ySITyQSJ+1znueXl5evPf1UqVR64403nn3q6UKhsL+/f+nSpVu33xV8IsXQN958K9OXwhH46le/+tzzz9cazcPi0XMvPrv48BHHsBNj4yd/oq+vb2Ntc3BwsL8/fXC4Pzw6bBiabZuJROy4lKdJotNqv/76myzPzc3Nvfbq9yzDHEr3u6Y9mB2IBENf+9rXgsGgXxBfe+218fHxZrOp9KRUrK9Rq587d25uesbzvEw2E4lFYrFYNpvVdP3kX73b7VqGefH8BUmSCJq6evXq+vr65cuXg7FQrpA/f/HC7du3bdM6fWrB1A1DNZ66fn1nZyeTycTj8cOjo7Pnz2SHh5q9zr/7mZ+1LScaiTWbTY5mXNeGGFJNAyDomi6wHLnTCfs4t2f5SC8oUDSJaJoEEMtXmrSPYYP0Xrnz1VeffPZbb1d0kooMlbvuzlEVkTzL+3qSBiAGEa4oitRT/H4x3hcxTKkndQgcQddhabzXbUtSB2IAIyBGEuFoXPQF8sdFHGHxcAjautHrIMsMCozZ63hyhzR7KQ7llm9/46//6Kt/+Am5sIXjNrAU19IxBBzXwQicIAgEEfCQ57oQYBjEPQARIhBCAEDHdU+wFgxBhACEHo5hpmm0O23HtRzHOdG7Q4i5NsRwCgIcAgx46OTpHsMghBBB6LquZRkkSbIMb1lWq9VRVZ0gKJ/PJwg+giBPwErPc06K347jyIp8wvtyLGc51vs4vOsahoZM3QMeCSDCcQSgEwmIiWgoEvL5RVpRe7KtAZLAGVazbFXrtmolx9Qc07BVTZFlfzx50FJevb/7xTfX39uRz3zgJ8mB6R///37dYgP/4md/4cWXfvRTn/qbcqlw5uxMs936+f/7P/3ZH/1tJdexLDcVC9FGtbx+l/e0q3PTQVd7an6Co0hZNxTPdlngENrj9ftvvPla8bjaaHUpTsz0Dz797AfGx2YdDy+Xq4d7h/VCvVvrOZJl67asm02p5+FQ12ToGJ6uUB5qFuvNSpOiqEal99IHnvr9//2/7i4+furp5wL+0B/+yZ++/vaN41qpI/Vojk0kk/2pdCaZ5AiyV6tGGObK/PzT589fP3v29Nj4cDLJ4cixTVWTPQBaHem40u51HMPEOT+fyCZszGF4Fge2oUnFetHE3Hg87hf8HcPs6Jammz1JMj2nWC1G4mEMA8OjI4OjozjNGo5nWRZJ4p1Or1yqRuOxvnTa1NXR4aFut12r1S5cumRaTigSHRwcbrU64Vi0L51afLwo8pxfFA8O9q5dvby9tVErl5Lp1Nr2ZiQSGRodyZWPVc9utFvterN6XAaOe7izd7i3f+H8+VKpJIpiIpF45XvfS6VSvmBge3cnEguXq6ViqcDw3Dde+U6903Jdd2lpqSX3VNvsSD3TtTCaaLSakiSJfp8/Gj5uVHlRaPc6tuu8+OKLGxsbpmFAx1tbWeV9YjAUKpZLqf6M4zgEhlu6gUF04kPnREFSlZ2jAwcB23aLuaLWU/2cD2DIw0BP6eTzR2cW5g1Z3dvaIyHVa3aDvqDUlQVODPlCcltKROLpvlSj0ZianqnUqi7wzpw5Uzo+VputZCB8+/btSCzqOl6pUj175ly303vvwf3h0RHbce89fDg1M63qWr3ZmJmZ0TTt+Pj4/PnzDx8+jMViJEnuH+6PTowHw6H37t8fGBoul8u3b98+c+bM+vp6tVq9cuXKwcGBaZo4ji9vrImhAEWQpq4Pj446ntvrdLP9/fVGI5FMDg8O5Y9y2YGBRCJx/869F559YXN7w7D0kYEhtdOLxJLZseHHy0+CsUBHlV0EkQcapQqDETzL4AyB02Sj0bA0gycp2zQghATD7B4eWp6T7k8VivlIJPLss89ubGwE/QE/wws4RUDEMAzOUKpjLcyfUnuS1OmGfQGSIGzbJXFqZmrasd2DvX3LNIv5wuDg4O7+3p1774WiEcdzeZ947sxZqd2Jh2IkJA72DnlebLSahmUel4sczyOGery6cvrUQrVcNU3T7/PZqm6p5sjISLVe4Rk6EY+sb60TNEFR1N27dxv1usCwPl7QNcWzDdfRPWBVKqVYKGipckRkf/DaxY9dGeOaVqckkSTQLd1wTdEvmo6tqToGIcWTy/vyr/7xt778zqrChB3Wf1So9VoaTvhUGboOpWsQIqZWbymaxApkICBqSk/qtpDnEhiStS4puJrTtSyr2WwzIi/4fZKudmXJgcAfEGhgKeUiq2tJngsytN5rx0QmJeDK/vJnfudX3vj0J0GnipCDoOtC23It3TQAdAF0kOsAz0UIeY6HAxwB7J+msT3P88D76LtlGxRJUBSpaVpP6riujRA4UdAA7/9vgUcYcD3LcU0AHN1QHcfCcKgbqqrJBEH4/f5wMMqzfggxBDEIIATwZCpEUaRWu3FitA4GgjzLe8DDMdw0Dds0PMdG0EOBQMB1bdtzPeg6ljkzPxPwcTRNhUJBiEOKoXGCkGV5b2+v02yZplmrdbd2jvKFyvpevqp7GuV7+uV/VbWYd1fz11/+iQ//xM/+0i9//KP/4l8/fG+J4UOsGACuXTw6ggS9sX3wra9/44f/2ct3br/nONalczMTw4lmJW8b0sbK4+WlpUAw0lStJ4fHby0+qUmt4vGBrSs0jsUT6dGJ6clTp5tdZXVje2V1vSNL1Waj0+n1ml25J+myZGsSMjUWIhrh9Uqn2zGazfZzH/zAx37q304vnElmIxevXTFN82d+5mdOtNSbW+vpTN/s1NTFC+fj8bgo+hFCBIaTFJ5OJZPxGM9yJE7VypV2rar0upqpQwx2u91er9fsyAAncJJgGMrznG63raqqpsq2YTqOE4xFPAz22h2lKxEE5g/6dNOxLMeyrEIhR0AUCfgd115eXu62JJZiLcMGnhcMBrMDmVwuB5GXHRxeWVtt1VsTExOvvv5ardlQFX11ddXn85mmubj4qK8vIXV7+YPDdCL52vdf7bbaoii+9tprVy5dXltbW9tYjyYTDx8vXblypS+eAK57/szZqYnJmalp2zAz6TTHcZVKZWBggOP5UqmkW2a91Wz3umcvnH/7nZuTM1OSIq+trb3wwguNRqPZaY9NThzmcsFg8PLly91ul6KovYP9QDB4fHw8MTYWCARWVlbOnz2XTCbX19f70ql6vV5r1JPJpCrJuq5TDL22uTk+OdHpdFqNpsgLtVqNJIh0Or13cNDfnw2Hwyf1UZIkq/UawzOFQs513ZnZud39vWaznclkOrJ0AtfWKlWOZkiMrFRqpm5AhCEc60g9AKHICzRNh0MRhOMT01MQwkLpGOFYf3+/JEkcx6VSqXq9PjIyQhDEcbk0Oz8XCAZ3t3ei4cjx8TEAwBcMPFleNixzaGjo0YOHA/3Z+ZnZQi7/3HPPQQw9XFx87tkXWvVao1r72I//WKVWJQhifnZu+fHjQCDgDwY2Njf7+voqlQrwPIok9/f3dcN47uln7ty+HYyEPQQ7rZZre6urqzRNcyKv67ppaLZpQQj9fr/nuYIgHB0d5YvFaDR6Mh4Sj8drtRqO475AQFZ1jCQohkYIaZqWSqVKpSJyPeihvmRK1w3dsARBkBR5dHQcwzDTtBr1Fo6TLMsWiyUcJw3DLBdLg/3Z27duhQLB8+fPl49LhmE0qrU7795CHrBN6+jgMOQPbK1vSN2ermosyy4vL3c6ncHBYV03CYJwLWdlZWV8bNJQ9f3dg9mZGYokttZWT8/NPVla9Pt9MzMzhumm+4ca7TZBEIam0jjmIYARWMDvV7tSt1mHhnx5eviX/vVz57KU29BZ4CLXxQnECwInMBhFa6bH+H3IJ3zh9cVP/Pk/vLd9TMeGgBDLVXvVri5rrqo5AOEsy7uurWiyosmRWOykzIEwQJEIJzzXM13bCAf8pi7jBMJx3PFcDMOlnoJDFPH7fAzVLh97pk5CN8hRntQOE3YYMyrrj/7uf//W7a9/DihN0lIo4FA4sC0DAIBIEjgOAIDAcM91TzgWDGEIYQCcpOcQQZzAKdcDNE3zHMdxbE9qq5oMkQMxcLLd6nkn3VRwAsB4nodhmG3biqw5tsdzos8XoGnBsp2TC+qJgwwAoGpqqXxMMyTHsQxDAwAc1/EANHTLMI0TFt71bAghclwVozxJkTCS6LbrAHMFv4/jBUTzGM4oklo6yh9sbxG2F/JFDR1Kurd/3Mk1jckLz3/oJ3/un/8/v5aZPfu57746dvbCf/3E//iH77ym2ni7pb7z1u1CtUrznCWpn/rDP7v+7Id3C+VGK/fss6fLlYLjglqzpZva7v72P/sXP/Ij/+Yn/aks5QsfVKUvvf1ko2qUmj1NVfw0MT0y8Pzzzzcl5dW3br1z92G51gQE1rEUh/Js14XQ43iGQrZRaoOGple7rULtP/3fP3///t2mLn/5lVc++hM/eVRvI8H3m7/3Bx//+G+067VoNDwxOTp/amZhdtJPEnq7A1yHYShBEPzBgCCKx7X6YbXS1NS2JAFEYDje7nWL7UauViEJGkKcETiCxTnBde0usB0KUCxOCjRP0zzLB1TVJjGaI1kfTQd4Wmo3U4lkqVjVJYUGhFxt98fSd27dHhkZGR0clhvdwfQAR7IQeslUAsdRp9v1x8Lj0zNTUzOe5wlBfzAabDXrmAtInEQeQgjLpNMBXgxwQoAXZyYmR0ZGCAyfGhkr5Qqe55mWdXB0ODw8/O6776qqPjIysrW15Xo2jqP19XWO43Rdr9fryUQCQahK8uzUdKveYEhq8cHDZDJZrVYdy2ZI6nB3j8Lwvnji1q1bYyMjLEk9vvdwdmam02qbutEXi1+/crVSLNmagQG4ublJEMSVp64dFfIAwWq5sruxNT0+ATFUbTbOX71879HDRCQ6nMl2m52F6XnSAZVc8dTphWavbVjm7Ozs5tomcMD01OzO3j5BM5DEbz98b3R60kVwa38/lEis7Oy0JSmWSB4d5XVdHxkYqtdqBIazDLe4tsKH/IAk93NHk9NT9Xpjd39vcnrKBR7vE6Ox2N7hQavbARAapnVwlMMIotPplMtly7I0TWs1mn6/37CtQCiY7s8c5YvHx+UXXnjBtW1T1TGEDo8O/H5/o9Xc2dsdGRmhaPLdd99N9/e7wNva2Z4YG2/U6ge5o8GxkcXFxWgkwnEcSZIIIdd1q9XqxMSELKuargejkbGJ0ZnJScewRoaGHi8u+XDaRzJHR0d9/ZlkJrW2tjY0NDA8kj3M7V24er6jKvnj8g995OUnj5cpnPzQh15afrIm+PzXn7l6lN+nafzMmTPH5VI8mehKvUqlMpDNHh+X25IsRkItWbYgvHDp4v7hQUdVKUFYWVs9c/ZsMBxeXFx65upT3XrrYHt3ZHh4f3Mbh2hibHJ3ey+Z6tN1XelKH/2BD3M43a01+lPpydGxkBhkaa7X6wmCEAgEQsHI/uEhxTKNak1kuMPdHZ4kW6WS2VOO9vaL5UrXMHeKRRNiBMMyBMlzjOEYDo7ara7Ulfx+P8JcDxkhzvvP//KjP3FtIgZBkACK1Gl0W5oHJMs2AZBN3fQg5xcrBvibV5Z+5VNfeHAsu4GESws6xGlR7PTaHamDkyxEjG5atusARDSbbY6mRJaV21IyGiUwx9A6FI5IDPP7RYygKJrlBL9mOarl6LaVSMR4EhcIHJiWwFIMCWM+ivUU0eiuv/HtT/3X/7h981VgShBYOPIUXXUgcnDCBR4ArgcsDzju+1N7EEDoAQxA3AOYB6Djeo7negDQNMUJLEkj2zYsU3dsE2EAnqy3AuyEhsQwksBplvH5xLDAB3GMcWwEAEbgtGN7COII4qZllitlXdei0TAAHk2RCIKTy61lWRBCAqcc2yQwRGDINg2kG12SwliGCvkDuq7L9YYsS6quybJsWE4hX1Z1l6AjPYt9e3HvjUe7X379/rnnP/pzn/jD8PCMSYcdj6vI9u/+7z/xGKouSysbW2fOnKuV64uP7v/UT/34+fPnZaWn6+ZzL3zQBd477745OTWsGspebt+GXmZoFOd9+bZUt8mNUuvW4lq+3lEAEU4NXn/6hbGh8fHRMYZh7ty9e3iQq5UrUrfX68qGabsAOY7DQMNstpVKA8j2j/7AqU/82k//p1/4vyRH/63/+buypv/CL/zCzMzEj33sY71OVzftsYnxa9eunZqfjUVDPEcZmpo7OISuJ7BcIBAAGOop8s7eXqPV1Axd9Ptc4Fm22+52aq12q9vp9bo4jhuGKYoi7+M96NiWDlyTpalet2trRrvZKpfLnWbLNa3qcUmTetVKuVEpcywtqYpl2whhJIZn+zLtWmMwO0DTdKlQTCQSzVq9WW8g1ynkcmPjo8lU387uvqIbOI4fHR4mk0me5zPZ/tlT8wxFkTiWSaVtw1QUhef5aDiCAShJEs/zDEnRJIMwrNFonJk/ZWm6ZzsMQ62vr0ej0Vw+/8bNt688df3J8jJJEKfm5197/XWcIEbGx27duhUKhC9fvOQTxGgoHI/EYrFYIh7HcTyVSsmyPDw8rMpKt92Znpra297r6+sL+v3337unyQpNMzMzc5lMNhqNmqb5ZHXlmeefa7Sak9NT585d2NraEnhfLJa4897dZ59/zjTNXC43Oz2zvbmFITwWix0eHqYy6Wg8fufOnYmxccs0W432lUtXZVnGSeL8xYu337vDcGw6k2m0mslUHycIG1tb6f7+er15cHAwPDSqKMpxpTwyNrqxs+1BkMlkb926lUqlxiYmXnvjjXg8TpLk0tLS+Pj49MyMrut+v98wDNM00+l0rVbxgMuybDgaOdlubbVakiT19fVNz06tr6+eSAo7zdbI0AjDMO1eN9Wf2dzZ7ktnLl68eOfOnbGxsbGxscJR4drlK5zA1+v14eHhE3JUVhSEY7woAAwpqhqLxaLRqKSrT9ZWEcIajcba6vrU+ES72dRUOTXQv7m7fefOnVOnT+fzeVmWM0PZb3zn29nhEX8o/NaNt8fGxlTD3N8/JEk6f5jf2NiIxaPNZvMwdxSJxR4+fFgoFAKBwONHixzNxOPxx0+eDI+M6Zb5eHklm81ahlmr1abnZtfX14PBYDgUPNje9Qtit9tVVbUv3c9zok8Qp6amIIRzp2aHR4du3rw5ODg4MjJ2XCy3Wi1DU0xFq9UajXoLx3GKolrdtiT1pqYnKuXjVLIvGAwahhGJRFiOx3Ecx/F2ux0JhVv1pm3btuPJthNMJS2KIAKBjuVgwUgPJ3MdaXVrNxgMX7l6MRqJ0TQpCDxCCEAMEqQDoQddF2GQ4iHPH7Ts3/vr7//x575Zc0iDjVY0oLiY5SHRF8RxAiBM0rR2pwMAsC2LoRgSJ2uVGsNQvMCSOFCkdk9qMyzpQhenMCHo12xTtywXApZloecSGORoSiBwT5GChBPAjKyIU0rtzS995m9/45c3brwCoM4xmGMpjm2czLRiCGEIIQg872Q6CUGIeR50XM92XBzDEUIe8EzbBAhhEMMJAmLoZEDEA96JhB14CAAEwUkDFiGEXNtDEMdxyrHeHwBxPbvZqtu27ff7g4EgAABCDwDXcgzHtTD8fe7Sc10cxzXNUCQJeh5ChOM5OkvRjuE5Fjg8zBEQdZrNWqVSKpQAJPNl5StvrvzNq8u/843bv/ONxQODf33psGdgHcjcXNz99d/7yx//yZ//87/6+7sPHr72xqvXr1yhkHPtyvjsdLqU3xvqT2mW+eDJo/5s0rK1paXH8WQqEg8cFvZCkWgsM7qSr/3h57/zi5/4q9/68y8+2tn3RQKnF2YYAlt9uHKwf3xYqt9dfLy1teUYOmkbSNd0WbFk3VMsu6MSbf0HziR+86evv/a5//YffubHSJGsWO2f+c8/N3Jq5uUf+ef/8Ld/Rxv2eCQ+lkiemZzMJBOWZdSbtW67bps6SZLRRBJhBIBYrVpXNFX3LMSQGEn4RV+v0VJ7MsJhT5VN2xEEX8znozx3ZGCAwDBNajM4dEwXOJhhGKLI8yzn57loOOQTWB9BZMJR6Hme52iqUa1WNUNJD6XEYKB/aPj+w/u8wPl4cW9399rzzyyuLgcCgVgksvbkycTQ0OH+wa1bd85cuFgolRuN2uTY6PbqarNeS/f3f/aLn0MYwIG3vLiYyWRDsdjW/j7D8fl8sVquLcyf3t0/xEkqGo1LXZkCWK/RoimCF9hep+26Ls5Q/UODj9eeiAERQnh4eDg/v1Brtkq1+rPPv6j2pFqxwjJMp9NBCLXbXZ8YUFXVwxAnCrZhGpre6XRX19bS/f3driS1u+fPnnuyuuK4br5Y2d7dRwiXlF5ff2rncC8YjhwV8isb6+F4ol6uNSutmfHZV175PkbhXFBYerLI+cWOJpcajfPnL26sbuzt7Y2Oj2xsrBEI8/PC3vpO0BckcWJzffW5Z57SdV1V9XAwdKI/ffrpp+uNajKdHJsYv3vvPYIghoaGmtXa3NR0o1HzPOfsqbMHuwf7+/s//dM/fff2bcvQp6YnV9ZWSJoaHR978uRJXzwxPDj45MlSMhFrNuu6a6cGB+uNVr3egACpqkpSuOc55UalUq8QBAEAatRbkqQQBFY4zmf6B3b3DiLhWDwSf/fmO8V8QVK0J6troUAgHo9hGNHtyR4A8wunIvFYq9uLJxKappmmZtn6xubm+atXtw4OCJKZm5vb2toaGh3yR8O5Qn56dr4/M9jrKZcuXy3XG6ubG8+++MLmzna5Urlw4cK9ew8YQYz1pQr548HsUEgMF48Kw8OjPM9vbG0unF0I+gNytzc7PXNczO9ublw8e+7OO+/iEL927dry8vLIQHZ2fHRlZeXy1SvHhXy7UV84NX94dDA0Mjw6Mbm5v3/24sVeT97e3BobG/GQt7qz8ZGP/fArb74OKGLy1OzK5mp/X9+z16+1m42nnnoKo+jVzfUXXnhOFPlHjx68/PLLy0vLuULx+R/8wXyzOjU7Ewz6gaNn4xFH6k2NjCKSamp61/a2q/Wbm6tvba69Vyj96bfe+LW//d4ffOvBJ1999Gev3/rKjXslWdNt13MAcjzMAwzH4jjmAddzLc/zTNOhGc7CwGbN/pU/+/afffOdJgoBMX3c1Hf28p4HMZzEcJLlOVZgXQeoio5hhC8Y4cRAs9MlGTKZiThAk7WGpDRszzBsnRdFXvBVay3DgxhNW5aBLAvphohB1tVDtOsnrP4gl+SAdrz76uf+/M//y88WH94kkUU6DjBtCIB9ctj0III4cD0EMMf2IEAAAAzDPOBBAF0AcJz2ALI85AIMYuTJ/VQztK7UtT0bImC7lgdchADwHM+1EQIAugA4EAOuY+mGommy6OcZhmFoBngAAcxzbds2gGcj6ALXwSDAIbB0zdRM6EKW5lheQAzFUgTl6CbP0jhBlWoNSXMPC41Kx9wqNL/4jbdWc637e40vvbMSGj8t9Ge2K9K33nngCsGvfv+Nn/i3P/3Fr37ddqBmOp1OL5NKb6+uqrJsQefH/vWPry49nB4bJkh6dWeHF5lz585s7ezV602WJDDPPjjM/f4f/tErr9/41vdv7FdlITk4NnfK9JzDw8PNtXVdMQOBUNewXJykSLJTr7RbHV9QePEHXwplB889/+KLH/nIT/67H/nt//k/PvrPf+T2vYe//Ku/9eef/sxnPv/Vv/r03+q2lUgnnr52fbA/nYiGBIasV8vNZtPyLDHgTyRjjmMbhnF8fNzpSQAiThRsz0UY5npeT5aq1SoAiGfZeqPG8pzo97cbTbnZOTs3R+Co3WmSBCb3uq7pUhTrQdCTu7ZjOq7lejZJ4iRO6KpC0oQLPJIkgYds16q3G51et1qvL5w9o+l6Pp8/f+HS2+++Mzk35SGoKfKF02c319eB4509e3ZlZUXudTPpvlazEQgE5ufnN/d2zpw9y9OU2utev3xFluW17c1EJpUrFiKRyAnMl8r0kzTlOM7U5GSleKxLSiqVVBQFx3FJVVzg6aZxslEJIexPpXd3d0eHhlmGf+OtN0cmJllRWN/YmJmaVlWdpmmCIjudztbmpqFqyAOOaUWiUYymt3d3JEny+/17OzuJRMJy7Far1deXkqSu4zjVapXjuHa7TdMsw7HlcjkcDF84fXZzfWOgP+tBkM8fXbh8SZZlXyA0PT3zxutvUhQVCPpkWe7v7ydJspgvWpYV9Ad8Pl8kHKxUKi4EBEHYtp1KJvP5/MHBActxGxsb5Wrl9OnTpm3l83mfIO7t7I6MjHQ7nZUnSxxNeY7z3e9+d2HhdLfVPTo4mpmeXXzyeP/o8CMf/aGNzfWtra3x0bGDw1yiLzk2OfHN73w7k8kkool6rTYzNa3r6s7uFkVR8Xg8Fov5/f4Td3w4HPQ8Z2d/LxAILD5cCgYCsViMZVmSpiDCFU1TVdWwTFEUMQLfPzrUdFM3jO9///sYhhEEwfN8dmjwjbffiidTLvCePHmSSadcCFqd9vDoeLFYsm1X6nRXV1YCom9ibKpRa/A8q+lKqVR56qln6pVq/vDo+pXr+Xy+0+kROHV4eGiaut8vSpIUi8VCoVC1XpuZnaUIUpPkuZlZ6Hh33rk11D+gq0qj0eh0WpZlheMxIegPh4OBQOD+o4eHhXyn01l89LhVb7i2XSmXas1aR+rtHx7OnJrr9Lp7e3sfeOHFx48f7+5tX7py+c799xCOJdOpnd0t09L7+hJ/9/d/f/biBdtyb9y4MT09vbOzs7+/7+O5cqlI43i5XNYce798fFStlqVuyzFtjnYYtusiJp4kY0myL2lxfswfcDEC4aQoigFBdB1LVSSCwkWes23TdCyCoVTTITkBsIyBw6W9yq/+ry+8fn8HBbIq6d8pdU2MxSheMywAMYajdVPTZEWTJV3Xs9mMqqumqQeCguhjcRJIapticH9IxCmc4Zlet+26NstQADg0iRMI41i6WDjstmvxABNlUb+fyPCwvbvymf/9ic/+1n/vHG4h5AJHxxwLvA+zv8+u4BiEEGAI8zwPAgwADLieCzwI3uckgec6lgUAYCmGoehWq6Vo8ol5xnYdD3oQQYA8AFzbtTRdcTyLpkmO5QiMcIFru47l2Y5ne9B1XPvEMOM4jqLJmq4gZLO4xTEOQRiPXv0KUpqma0ACR4ZtAI44rDUfrh8u7Tb+8mtv39/vPMnr37u/64TjGgZky+wpCqTpjf2Dv/z7L3CBMC9QCGq1egl5QKR8xZ08Cd3sYH9dB0tbuW6teXpior9/4LhS/do3v6rpMk5zn/jt39Hancd33/vN3/iNt99+m3DdWCDQl04SNFUsHR8e5iqNphAI0SwjBgPBaMSyrMpBDloWHyR+6F9+9Pf/9A9/9j/9EhWPh6ZmHtV7P/N7n3rh5371dz/31nbJ87zI/PDM1VMXZien+voSHoVpwOxobd0zIE9TPoGiyFarub69U202aYL0sbyhm6VqrS33isfH9Xr9uFjkaD4cjvI8bzkOzXKqqmm6wtDki89/wDPB8XEBx4GiKARBnfBGBEkzLI/j0HUt3TB6imoBV3dtG3qAwCRddYFHcyyEEOIYK3A7u7uF0vHw+Njbt94Zn546yuUCIdEXEJv15tXz1/1ioHCYGx8eGB8ZqOQLZ+dP4xh5+/790xfPS4rs6OalhbO3b93RNWNqamrx8dKLL77ouu6dO3fOnDlzdHRk2/bAwMCNGzfOnFkYGh544403xiYmSpVyPJHgGHZzde3apcsIgHa7bdv2cCZb2D9slI4HBgY2DvduLT2IJxO3bt2q1+udTufBgwd9fel4KIJsV+T4/v7+Sq3a6XYDkTBG47KuXn/6ac9zeZYb6E+X8vlsKj2SHXQMM+IPsQSldqVIIEhAAF03d3jUF41HAsF2szU/M7/6ZFlgOZHj243mQLZ/ZHjYdV0cxxEOJVVxPc8FXrPd2tzeYjmuUq/Jcq/RqFVrZZom+7Np3VADgcDIyMhJg1cQhJM32WQyeXBwQJLk3MysXxB1Vctksttb+ywrjg6N3rt7n2EYB3n/5+8+PXd6niAISVIuX3+6XG/eu3fv/Jmzb731Vr1e/8iHfuA73/42TdOpTKbTbCWisePj41qjceHcucWHjzRFmZ+d6zRbsiSdWpi7d+duKBQyLBMR+PjU5N7BwWEuNzMzs729HQoEh7IDa6ur01NT6f7su7dvjU2M15oNVVVnp2ZzhaIHsReee/Fg/1AQfNFEvFqtRoKhiYmJVqs1NjhYPy7vr+9RgOQwFPX7OIbf3tpNRmNn5ufefOv1VCYNIazXm2NjI5VatSv10pnMYTG/ubuTGRzY2t1BOIYQunfrjq0bIwODxVy+XC4jhAKBUL583DG042Zzc3e3Wq1Ozc0GwqFEJOooOouTOPAwDGu32yPZQWA7N27cEEVxIJm+8b3XpmYm85XiGzffPnvxQqPT1g1jaHi4UCxKujp7+tRXvvGNVH9/fzTx1re/d+3yFctykulUMBIu1iqNXqenq+VO08IgxlC+SMjDEIbBsD/AEhSOIEEQgt/HchzPceFAUJMV27T8As8xuGPrHkSBSJT3s/Veh/aLhmMjAHAEXIQ7FPHn33zw+59/6+6xpkcHqjpsq7aq2bKsugT0MJvDAAdcqVHudRuCjzctB3ogGPQnUlGKxzVHKVVyJIeJPiYZ5nFX15Seaqo2BnTP0TyYHhgOBv2tRplwlAgD+1js/Gg6hpn1zaW//p2Pf/Z//FpjewNCCIDnOI5p6QB5HrIAtDxHh8CGAAcAQQ/DPIIAiAAeDlzkWRgAFEYBy3Nsj2P5cDDEM5yqKIqqIYS7AFnA1Rytq3VVXUY4OIlxHNdxPM8FwIauAWwH9xAiIMJsy1EVTTN0F3iUwFIUAHi3fPer3/zt/3D0zpcRQVCqolEECSEIBv0dWfnUX3/l1VsPCy195bA8c+mZ99Z3gBgAPp9jWgxJ1kuNoZHJ3/nd//WXf/lp6CFLUkG7naEZoqeKHlk+yEVC0elTZ1/+4R9/+63vUz42FI7ounXzrddCPt5xvfWNnU6no8hyKpE8ibwhcCvVUqlUUiR5INM/PjLBCb5ANABwt1w5ruWPfuiHX/7Zn//5WDpVajQYlkfAfbL46O0337r7cHFx60DDxOjQbDA1Ek+P9CUzIs16hmGommMZNE37A6JhG61Or1gsNRoNhqREf7CvL93uSR1ZsT0gybKumSRJ+kV/JpV2HEc3jU6vp+qa58FQIOiY1sjICEUQxWIRh8g0Teh6nuf5/X7TNCVVMx1bNw3DMk8u4F1ZghjqSj3DskiawkjMdW1VV8vVcrPVsj336WefMSw7Go3SNF1rVPOFIwzDCselg1ye5QTbdaReh8Txo/2DdrvdlXqtntTpdVvd1om1Wdd1mmMNyxwfH8/lcsVicWBg4LhckiQpEAgYhjE/P99sNvf29i5fvlwoFBLpVKlSllVldHT05s2bY2Nj/f39KysrwHJiwXBfIrm/vw9xbHpuvtlsBwMBx7Jjsdgzzzx3dHCIIxQJhR88eMCy7NjYWGYgI4oizTCdXu/ug3s9WaJocm1lOZNKbm6sHR0dzU5N7+3s+ny+wcHBQiEXiUQazdre/g5C6ODgIByOlstlz3EhhLs7O4qiJJPJWq3W19cXjUbr9TqOo1gsFg6HBZ+PYphqo34itwEAzM3NVSoVDMNoml5fX+/1etFo9O7duxzHxePxXC4XjUZrtdrR0ZGqa81O2zAsvyAKPhEgiHByfn6+VW+wFD02NnZwcOA5rqbqjx49cmzPx/tardbLL7/suu7i4uKVi5d29nZlWb5w/vy9e/d4UQgEAk+ePDlBJ3d2di5cuOC67uNHixMTE91W2zYdjqaWlx+PDY/RJHPz7RuRSGRp6cnOzt7TzzxzXCr5/f7xqemlpceeC3y+wGE+x7IsTdObm5uxWCyXy+WOCpVKxbDtWrVB4pShmdFwhKIJTVO67V6pWA4Gg4FAoNVqFYvFWCKBkyTDsY5nQxKfP70gcHy5XFZVdXR0lKHpoM+vyko8Hk+mU9mBAVXTHj1eOnPmzGHuqFQqPX39mVu37mTT/dCFBwcHgUCg0+mUq5Uf+4mPcQKbyWZJmioUCizLDg8MQggDgYDAcScDFP5Q0PLcXLHguq4sy6ZpDg0N9WTZsMxzF85rmkbT9MTo+NFRnmGY9fX1sbExjCT8sYiFQ18khNNkR+pJkkSSpN/v1xQVcxyBpDEP8DQV9vuAYxM4CooChB6OoZPDIACuokqO44giU2/UAII9WfMg7rjIw1kuGNqu9f7k8+/+/l9/6db6UQfwMuJripWrNnzBiIfhACCGpUxL73Q6EMM5TjB1q1QqBYPhcCAsiqKidC1b0bUegg5BY5Zj1hpV2dBMyzFM23Vdv8CzJNJ7DVfpJHlmMhkaj4lBTNWOt//0N37lm5/6I7tVw6BNIhcBG4eurqsIOzG8e/DkVIoQ8N5XNUIIPcsE0IM4jiECeBC4CADIMQwGPEXu6bpimubJAoHA+SiSQgghiABAALgIuBDYCNgAuJbt6JrtWi5DED4aE2iA273j5fe+9Qe/+r3P/yVjtfqDODIdhRWZVqftmBa0HI6iOR9P+0LnL1+RdDU12PfffvWXw5EEkIxOtfL7v/HrGE416j0/58M9zFKhVGj/1MXr//7KU3/8C//5P37sx1///o3FRys0zb7w0rVnX7yS23syNjWtq4bZbGXCYYbhGrJK+4VANHh0kItHE6qhVutVyzClZnt+YioeiuEewVK0CfVqq1DY3vRFg3/5+c9e+dAPag5dyNc+/MIHPvkbv0m260SrnBa5mYGB8Wx2cmQkEIvIniGZGgAuC1AmEA7RItTdUj4HTTsZjsUDMR8rINczVa1YLKmO2zN1w9JDQT9H0X2xuKZpmqHrpoEQcgFgGYEgCFWSSZwYGhiuNWsOcHRNwwF0HM9xvFqjyvJMyO+DrqebFssLJEObpgkh1E37JHSDOLQc2/Fc07YYjlUN1R8MdDqdYrE4MTHRabcjoejE2Hin01m4cA5nyf3i4cKZ047nFgqF6bnZcCxK0tTVq1dVVY1EIpKulpv1vnQmGAxPT80eF8umaV69etV1XU3TotHwnTu3cBy99r1XcAyLBEMHBwe+gL/WbqumNTYx8ebbb8eTyVdff31tff1Hf/RHHz18aBqGoigsy2IkoUhyUPQj00tEokqnt7q6KvhERVHK1crU7MzW7s7h4SFwXanXazWbyVRCDIgedFVdGR4dOq4cpzOZSCTy9W99+8q16wAg27an52a397Yj8cj5i+ceLD4Yn5zsSj1JVRYWFqRury+eCPj8Dx48EEWRZdnFxYdnL5ynGFpWlaGR4Vwhj+P45cuXj46OhoeHTyBLhNDJitvU1FS32zUM47nnnltb22g22yMjYzs7e+l0fzqTLTUaqf5sf3//1tZWIBqsdhrr2+u6aUxNTCsdiUDk8PAYjhHRSOT8qdNRf7CYL5z4FzmOEwTh0aNHzzz19GA2u7e399JLL/V6vWKxODIyAiGMRCIDAwOvvPr9/mw2HAsfHu5ns4PQ9R49fHj21MLe1raP5edmZo8ODhOJRDQRf/R4KRyLEhRTLBZPnTrV6XTqjcbZcxd2t3fjkTgf8AECi0QiAZ+PF4Xi8XGz2TR0S9ftTq/L+RlAANVwWM5/lN93oIFIzMYwjyB2jw5Kpfyp0/OFcqlSqw5ks35eSCZS9XpzfWnZklSR5ZaePME4ejW3p0N3cn623mycPrVganqn3pzsH8pt7F0+fwGHeClfiIQCkEDfeP27ErQ3D/cghoaHh1c21zuycnbh3OPFJ5VGHWdpCIBfEAFwOY5haDrACYQHO41mIBDCECF1ezRNexjWUWVEEnt7e2fnF958/Y2Lly73FMWCnotBx3GQB6DnCSfBHUlhjod0qy8YAqZtykqQF3AXWKbhEzmGIlmKFTieQA5PE8gDPMnGgyEcIoDhLqQs0zYMw7ZNmmVYDuSr4Os31v/XF169m+9JXEJx6O2jal11W5aHM4LrQUUzAEKGaVqWEwkmTcWW25rI+pOxOM8yLnQg7rE8Q9AEwWA4DjAPuKZNUYwHHMM2eI6hIKY2mmGGCuLOYJhJMs5EgDp6760/+JX/+I1PfxJ0KsCUgGkxNGc5nuMB4DnAswE8+bwPxwOAQRJzPdN2DQ+6HkAQ4ZZpQ4QYFhN4kgQusl2e4hmMsx3P8xBwPeB5AADPtRGwKGgzwMEMy1RNAuIkghg0gdHMvfPNN//stx9/+7NxAkwNZ5PJeDQZRzTPetDlOM4nijROIAADoSDNMp1WV2SEr33ly88///xT158DkJiYP3vjvUe/9T9/L5VKSb2OochSq/Xc+YuziVQ/yX777z5b3T9MBALHx+XhkTGI8Bc/+NTRwVYiHndtLyz6f/anfqrblY7yOTEg7h7shWJRzdANy5TkrtRpB/xiu9kBLjxRthYLR5ap/l8/+68WTs3+4i/+4m/91m/purm3s691u4OJ6FS6b3ZoaHp8LB4N0wTs9RquZ5IkDjFEUGTQLxqyKrd7BERBv4+hSanVMRRVV9RutwsBBhBO0ywAiGRo13UlqXuCEv8Tvua50DAMmiANw4hHE5qmdTot09R9Pp/rApZlaZrEMEyWJV3XPc+zHLcryYqieJ6DQWSb5kmvQdM0Bzie5/l8vlqzxgk8x3Hr6+vXr177k0/+0cToxNHe/u7W7vj4+Jtvvp7K9M3OTn/xi1+IhMKJaHx9fZ2l6FAg+K2vfXW0fxB5KJfPD4wO54qFe/fu6boei8Wazeb61uZh7iiT7e8fyDI85/f5JicnA4EASZIn3eV0sm9sdHR/d/fq1auO59Isg2HY48ePTy0s1Or1ervlCwZO7NUEhiWTSUWSNc0YGxtjOA4htLCw0Gw2k6m+aDT6eHEpk+hDjtdutzECNx073peUJOkEyu5IvRdffPHu/XsIx0zb2tnZiUSjhmHkj4tnz59b39yQVGV6evr73/++PyCGw+Hl1ZVkMllr1FZWVhKJxM2bNyVJCoVCN2/eZFnWcZxXX3/NHwisPH6ysrw8PzN7fHx8dHQ0NDy8tLQ0NT0dCoWePHly/fr1QCDw6NGj06dPm6bZarWuXbt2/9HDWrly9syZW7duJfqS4Vi02WySJBmLJdqtLscKDMOtrW4YhpHPH1E0kUwmLcuybZukqUg89u67725sbHiet7S05Louy3P5YjGfzzcajXa7feXipe2tLVWSI6Fwq9FUVXVseOjBg3vPPvt8Njto2caZswupVAbHSMuy9vb2EI5Fo9HcUSEYDEuSdHBw0N/fv7q6qmlarVZzXRcnycPDw+eff972XAjh6dOnaZo94Xk4jmM49szC6ZUnjxFCQ0NDezu7ly9czA4OvPHm6+Ojo4VC4cmTJ/2DA1t7O4Zlnj59+vHiIkPR6XT6jRtvP//CC+1up16tZrPZN954Y2ZmJhQKHRwcPfvs82/deMcwjLnpmUcPHlqWNT41WS4f+0OBRCKRy+UunL9omub3vve9l19+GScJx3GGhoffu3PXtu3x8fG1x8vQA+lEstvu8Aw7Ozu7f3jguK6iqblcrj+TYRimXqudmp+/ffu253keBLppWK7jIUiyTKvX9YWCOI0DBGmWrVQqOIQ0TRuGQVGkwLHddtO0dIFjCQzimEfiUGQ55AHHNDzbYmjaMjSfKBI4YhnKsVyOFWiWVAC+cSz9zTfvf+3GYtP1aVQ0r3gNGxUaPZziI8mEpCg9WXFsD4OIxoigX+y2m6qqhiIRkmVc5PWkNvTMoJ/rTydM07RtOxmLGoZhGAaG4wzLUxTTadQJz6JsKc7CMGn1cV4IyKtvvfKnv/n/vfOVz1utKrA0ArrQdSDyAHBdx3Y9BwBwwm4AAD0AIUIYhgPoOZ79fhQPgGs5nmURNMuwLHA913UJDAHouY7j2DaEHoERtm0bqmprBmbrAusxtA3N+tqrX/zS7/7Kve99kdUbp4dT/clYpi/j8wUsx0GaYbgetCzDUDUcIJak/H5Rk6VWsUoa0JE1CmKpeN8Hf+ynf+bX/uDvvv76a3fewWlgea5l6AmRzoSFuemJUqmUSWS21jd6antpdXFsaiZfKH3za1+tl4+fvnIt4g8f7u6ViuVMJkNRVK1SwnBICHzPtuqtJgKebahyt7O5vbW+ufHmO68/fHTX6HbPTs/l8/luq/3dz3+hdXQQ5MnJsexYNjM5OBTw+XXL7hpW19IR7gJHjYgMTxEsy+I0W+/2JF0lGFrRVIzA6s2qpvR4nsFpDJGE7jokRTuaJlAUDvBWuxuJRR3HNg0dc4Ft2jjCTUNzbFPTNNsDoUgEQmhZBga9bq/nAXSyr8bzvGXZuqEatkWTjOcAyzJ0Q5UlCQFo6oamKBiAFE6osnICohIEUS1Xnnvm2Xdu3Lx07sLmytpAKpuKp1Yfr55dmO926+/cvHH+7DlLtfMHxb54cm11ZW1p6QPPPFM8OOo1WsPDw6qukRR14eLFVqtVr9djsVij0aBputlsLi8vLyws1Gq1SDhczB01GjVVlQWeXXm09Pju/WQkce/OPY7jNE1zbDsQCDxeW6l0W6TIbR/ujw4Ne5bd6LQpnjVNm6Koar1ebzZiicR79+/NzM6qqsrz/Nzc3N3bt0OBQKqvL5/PRyKRarV6mD8iSZwgMZqm3333XYbmDg8Pi+VSdrBf0RWAvFAosLWzPTYx7vcFb995b3x8fH9/v1StfPCDH9zZ2REEYXBwcP/ocGpqimGYQqEwMjLS6/UwDIvFYpIkURTV15fY2tq4cOGczyfs7G6dPXu2Vqs1m82+dGplZUVRlJGRkW9+85uCIASDwVe/++rsxBRBEOvrq89cu3qcO3INI5vpX1pacjw4t3B6bXMLIHj+4oVSuewL+AVBuH//vue4DMftHR5gOD6Q6Y+EwhzP9w8O4Djuui7J0h6Oer0eAmB7a6s/lR7ozxaLRQwjQqFIs9eZmZv9zve/t7SyrGra0uPH1WpVUxTkerFwxDatWq02NzdHEEQoGESuNzw87DgOieGCIKxtbQpB/+T09KPHSwjHOD//7t2buq6KjP9w59Af4Drd+sbG1sLcuUalvrO+eX5u5uGdO8CF0VB8Z3VjcnQUp8jXbt6MD2QaUrfd7V26eGVvb49n2bnZ2e9+81shTiQcYKvm0MDw6sb64fFxOJMudFqBeIRg6UIxPzY8EuTF1nGFQ4Sr6Ie7+1pPPtrb9/v9A6ODj1cf0zTpuHar1frIyz8kcBywnUwmJXXbpeLxxMjo0uJioVAYn5rOFYo0yQiMUDsuH+7sMQzTbDaDgQCOMAIjA6EIRjMuQbQ02WWIjqG2TRX6uWOlhfG0R2IuQh6OTEcjSBiPR2mCRNBjKZLAEEWQALg0TVqWhSEPN9V4QDTVHnAc1zCR45m647kAAEjQJGTx79w+/OMv3vjeYrGJBToeX+hoi5u7muZm+wcRTtiupUhtTWt3utVYXxDhqNpoBONx3h8IBQU/R+Ce3WnWSAqxLJXP5xVJJknatJFkuaqHdIBbjqv1JGjIPtJJ+4iIK48KGN2pP/ret/74N/7r0qtfB2oLWSpwLPdkQM/1POB4AHgItwEGAe44wHZtADyIgAdMgAHPQx7iICECQHjAQxgAQDdN2TEVgsARcl3HMAxd112A8QTLIxJ4hcW7n/7vX/y1f5W//eWxADwzlomHBNu1bA/zMIrAWRwyCHqApghd14HrkQgLCQJNeH2pMMcTg9m+RrmLOe7l8xcUVf+Lz3we6PrDJ0tTs9OpTD8iaICgYekH+fzI1NRzLzz34Zd/iBPY5dUnwIWGqqVTCd2QAkFff3+/bjqyqgqC4BN5Fid5mqUoiubY4ZGRQCiYTMYZlmZF0cbQf/4v/+/CwoJpuOtr25VKzTS0+dmpi2dPj48Ohnw8Dtye1JFklWIFmhc9SBA0xTCUa5o0jum6IfUUgJCiaZV6oytJjVYTp8hIJNJoNBBCmmngJGVZDk3TnuMQJOYL+TRTF0XRx/Ku7XA0o6oqyzIsz3qu7eM5jMC0k2lax8FISrdMv1+0TV3uKSROURTlWAaJ4ZZuOKbDsQKA0HVdvyDiCFNVvdeTaZpuN1s0xQIAotHow3v3f/CllyqVyrUrVymCVFR9dn6u0axhECzMz62uriKIz88vPH78eHxi4ur1qysrK9lMJpvNbqyvz8zMRKPR1157LRaLlctl13U/+MEPViqVvr6+S1evvPbaa6dPn15bW8Mw7KMf/eg/HRgnJydPHuQxDDt16pTI8VK3x/nES9evdlV5Zm5Wk5Vus9Wfzuzv7+MEQTG0ZuiCTzQMIxlPLC0tKYpSKB3v7u5ev34dx/GNjY3nnnlWV9RmrX5mYX53d4ckScvQQsHI0MBANBxhKbpUPCZxHLheLpc7f/68LMs+n+/69euyLGez2Vgs9mDx0ZlzZ3EcrzcaCwsLvV7vRHJ5QtYLgnBcKFqmnkqlHMcRRXF5eTmfz585c+bJ6ookSYZtFQqFEzi90WxeuXJFluXd7Z1TczOdVjsejfT392+srQ+m++Ox2N7+ztTUVLVWfvO119PJRLFc2jvYjSXiEELDNM+fP3/nvbuGYcwtnLr34D7P88BxV1dXMRw3DKNer4sBv2lZ4WhUFMWTjm4ulzv5wdN13dB0hNAJJO66rqIoJxMKsqKdOXMGAWBZliBypVKx1+4MDg5+//vfHxwZDofDBwcHV65fW1lbvv/wwejoyOrGalfqRCKRXq/X63THRyeWlpampqYEjj/cP3j55ZdDAd/Kk+Vnn35GU/Rer3ft6pWDvX3Dtj7w0ofWVlZZimZp5vHK8qmzp4+rldXV1WeeetrH8UHBF4lEDMOYnJwslkutXjcQCcuKEgqFMpmM7VihoN8v+h4/fjwyMur3+23bPXNqYW939+Dg4NyFC7u7u4ZhCILwuc98NhoI+XhhaWlpYmLizJkz3/3ud3/yYz+ua9qdO3c++tGP5g6PGIa5cukyiRNz0zOFoxxFUf39/bqun/g7eZ4/CZRN20I4LikSQsi0jZ4sn6xtQOjphqbrOstQ0HU82yJJEkAXw5BharyPD/rFdNSvNtuk6xCuDR2bJDDXdSEkCIJyPWS5kBHprk1+8+b6H/zltzfKPS45hvvTq3vF7YMiRrGWBxleMG0LEqDTa+u6StFkV5IIEvOA4wEHxzGEkGtbJIXbluZ5XqvVUnUN4aTpORhFYhQpimK71ZC7LWDK6SA/HBH9njbgo3it+9oX/vZPf+U/P3z1m06vgRwdxzwMeY7juN77QnYXeDhG4oiwgQcAsD0XQ5jzPq2HTjTCjutA6JEkTuI4cC3HsjxDI2xNoBGF9NLiu1/7w1//+0/+mlLamOoPnJkeSEZFlsJEnrJs3bQtTdeb7Zapm4gEUOvJgiAMjwy6qopZhk8geREyIRcXrb4Ec//d28WDQ0vt9iXEn//dX7dMNRKJhuP9Lc01Of+X33z7vZXVu0uPXBoIIVE1LYqgjzZ3JgcGVpbvRRKBntxKD2S7qlGptxWlKzXrQDcFRLcKlbgvjOO4L+yfmZ/JDPWz4UC10/mHf/gijjEY58/Xmo1GKxQKRRPRrtzq1qqmLOmG7EAPcayHkZ4FGZwxVce2oKyqkqzKXa1crhdyeVmWAQCsIPD+IM3wumUihDwXkRhBAoBBZDkeokjN1DHoQtdDADqWBWzHsS2EI82xDcexNI2nqIBPNG2DICiCoBBCDEM1mlWOYz0Pug4wdIuiKM+1CRzRFCXLKk5Stut0JYVmmROvlqnrGEKapoVCEZ7nG41GqVKpNerNdqtQKd15dB9jqLWN9UqlEgxHDMvmBCEYCVMso1qGDl3NsxtKp1Apjo+PVsuVlZWVwcFhx/EioSjPsOsrq/2DA4jAHz9+HIlEDg4OUqnU+Pj4+74UijrI51yEETwbS/fZrrO6unpSdBZFsVypiLzw6L37elfydGvpwcPp6WlEYa1eNxAKHh8fHx8fx6Ox42IxHA7TNB1PJnqqUm82E8nk22+/TeN0f1/6wd176UTycGfPUvV4OLK3s+tYdiQUDvr8uqpBz03F47tbm7tbm6rcK+SPCIo0TDNfKCCE2u327v5evV4vlUq6rjcaDanbk3vS7vZOq9GYn5ntiydqtVqn0zkJzUZGxsqlqq7rPM/39/cfHh46wEulUslkstls5vP50bHhQuHIMtTCcVGW5ZHBEcuwq7WGIIr5Yi4c8Ecj/tJxPhDwNTvtrY1tnhdZlm+0mvMLp6rNeqvbGZuc2N/fl1Rl/vTCu3duCz5/uj/7aHGxf3BA13VZVWN9yY4sYTQ5NjFZLpeB6yWisYPtXWDamAdSfX0z09OSIueOi4l06jvfe4Xm6Egw9OjBw3RfH4kT9WolnU41m/Vms55IxPf3d1PpNEGRtUb9hRdeqFQqOMJOnz7d6bV1x5gYn9zb3uMoOugXN7Y3aq0mxPDcQU7gGB/Hdjrd7MCgoRp765tnx6fDNO+ahs8ntDWFj4VYgfcg2Nvfz5WKhucsri4jiKdiicLhkdLt6ZrW7XZZln3w4AEicN7PJ7PZYq3iQC+Z6qtVqlOj4ziAh/v7qf6s5bi6pp09tbC3vSP3JJZlt3Z3Or3u9OTUzbdvDCTTfk44LhTD0UipUl7ZWCMo8u7tO5lk3+HhYaVWbTSbJEkakiI32hTEbEW3NQM6LmbYtAORDUSWsU0DxyCD0wxB4xhm2xbwbJzAbNfDcFL0saKPIUi822kN9yfOTaaBZIkQUTggSEhQpGFZCJIIYK7rshxtGAYv0iTP//kX73zu1fv7HQ/zpUst86jYRCTXbMseojGCBgC4jtFr1wjkOo4lKxrN+RBNARzz+XnHUFmOFn1sOBpiWdrQuyGf4AFHMUwNuKTooznRcjzL0KBjDCdDQWjFMWdYYEGrduc7X/v0//hv97/7RWB2ILRcx3RtB570loBtAcsFOASk5UIEaRcABzg2sB3gugC4AAcQtx0IAKYpqq5YJMQJAkN27/jtL3/vE//hxl/8N6GxfnYo1t8Xx1i/hjjDIz2csFzLMlVVa1u2yjAkwhGyHRPDAc/R9XoVAIemcFvXS/l8KOjr9NoAA48fPx4bGvzwSx/4oZdfeukjL3oArG9seQSB87zHsTbP7fU6A+fPbtZKr95+l+QEiMjRwSFREJptaenJcuW42JeI6ba7sbf/9FPXm9VqpVZttVrddkdTVIBgIp2RDa2ryIZhBP2Beql279Z73Y58sri4svJEURRdNXiaE3nRHwhADLjA68nddqdum4bjQEnWu4parFRbvY4gioLfRzAUxtKa6/ZkwwW4bXkEQdq2iyBO4pShKQghiqIghKZpqrJC4jgCiCAIhACGQcex5V5XVeVkPOa4VqfTUTXD9aCmaZ7nCX5fvdWkaRpC6Ng2jgjT1EkSRwBzHEc3TVLgVMvoyaqsqgAgVdYYki6XK+ubG/l8HhFErVEPJ2KKqWay/deevtZoNLL9g4IYlBV1YmKi02kvPnw0NDSEENra2QnFogAAXdVq5YqlG9ls1jRNTVErlYplWZFIZGdnZ2tra2RkpFKpDA0NGYaxsbFx6dKlVqsFIXb67Lm3331namZ6a2sLuF5AEBcfPpqZmi6VSq1mM5VIYhDRND0zP3f67Jn7Dx+GE7HsUHb/8LCvr4/n+c3trcnJScMwTNvyIFar1XCSyOeL169c3dzc3FhbOwnlM5nM4ODgwcHe9ORUo9HIHR6Njo62G03TsE5oxWvXrrWbjXw+H4vFFN2oVquCIHSlXjyWGBsba7fbNE33urLjOFNTU3Kvp8pKOBze2dnz+Xyzs7OlUgnDMJZlO53O4OCg67q1Wm1+ft5xHEVRbNs+ebonSNIf8jM8A6CnKMr29rbjua1Ou9Vpj4+O5XNHjUr10oUL/mAAx/EzC6eVroIAnJiY6MrS2Qvna7Vat9t94YMfwEmiclx6+cMfaXU6tUbj+vXrS0tLqVSK47iHDx8mUn2SpPR6vfn5ecMwXNc9d+7c4eFhIBCIxWIbGxsnM1KLSw/PX7zw5MmTXO5wbGxsZWWlP5uemZlRJXkwmzUdu1qvLZw5vb+/39eXSKYSjx49GMhmQ6HIrXfenZub0XVVVXUcJ6v1GsTh+/VaxzU9x7Asy3V29vdypWIoFs0MZIv5gqGog5l+URQbrSbDMCRJ4jgeCARC4bALPMdxcIRIHI+GIu1Gc2Zy6mB3r5DLZ7PZo6OjjtSrNRsEw3oQ1Wo1AMDy8jJJkizLrqyu0jTNM+zWxmamvz9fLBiGkU6nDU1/+PDh+fPn37n1rmEYoihurm9cvXrVtR1N0+bm5pZXV8fHxwulY5KhLcvCCJwkSQzD/H4/zwq4h3mWS2L4yTf4vgsFIQAQhuMERQqC4NoOx7CuY6m9LjBNCgCRYUql8lNXLr/83JkUj1ltlbBVTe7hBDoZIOV5sdXqQAhxjLRcJPDUncX833z55rduLmpMpG6QG7lmz6FtQtAtvNGQWcbPcYLck2zT4n2i7liaYQAMMCxB0DhB4IJP8PvFUNjHsURPqvtFhucoCDHb9SwAGI6XFK3TbdmmGhKYKEdxrpqggc+RULvy6NVvfPKXf2npze+SOCBwF7mmaxk4AAggy7VtB+AItywTA4CAHgZcAC3Hs13gIAhx4Fm9BsNiNFJBK3f3S//nrz/+C/de+ZzfbV+dGZgfTkZFEdgOTdCKajR7ak9z6h3do32QDuFCfLfS/dy330AQcxDmdntNWWmzYbZnqT7KH2ficT6ZimXYQGT76HB7dTW3u/vqzbe+/vqrtmn2dP3ys9dUR7c9m4tHX9vb+PgXP/Pxv/rLr9y8fVBpNTvK5//hS6cXLtxb3P7RH/3p9nHlhUuXhsbG17a2w37fxORQQ+u2bE21zVqjTlBMsVIu1Co9VRlIpgbC8eH+ofHRiWcvXScc0G71YrFYOBhJJfqhS7RavXqn1VG6tqNBzLQcKV/abzab5Wq9a+k2BWkf1zV6NoV0AnZcA3I8AJTaMR3d9WyIE5SHMIiQKIrANjWlB10PQZzjuGqjquqqZuiqZkqSAnRdpEkIvWhfzHVd0zQBgUEMkSROYjgEOM3wAHkYDjGIG5qJYYSiKLquBwIBDweNToPzcbZrERSNkwTHCqqim6ZZq9ZPnz4jyfLg2Mj9xUcky3Skzmvf/vbs+Hij3u501cGR0cUnj11Lf/GZ6/feudVttEeHx3a2dh3dTCfiR/v7sXD4JO/OZrOT4+OGpouiSNM0juOaroui2Ol0SAyPReIP7z8KhMIjY6OLj5emp6dbjWZ/Kh0TAwFWGB4carVaHMPEI9GN1bVsNltpN/O1cldTKI5ZX1+/d+/e+fPnSZLUDD2WiO/t74t+n8gLzVp9YmIimUyGAsFyuTo+Pj45Oem6IBqJ9WXS+ePiB1/60N7BLk4SLM99+5XvDY+NBsKhg8OjZDL59o0b4WgkkYg9fvw4nU4PDAzIsjw2Ou553u7u7sLCmZWVNY7jMpnM0dHRiXTh5o0bk+MT9Xq9WCxNT88Wi6V6vd7f37+3t4dhmOd5JwE9hmErKyuRaJSiqOPjY0SSB/k8RhDJZJIVWF/Ql8gkBb+PoelYMGyb1t7O7vHxMU3Tuf29brNhWdaDxUftTmdldZUgiGQy+fbNG4Zlea5788YNTuB7spTP54M+/9LSUq/XO3Nq4d7tOyfK3+XVFVYUcqXi8sba5evXmu3Wg4cPSYqqVyssSZxfOFMtlhYWFmKxmNTpToyNtzud27dvO5a9v7uH4zjDc7nDI5IgZE2VJMnn88myTJK4IAgCx/MUc7C3Pz09ncymHq08mZyYzuUKPU3KTgyv7m6Vms2x6cmjUrHnGHw01JS6qXS62Wq9/eabLz737NbqqmuZiUj41VdfHRwcPDo4lHu9D33gg/u7exzD/uBLP/CFv/vMxdNnR4aGdza2zi6c5kiW50TbtmVdK9Vq165d8wcDPV2dPDXHc5wma3Nzp3TbgRQ1MjHZrLdS4QSy3XQiidPk8OQ4LXA4jjMk02u1z55aCPhF3bEGx0d7ihyLxWq1mum4PdM0METxgqwqOIbRNE3QlGqaBEnZACKScoCnmo5puxZw272uaToYwHHHEQlSgESCDQgO8NPCfq5VbSuzA8kPnRt76XSIM9y4HxHQgZ7led7JAxxG4Jqu4MAjIPJzJEZSby4d//bffPtBQbHDo3kZ7h3LzRbw+wdcxLuAgYhUVd0wDI5jAj5R4GjDNRxkezhgWRZB0O00RR8VCjGSVMOAlY7FGJy2HU+xLDEctgBWbfeacs/29FiIGQxSgyIR9DTeUFC5+PZn/u5Tv/wLhcd3gacSuAdMyzVNBKAHHMdxKQwHto1sG9gq9CwTmBa0DFOG0CBIU19959Hf/NbXfvvfdZZfmYvjk0ORcCygAmQC2tBdH+0jTFeqNU3dbisOGRiuqeJbi6WP/8lXf/vT331jo44c4HnIw3HccZyOrAAPBXnO7CmHW0e2gQAiyuXS6oP3Pvris1K99W9//Kf+zc//UqOnSLrB+0QKkQzDKp67VjjqIvfvv/K5V9962/LQ5uam6PMRtH9z+2hteQV6jm3buUJ+enr6x37sXzIcG4yEEY4q1erq6nqpUCrX6sFgkEJ4MOALhkSKxhSpl0okp6Ymo/FEuVrLFwtdRSYZ1jAMhFA+f1SplExTh65nWVYkEhV8fopjdVujGNKFwPFcx3VVVeUYQRB8gk9EwKuXS8h1pF4PekBRFAwiRddkTbUcm+NFBDGKpB3gnaCBqib7RD4cCHY6HYRwx7ROup2aphmGxdCcqsqGoWEYZtm253kcJzAMo6pqT5bYkyjWMFzXVXUTACSKfkXVLddZ39iIJeKNZnN4dIRmmUIhd/nC+U67HQzH+FDo/tLjRDLFMMz99+4kk8m+vr52uz06OspzXL1UuXrpci6X29/fP7nO5fN5hFD+KJdO9mUyGUmSMBz38UKtVkMInT17Fsfxnf09kqYLx0XbsF3TsnSDp5k7d+5ohq7r+sHe/vjI6OOlJX8gUG81V9bXYsn/H03//eVbut/1gU/Yz87fvb85VX0r5zo5dZ9zuvve2/eKK4lrC7EwWLbABMOa8Rh7wLPGMwibYWFgLQwOCxkMY+yFBMiSEAroSjf07Xi6++RQp3Kub85h5/g880Mxf0NVPbU/n8/7/XqV5ufnFxcXd96+NQzj3v13a83Gh9/59uVt89atW7Vao1FrKqLS7/aiKOoOR4Zp7Rzu/+ijH/th8Lu//7vLG2vZfM60rdm5ucFwPBoblbnZvf39h+89GJuT8+rF7Oxs7fxiMBisrKx88dnngiAsLi5//fXXDx480HX95ORkfX1dVdVsNru5ufnmzRtFUSCE9Xr92rVrk8mk1+u99957oiLPzs7Ozc3VarWY0kqlUqvVEEL5fHF/7+DG9ZvdbrfZbAgCb1kWx3EQwtevXyeSyamp6Ziy2fl5JaFmctlyuWhZVrk0nS+VE4nE6urqwcHBw/ffv6SjfO973xsOh5ubm8tLS/t7ezeuXctn8t1u/xf+o19Mp7IUoP/4T//p0Xiga6mbN25/8eXnS6tL6+ubhAgPHz4URfHo6OhyX2R7vhcGewf7EOLFxUXHcZaXl6MoGvUHhBCEkCrJGOPL/nC72ZoqFfb29nzfz6YzO9tbhmFUKpXT4+Mb165LkrSzt1solNLpLMTk9u3bjUbj7du3vCg0e51as5EvFh99+tl3vvWhY1rNZvPmndu1Wu3WjZuqKH/++ecrKysTc/L8+fMH773X6bRTeiKbSm+9ep3JZDzPM03btdxsOvPll1+KIh/HYa/XU1VV5IXBYFAulyVJUnVtdX2t0++VpsoTw2g2m4VCwTAMQRAIx7mG9ejRI0EUu71eGEfpXJYjgiBIoigCBBGHLctKJPQopFEQh2GsaRpjTBCEKKSOF8VxjDgsSUoqnREFGWMcMUoI0XX9cjgWBAHz0ulFXRRwUkZXF8s/983FJKM6DCNzkk6Ivm0lNZVAoEgypRRCCCiAiMQ8GgTgD79+83f/yb/4cr/e8EnDQW/Ouj0r1rNTQUgZQBFj44l5aT6amipBDAGgXujFjKbTScJDP7AyKZXFXrfd0DU5oYo8hxkEHE9SuQzgUHfQbdZrIHDSEjef1WZUoSLjZGyzfv1/+/t/61f+3t8yqycQAwIZATGBlMcIQsZoBECMQo8LzAT01GAkRoPakx99/P/9+7/3z/8nv7d/fSF1Y7G4WE4VsolMVkM8Gtl2azA+Om+aISwvXJlavlEfxv/8tz/6O//oX/zmD79qO1DIzQu5OcQRPggZRyQIJQEl04lsaI1WZovlXMnoOwqvBqYH/UEahXkK7ebg6s133rx6KygJRpE/8qO+881bd/7in/7FP/tn/8xpo25awc0bd15vvaq2ajHEzf54Znn+6+df63pCluVf+m/++vMnz7/94JvXV9aXl5YWFhbSmp5OpFYXVlRJHduG4ZsDu9M1mqZnZgv52ZkFjHifBi70x6F53r4IwrDb6spSQpF1wISEms4mMyCmoRfTkBKO4zByJ5aE+JQgp0UpDr2IhpZrCCKX19WUQFSR931fUdQwYqqWkFVlaBiO73uOb5uOLIi270UcpJjLZzO+aXpu4HkeQRDS2At8xAthENOIcQjzPB8DhjgSh5TnhJjSIA5URQJRKBA+l8sBABRFDaN4MDEowrVWOz819ebNW1EU+93eZDy8d/vO9pstx7T2To+rrXZxZua0XjVt69a9d/aPjw5OjqdmZx4/fjwej69c3fzxxz9WNeXW7du/8zu/E3hOLpM+2NtfXl4eDHrV6vl0pby9s+W67rVr137zN3/TsqxOp3N0fLy0sRZjiAhn23YYhhzHFUr5VCYZxEE2l3YM897N27Wzc8TAw3fvDzu9yA/6nS4CUNf1z7/44s6777x58+bt67fZVPbF8+dTU1OXcckHD97rDYeIJ5lisVSZ2bh2NV8uyVpie/etYRv5cklOqJwgSmri/KK2fu3Ky603jmc/eHC/0Wjks7n1tbUf/vCHd+/ejcOoWq0uLi6enJxcvvitZtsLQlGWjo6Pr1y7apvOZGyWy9MXF7XlpdV0OvvkyTNJUlqt1vbebrlclmW52+0uLCyEYTgaje7fvf/pR5+W8sW5ubmjo6NsNkMwrtVqd9+9v713+Hpnv1SZq9UaZxcXej795mCX53k9kTAMYzwe12q1yWRijIzADXa23g66vW67Y5pmvV5fW1uzDLvb7bZ63a3dnScvnzfbrX/6T/9pFEWpZPLi/HymPPX0y6993x8MBh999BHkoCALIaAMowiwxdUVw7JCP6AUhHGMMFF4OZ1Mr29uBGHYajTnZ2b3tneKubwi8J/85OPZylw6nT462F2en+80OtbIXl9f/eLRZyAG3/7gO6NOL59MZ1Ppl09frlXm+YjVLqrL62uA5wzHevjw4W//699aWFiYnpt98ebF/ffuD3v9XC63efXKq7dbqq6988H9Z9svb7xz58Wz5+P+4N69d7e3d13X/6lvf8ccTKayJYzxj3/847t37sS+9+LpiwfvPpyMjK1XW+vLK0+fPj1uVKdWF79+8zJZyL333ns/+eGP0uk0x3GNam1zc9PxvGa7lc5mer1erzfY2dkJw9B3PQFiHmAOoFFvwCPsWJbMi8Zk4nuuLIoYcLIg6QkV0diYOCAmfhjwshBD4LCoPuz5PIpVfhTZgipt7+22JmMPMEDj6VTiZ28v3Z9JZTAA9lBXRRaGHKU08APIHBYQiWeMpvQkhACI8pjhf/3l4a9++vizs9ZITO3U+jvHtRjwsqKJSirGnOMHYRg2a3VZkhRFgQhRxLzY53mS0hOKLAg84gU27DdQ5ORSMuGYmhAppHEc5wrTIcN9y7UdLzTHaTGeyQplDSixNauSzs7LX/6bv/Rvfvkf0FEP+jYXR35ge7HnAg8QBngIAgMcvtj6tX/423/j//rlv/wf+HF1Y6GQzKgxj0NCmCAOx+NetwmBL6lCtjKrVparrvQ7j4//yt/5J//rb/34R8/3Q1knuQwQRcQRx3IRiokqqK7tcAR5YdTr9WQRJxJk48qViWFpIi8gsP36eVKRzrZ2/tkv/5OR5QIGgyheXV6KQl8QiOfbf+Nv/NJf+2t/La2l+52+MTJYGP7Mz/zM1Mzi1s7+9XfuPXv9PKUnFqanVSlBIcIYe543Hg4goLlMLqUlIzvstbscx8Ugtn2vPFMpTJc5URoMRr3uIIio4/qQ5wRZCQKazxcTiYQgiIQTAcOm5QQx5XmRxYxD2DEtTJEASeR4jjHhORwGjuM4EAOOJ7brQAhpGCGEbNs2DGM4GhmWCQHmRSmOmRcGvu/bvldt1Au5PKNxFIaQAcQAjzkO4cuyOwCA43ie5yGHAUIQcaZlhXGU0vTYCxBlLIxiPwAMTgxDTqiSrAiKKqkJ03LuvvuOIsssikXEdZstQoTF5VVZFhvNmp7WKUB6Km1YTrE8vbSybBhjXU/cuHpFUZQHDx6sra1tvXlz//59COH5+fnDhw/r9Xoikcjn841GI5PJmKYZx/GdO3eG45GkKqsbm/uHRwhxxmjsW17ohYhws/NzMWCSLLc7nUql8ubVq9nKzMbK6uuXr/L5fBzHtmVtbqw1avXLBmkYxpcbD56Ih8fH3V5/bmH++9///vT0tKbptXpTEKQwjI+OjhKamkio+WKBEIIQNx6PLcu6ceNGr9erzEwlNGVn5+2tm9ffvHnTbbUfvPPu4cGBYRiLi4vtdntjY0NRlIODI0rpYDBo1Furq+uO4+TzeVmW281WIpGoVqvNTvuS/95qt0ul0tOnT4fD4erq6rNnz0qlkmVZBwcHxXKJEMEwzKWVlUaradvOeDxuNpsUgMW1lTc720Tgs9nszt5eaXrKD4N6vZ7Wk5KkuK4/MzOzt7sLAbh582atVsvlcuPhCAMIYioSvlgsViqV9c2Ne+/cn5lbuHr9Csa41ezQGPT73StXNjzPK5VKSyuLx8fHY2PCC4JpOaKknF5UJUWFmIMQptNpx3FEUZwMR19+8ahcKimSuvVqa3F2YW9nFyHuxo0bjVbTsMy1zY0gDBfm5lRZGY1G77zzDs+Rnddb2XTu5Oi41+lLvMDCSJfVKIoarSYnCRPL7He6N69eOzw4aLfb6Vy2Px4eHB+1G82Dvf1kMtntdpvN5vT09GeffTY/P18sFk+PT2ZnZy3H3treWVpY3ts7KJenb9++vbe3l0qlNq9f++jTjwkhhXx+b2d/droS0bjaqK9vbuwfHpydnN65dbt2fuGHwdzS4vHF2fTcjJ5OOb6XzWZlQZwpT8VewAGsymocxxhjQniAoKQk3DAEEBNCAs/3XS8KwsDzAEAcJmEU87wYhjEmXERjLalDDBmkoixms+mR6Z90OlKuIKuJ0LTmkvoH1zf/2IdXl4sqcWwVBgKGPM+7oYslIYCAImTbpsDxhukEMVLSan3o/ptPXv/rn3xxYYRtHx40R4fN4cQDDEm2GwFECBFazTallNEok01xHArjgIiEQaqlVIRpvphCzLPGXR7HPEE8YkHgIQDLUxUniPmEigRCeC6K3Xxaub44VRBAiYvLJNr76uO/90v/1Q9+7VeAb4oclbhQ5qLJ6Zunv/m//egf/d1f+x//P729J5UEuDqbm87IqoAW5+cUPTEync7QDDkl5tN8ZsYmqc92qv/0tz/6B7/6W//yh48mULOQmpqaj3nFCaOxY5muCyBCzEDUARLPwsiQVI7ImAI3RuFppx4LoJhTZ6eU03r99eGhKMtSQr374TdANnt8vH/r5mbMR0JBfXn69qvXT37nt/7Nt+5/cHqw//79dwED//pf/5vrt+/94JNP/+bf/dvPXj2PHZu6jmOY/cHo0fMnjV6nWCzmUmkWRJ7pJkRVkxKR68cRy2amTDOaWJZhmV4YcBzvGq5CEhwVEeNFJRkDHASBMZ6wCFHAIUVEomiYLs/LiAFVVkTIA5+KhMMYRLGnq0o2mep1h2MaTyCrNpscx/m+z/O8admiouh6yg8DLwj8OMKciDEncmS2MlWaKo8mY9cyBZ43HTeKKEHENS2OgDByoxDYVogYwBwEBCKB6Lruu0EmoSuYlxGhfkg4ThRF2/EiyBiHW72+bdvm2Bz1B9P5YiWbW19cTqbzEQALM5WNhVnfMJbm5qOQ9QfjbD43Go00PcEAffLkSe2iVj09e/vqdTKhMhoV84XN9Y3D/f3rV691u92tra319fVLTJgoihcXF9evXwcMvXr16jsffvvs6AQE0cN77+7t7d26fffZyxf94UBOqPVOqzBVxoSErgdjViwWs9ms5TqVmZmj/YNyqcAorZ2dF3L5TCYzOzs7soyFlaUA0FdbW/fffXh6em5Zzub6ldOjU8jAnVu3T4+OS4ViYLu7W2/TevJSb39xVjUGo0G7q6mKwHPDXr9cLFqGaZuOruu6rjuO02639/b2HMfJFfLdfq9QKra6nYllnpyc2bYbRbTfG8YRu8QzdTodjuMu433lcpkxFgTBn/yTf3J/f39pZXnl6kaz12GMVabnj89qRNZGlkt4OZvN6gnFcaw/+r2faTRqURTdu/3O2fHZbGXu5s2b29vbmqZpuj4cjjfX1kVMBoPB5uZmr9VOqolcLnd2dgYhDHy/1WqdnZ29ePFiMpkUi8XBcOgF4cLSYsRoq9O+DPsDAObmZlVZ8X2/3myYtiVJ0vHxcWlqSteTuzv787Pz27t7oqJ+8PCDvZ3dUqGwvrrWrDe+9Y1vejTaOzuuLMz4UfiDj39CFPno7DSioZpMHRwe27YNGZgYBkBQU1VVTnACL2kqkcXiVLnf78uCuDy/+OrJs/WllXwuZ/seL0uIg+lMslIs26PJbGVKFYXQdr754AFBeDAYPHjw4PjwaH52bmFpae/k6OrN66lU6u2b7Q++8Y3j2sVB9exP/OIv9vpDGkYbGxvnpxe6rN66dv37v/tv5yszCUV98eLF3bt3Qxq/2t+5cvfWyzevk8nkZDKpNuoIQGM41kRZxMR1Xc/3KcKCLFmeywju+zaVCOGF0A8IjyACNIYcLwEMvNALwxAhLgri0PVxEMkAAseXKEIszJW0L3Z2DUxMM9TlbODFAoSVlPK9925/9/p0Hkc0dDwachJPOeQAOPIDiDlJEBVBlJAYW6EmJYnI7daGv/ajpz96fWKppV6UOL4YhaGgqNl0poQ4SddTcRCOBr3QtXOZpOU6jU47RMANvUQyAUCU0CSRR8z3EhxGUZzTddsyXNdOZdJjx/F44WI8mfiR7XuxZ81lE1MiLQAnGzuKM3754+//g//Hf37+yR/u/dtf/zd/66/+4f/yNzsvvq9HzWtr+elKjtdVJZ2NAeQgalSbruWnMwXKJwKhOOSmf/UnZ//1L//b//X7z7b6oa8XWTLbD2IgKlYYT2wHYx5Czg2DkMMIQYgAwJiL45jnOFUWc5l0t9t+/5vvT1WKk0lvqpwfTcZXrt348//pXzg4PmIIX33/4YsnTyxj7IK4Z46XNzb+m7/+3476g1cvnjEM6p2GJIuPvviq0+lJovLq1RsaRY2L84Ot7ePjY1mWHzx4QAiZmEZ/OIzjGCHUaDQIIRhjxJBhOhPTnUzMwWDgui6HcD6TZyG1TScMY4SQZVkSISLCCFCCseM4o9EAxJQFked4iLLL07xhWwldY4yapulYNuRIAIAThPlCiVLquo7rOgLhHdMSBIExBjGSZTliABPBsUyeI5gQy/M4jnNdV9M027Z9P0ylUoBGGECCMc8R3/UgZAwCBpDjeFFEPc+LogBiADlMBAIQiGDsRSFE6NJoenx4tDi/cHp6enR0lM/nH339VRBRVVKPD49XFpd4jnz15PHSypLrumdnZxBhy3YSiUQpX6ienScUTddTx0ene3t73W7X9/12u10oT83OLfCcMFOaunbtCgD09u3bb99sTSaTa5tX+t3e/Ozswtz802ePr1+/fnx0kEmmZmZmDMO4e/fu518+4niSSCQ6nU673T48PAzDcGtra2VlpdfrHezuzc3N7ezsNJvNwXhkmpM4DtOppO97xyeHpVJpZmbmJz/5yczMTORHW6+2Hrz74PmTZ6EbLs4vffHZI5EXPMftttvZbNYwLIR434tN28WIpLM5SZIopYZh9Xq91dVVVVUlScEYXoJHSvnC6enp3bt3j4+PR6PByvqyYY7DMExp+qWjNYqoYRg85jgM9/f3f/u3f3tuceHJ82eGNVnbWO13eydHB8ViMYyZJEmyLLuuW66Ux8bohz/+QUpPOoa5v7tXyBW3tl5Xq+d3b985OzlFEK6trVXPL+I4VhTl6Yvn3/zwW17gHhwfLa+tthpNVZIr0+VmvVEqFE3T/PSLz2dnZ3VdPz09hxBHjE5MY3Z+jlLqeX7gecNev1wsWZalJRI3rl3/ycc/Ho9HC4tzp6enK2trhmEc7O9fWVt3bcs2J4qiNNsdAADGhBAiy/Lc/GIM4FRl2rSt0A9814MsVlW11WoRTrAMs16tIoxH4/GlQLnVas3OzjqWRSkVRbFer9uGCSGsNeqFUgkwZgxHG2vr1Wq13+0TThiZlhME4/GYEBJEYUTj3mgoymqvNxgOh/1+PwiCsTHp9nuDyXhkGnEYTZeLxmRi23ahUDg9PU7pekJRT44OFVEgAldrNWdnZzudTj6fD70wDENRlA3LxDyhNAYAOI4VAypIYggiSZEd140YZRBggsI4wjy59EGrsuR7YRTEIhFFjnAY0yhWiIABo2GY0HXDB62xLSczbuArCTUOfZkgMfLeWVv8+Q8frlayEvOVOICeDSHERIiiCLCYYI6FoSaovu1DwMcIGxH4vc9P/v7//vvPTrp1G745G+zWhu1x6IaI8LLv+wlF7bTanU5HluVUJksBixGwXTOMPEJQOqOrMo8RS8gcpH4ho6sSxyIfUoYxDmNAEbZMz7IcYzzUZVFCdKmQSsIgw8WiO/lX//gffvSb/wqMBkVZuL6yWC6mFVVKJBVBFCeWGYah78UQ4DBCXsQhpfjpy6O/84/+5b/60ZMBU7l0xYWiEdGY8IAIEYVBGCKEbMtHkBdE1bJdRKETgzCIOc/BMIJGd8QCJgCe+n5aV8fjAYa0Xb3Yev60XCxdu3bt0aPP93a3Rr322srq2tVbboD/h7/7P//ZP/Vn/p//1V+FHPgz/7e/+At/6c/Lmi4R4bOffBTZroL5SWfome69e/f+0l/6T9978HBQa8IwxoIo6bpPQy9wtZxm+nYYx+PxpNdpizynqUpG1/LpFI9gu1GXRUGSBIwhgBGPAXODjKJIPOfZk5QklvSkxkFd5JOyQqMY88AJbcBzvdHIdt04jjHhPM+hgU+D0DRNQZAAgAAAXsAQMN91COZ4gj3Pc3yPl8XRaDA3O81xAkRcSOMg8kEcCQIBGNieCwFmlIoIaCLBCERR5AVBSOM4ogyCCIEQAxdFMQFB5AIMXBoIqkRZDAGVCP+N9z8YGxNVS6xcufLV86drV1YEgewdnC4vXTnaPxlPhu/cv3twsjcaDRYXF4OQJVO5xcXFfr9XLJZL5alWZzAzu6CnklpST6bTYUyPTqt+DFzD6TRbr9+84gWytfV6amqqMjX1+uWrUa9/cnTc6rTXNzcfP/7qyvpGo1Y/2NuXeOGLzx7duXlnMBhZnqulk/1+X9f1lKbfuXW33mpOVyrXbl579urlzXt3QgBMc/Leu+/0m81es5ZOqhyPx+b4+YsXaxurjUbDsqxSsfjZx59lk1nXdOpnteW5xfF4LIri4uLiixev3n/vmxcXLUikqdm5sWvvHx9xkvBq641PI0VRWs2OKqlRFG1vb4uiKBLec92b167v7++urCzNzlUAB6fnKtlcejIay7JcrVYVRcnlco5lsjjKZFOrG6uUsUqlUruojrp9DgPIqGmMPcvkEcQcIyI5OjlWtcTC3DxH6fL8/NzcXKNVX1xegJB9/MlH9+7c4iHZfrH1R3/me/3+kDF47+67v/v935tbWrxx48bbt2+uXNlgLP7ys89vXNkcD/u8wN26defg+ESRxIWZyutXW6ur6xSg0cScWVjsdPs0YuvLK9XTs/mpSrtaf/zoi5/7oz87GPSOj4+m5iqW5ST1dD6T7TYaC3PTuq4Y1kRPaeXiFKLgeO+gdn5BGKyd137y6JEbRjLHkThemJvlCNI1rVgoYIiK+VzsRwjgOIiNkbG2vNbrDTq9gZrUa+1mvpiTOS6n67dv37yonSeTGsehvb0dUZENN4gxz8nK2LIBhw3H4CScK2Snp6ePjo4ZhdPTMyyMMqqmcsQxjcJMCQjcwuLMZDh47/4DQRDGrrF8Zc3znMmg/+D+u8NBHwKaSqiff/LpBw8/OD44zufzccywJNgsojwkhBAO60mVFxClEQsijZcUUYpZ7MZ+z5r4IHJDD0KgKwnqx5IoQghhRBNyImSUIsBYTKNAkUWMiUCkZy+2hLTGZxUfezEKIzeUORFEQUZCf+L92z93e3FZAgVGdQgSAo8hoDSOQk8QCWMxAIiFDFIMCFHSwiQEHz3b/z8/efG4Pj420auLQdcO7YAyhC+XQm4QGqYNAUaYKKqGBKxnk0EQhX5IJIJJzJFYVVHoDKgzKesJ5DnQ9VWOCIgEPvNCBMVk27ShINA4yMgkKwAhcubyGZkXW91JaxTV+zTm8jFKjMYWBBHBMUJcEAEvjnk13bbQH3y+9TsfPZ5EKD09hSUBsAiBWCJc6HjOxBWwhBGilLpO0BsYrUbHnthIEDnbtaIYEkHmMafJEoGIx2jU71xZW4MUFPLF8Qh8/vHHxWzm+GD/7t27f+u/+++ifn84HmFJijjy4x999g/+/v/07//cz//Ud7/zgx/8wS/9v/9fy4tLCKFMOod5whhMp7K5Qh4T8ru/+7u/9Zu/HgVhSk/22p3hcCjKkpbSL/MMcRwLhC9ksjld920nCkLLMnqD7tRUCWHAc4jRiNJYEASEge/7URxIiogApFGkSOJkNLQsiyLAIAMIWJcQc8+jEIRRpGkaz5FsOsUhcsmY9TyPYI4QfDnsAwgDFhNR6HQ6SVXNZ7J+EIwMM4h8WRYZiD3PQwwwxnw/hBBCjGzbVGVR5EgUBIgBjmBBEAjCl0t533f1hMYhoEii77uQURGTxcXFTz/9VNO0ytzs733/969cv9brdRzHuX3r7sHhaalUkgX+8HB/eWE+oSrHx8e3b98VRfnVq1d3794dTsaPHn317rv3nz17NplMSqXC1tZ2FLO777zz5s2WpMiFQiH0A0EQLMu6BMtlUqn5+dlsNq1qCseTmfk5Lwzm5ubWltcC319eWGy0G5IqcTw5PDmcmpoihMQhPTk5yWQy4/H4ola9du2aaVmGac7MzHz99deKwKe0hG0aGMPBaJjOZavV6sr6ysiYdLrdYr5gTibpVHa2MmuaZrlQZHH49u2bhw8f7uztyrImiMr+4UEml93c3PzJT35y7eatKIrOzs6mpqbOz8+N8aRYLL9+/Xo8HmuKenZyWigUDHPc7XYpi05Oj4IgSKa0J189yaRzlzr58vSUJEnD4UAUxYODo8FwPD+/SDA3PzObUJVhrz9VLiMIjfFkMhlLqhwEQRD4mpp4+2aLsXhxcXEwGHieNz09XavVeJ4Pw/A3fuM3lpeXKaX7+/vr6+snJyf7+7szMzNHp0eGYdy6fmP77VuOQ4jR+kW1XCh2Op1mu722tnZ0eJJMJqempl68eJFJp0PPr1arU1NT4/F4dWkxk0ztvt1aXpzN5DO26/T7fdu2A8+3bXvn7Xav02122rbrD0bDg6PDlZUVDBFC6Obt2yur68lMmuf5OA4nkwkRuItqtVjMC4JwcnIyMzPjuu5gMHj48OHLF6/zmYIgCAcHB9/61rfOTk4XF+Y5AH74wx9++J1vn56fmba1srb65OnzfLGwvL72yWdfXLtxfTgenZ6eLq+sHJ+dDofDb3/7w4ODA56It27cbF7U8unMwuzM48dfrayvDIfD7TdbkiRpun58enLl2jVN00RZSiaTtuuIopDNZiuVSrVaTyR0RUlsXNkcG5NEOhnSOKIhhyHHIYyxKBBVFBSB5xEkhIiKnEqlFEWJAp8Q7Pu+KIpRHCuKghByHOdy2UhEQdO0KKKIwUwqOxgaB7ULKhE38F3XJRAQCFRFigNbRMGN5dk//q2HKxlFjyziDHUe8ogJAoEQRHEgCzzHIYwIx4mWF6qZDJfM1gz/89fHj/caOy17r24dNo2+ExNZn9huKpnGDPq2G3j+aDQIIxpRIMiSaZuua5nmmPBAUwVdFxQJ9Xq1SjmX1qSMrqqimEnphmGMJmNIeCyIcRzHYaBwYL6Y4UIHBS7huNHYfPn28Pnro17fURMFQiQIiO14hBAM+F53XG+MDk5aUElLudLEcsIgRghhAKnnJwRBV2TXNsfDbrfXMl0jofDMt5MihwIvBBQCHMXAH456GMZhaCYU/uLoSCHioD1OJ3MLC9r21ktV4B7eu/frv/7rQUxBInV0Uc3PVULMfuVf/UvbD/7xP/vf//O//H//X/7Ofz+bSpWL2Vav84c/+mFrMGj2+gcXpztnxy92tqKIpvS0H0eGZZaz+VImBygbjUbGcCRhMl0oJdWEzLDR6vEQe4HPi4KYUC46zYlhCAyleDkKQtt1bc+nHDIC1wq9ereNed7xfEgIlAQz8KMwBhQ4lo0xzmQyCCEkECf0GaWeZSuSZJsWiKmuqHEYXsrLAxr1jJGiJSzLkgVeV9RUQhtOxpQxjsNR6JmmiRHiOUIQARyBojz2PI+x0XBCANJEURE4xkLbGkeuk5YTMGAcRYNOF8dMgFghAs+gqijmxMgV8o7vvXjx4r333qtWq+PxeGxM+sOBruutXpeIgoi4qWLpksOzvfXGGI8Wlpeavc7de++8/41v9AeD+/fvLy0sPnnypDBVWllf23m7/e69dyhig/FIFEVzYqSzmfag57ru1MyU7VoxCKbLU/1+f2JYjXaHEMH3/ampqbe7b27evuYF9vPnTx+88+7O3n4Y08vC/cHBQb1ej6LIdd1atVqZnj44OpxbmB9OxsZ4cvXKlcDzy6VpUZau3bwRxfHG1StrVzdlVbpy7Zrl2O1B9869OyfnJwSjD957uL37NlvIp5OpZrV6/er1VqO59ebF937mp/e2dzAilUqlUbtYXJyHEGJMvv2t7zi2FYb+1NTU/v5hsVAmhLx49vzmzZsTwzg5Pb13757vuhdn54IgPH/10nIcWVbazebC8pLneYdHR7wo7O3tTSZGoVB68+ZNEASz0xWe40TCB0HQbrcvn4+JaRi2ldQTuqZ6nje/tHhePWOYbVy70my3oii6srFROzvP6MnFxcUnT54srayOLKPRac8vLIzHYwThzavXDvb2v/nhtwrT5b29vdu3b18+uNc2NhvVGkNwdW3ton6RL6QH4yFl8dVrV9rt9mgyBoCGYRgHMcScIEgYE56X87myaTt2GOrZtGGZWlKPEbioVZOa3u90t4/2s+XiaDKmlJZmpp++ful4rq7rw8lwembK8bz9w8MHD9/d291eml+4ceXq2eHx+tJKt9WZWJakJn73+39AJBkL4sS0r1y54vtuo127cn398bPHMzMzpdLUj3/4UT6TRYC2282FxTnXsXw3mC5XWvUGDcK58nTzrCoLckJP/eTTn3AELcwvvXn5GvOcHwWvdt5euXFdS+rdfk9UZFlLIJ5Ynvtbv/NbjLHADTAihBCOx57tTYaThCgLPG+Mxp7t+K4nIZ45vkBhOpnyPE9KKIbv8IrkhkGEIS9LiEHbtFzPs32PIwQAwCMIIdu7qEIhIXB6VslqihKFnm2bgOcmvmP7Dgbhz3/7/s/enl/VGbICGPsYYwCRIKKI2pQFUejCIBY44oPQZX6uWMC88uKw/qu//+yjN42jMa4a8KRj2nbsGl5G0mPL6VarPASSINu260URFnnKAk0VzVHfNIaSzGXyCUXDtdaxZY8wjgkIqG/OzxS0hCCIeDQZMo5wvKBKKnPcJGEr09mMCLIJMaPr1YvG8xf7b7bOwkhNaEVdT3q+w4I4q6aeffpVv9VxGPAYEEkCxng4Nr0oYn7o9AfjXqvbqoahx/FM1dHxyemDW6t/8me/iTgsXFbRMMcoCxBigsClknLn4mRldm597Vq9NYxoTHj04uuvvNGwVT0vF4qrN25VG92EqiLIiMBNVcr/2X/xl90gvHHzmmmaxmQyW6msbWzMzs1rufTqjaupYl7WE4ZljiZjWZZ5ngdR3G20bMMsZHNLs/OYgcl4fFlvkxOqmFAkVTFsCyEkSRKRRYgR5DBjjAKWSCUFRWUQxgwKqjIwxhPTiKJoZEwgQhTDIAqnpqaG/cFoNDJN0/G9OI4v5Zkj0wA8HzEQxlQUZNM0Lzm9sqr6kS9gBP0orWqKJE8mk4iGgLEoihCAqqpyHBdF0SVUnXEIES6VzVi2QSDwHTsMvIQi8xAHjssBmNKTmpoAAABKYUyjIEzIChH4Tq8rStLde/e2traWlpYKpaKkKowxwxjnMtk4CGVJ2N/ZtW07n8+nkklRFDudzsQ0vv/DHzx+/tR0rOOTo93d3Xt37/ZH/VanvbK49OzxE1XXGIdOjo7v3bk7MkzC87fu3Pn93//9YimfTqc+/vST97/xQafXZwxCzL169SqhaZVKxTSNdDr93e9+9/XW9k9992eMiQUxeu+991Kp1NLSUhxGkLJL72i5XO73+ysrKytrq9vb28vLy/V6HUMu8KPT47OTk5OjoyMiCidnx0QkqqZ8+fjLXCGrp7Tz6kVlbrZWq+3ubi8sLHz16ItyMT8zNb29vV0qleZmZmzTWltb6/f7mqYtLs7/8Id/+DM//bOapp2dnd27d6/VasVxfP3K1a8efckQ1NOpr776ClA2Pzvb7XYv+cCTyWR1Zf31i5daKnnl+rXDk+PVjXWe54+Ojr7zne+AmDbrjeXFpW6749rOg/v39/f3i+VSvlT84tEjUZbCOAqCwHEcXdc1TbvsKAwGA8uyLMtqNBqXx9XDw0OEULaQrzbquVyu3W4/f/pMTySePH9mGMbU1NTHH3/86tWrubk5grlyuayoqmFbhOCIUs/zLNfqdrs8z49Gg1wuN5lMilPlyxbulSvXHNvb3z9YXlopFAoMgNu3bx8eHQyNyb2H77x+/VpRlI3Nzd5wsL6+7vv+9u7O2sZ6t99rtJrvP3j49VdfzS3MJ5PJjz766Ls/89OHh4evX20tLS19+umnqVTq+ORkMBz+3B//ecuxJUmamZl7+fKlqMiHh4eff/7ZH/tjf+z58+dxHH/zmx8+f/pM1xNJTW21GktLCyAGv/3bv/2d7/yRfD5/dLi/vrF6cnyGMX748OEXX3zR7XTu3Xv33/7bfxuG4ezs3I9//FE2nZFFsV6vf+OD975++sR07MXFxUuDq0DIZeo8jJkoyrZtt9ttXhIFRRVFKfB8GLDAcjzPw4SzbRtjHMZRQAMvCrww4BAGFDKIRDUR0hhCQCDUdf241p0EDEAu9EIQM9d2ZEHkEAxDn/BQTYjmuLM5V/zWzZV3N5IlCUDDxKEl8zj0A0I4VZUBjBBkAiEQgDCgMeAELSul088P67/xo69+59Hr7aZ1PmYHtdHBRTtmnK6nZFEKfDcMwzD2TWeCEKIsTiaTjMW9XsuwB4m0mM4qk0nHt0cERatLFU1BBAWazCdUmSOIAtZqtYLQcz2rWT/XEzKkIYv8pKqyKK5etD7/7MnZeQtwhDKIEOq2e62L6slhrdfvm66XUBI8JhzHm6bdbLRs2+YQpJFPOMQBEDrGL/6pb4+6jXbtDFm2q2lJ27TiMOJ5pOiCaU1knncnPY4FSqpwcNZZ3rj6+NlhuZC5dWWteXy0MD0zu7S2vXNSUrN5QVUFbrpSOr44Paqdvj0/HriW7/uEwWIqrUuyoqrH5xed/mBkWslsBnGcOZk4hkljlk1llmYXQsc3DANhHASBH4UuiMah46K4Nxlm9ZSECQLQC4OWMxlRX02nYgDGplFvNUEMCCGGbU1cFxLecX0UQ9f2YgxDDrbbbUWWEYDZVNoYjaMoIoQQSeQScs8xkCyImjqcjMvlsipKIKJBEExGI50XkBeUs1kBYokX4jiGABFOpJRODMMKHSwiRRVkiQgYhIEX0TCGgFLKIUQw79meLKs0pAwgCqBpO54XMMYkXmCUIoQYhJbj9EfDiWUahpHQtFqr+XZ7+9bdO+PxmAbhfGXm+YsXhXIpiMI3b94kE1rg+/XqxfLy8tTcjOk62WwWY5hJJ23bZgBMz8xEYfjwvQee512izxuNhmXbC8srW1tbt27d6vf7hmGsXdn88vHX87NzkiDWm82bd+9MLJNSur+7rye0Z69fKkntV37lV2RZRgj9wR/8wex0pVWrz8/Mzs/Pe77f7/cdy+U54Wj/aGtre25x6fOvvy6Xy+Z48vbNG5HnI8dzDYsQgjhsuxYFcXmqqKdTb/Z38zPl8XicTqfzxcLW9vZ3v/tdURQPD48rUzPGeNTpNObn5/d3D6anZwzD2Np6vbq28pNPPq02mtl87vBwP6FIvuP2Ot0PPviA53nTNO/euCUgDlCW0vSzs7PK7Fxlavpgf39lZc0ynUajpeja1u52xOj87NzB9q5j2Z7rfvnFI5Hnk7L69vWbXCEPCVdrNa/fuXV8fuaHIQNgMhxZhplNZ2gUI4QyuZzh2KKqTM/NMgbHYyOrZ0In6HQ6y6srQRBks9nZ+TlMCAbQNsyxMZment5cXycYd3rdy55BtVplEDUaDQDAwvyS6/v1VjOlp1uN9tzC/MvXr3qd7jt375ydnYmylMvlmo2G73oQgM8//zxfLNabjZ3d3bmFedOx+72eLIgvn7/QdX1mZmZnfy+Vydie+/zps6liqd1sFUrFbD4/moxFRZZUxfHcQqk8NCZXrl1VtIQ1McyJcXZ2RghJZTMHBwc3b95cWFo+Pj25efvW/v5+t9N6cP/+/t6eKkvddudSpV0ul3f29g5PT0RFnhhGJp/rDwcRjSuVCo+50A8WZucH3Z7vuKHjffHJp7HrAz/6vd/67ZlyKZfOIIZEjgiYM8YTQRLHlo0IRARRhGUtEQDq0cjyXEw4meNFIkKCTd/lOYwojaIAEY6IQsyo7/scx3G8aDk2gyAIPcIhVdXMAJw0ukRViSyarpdMpiLf52gkYy72/QgGWOIjyrKp5HfvXf3mWvmD5USOxpFhSLzke1HMqKjxDAQqx2fllMBJsqpBzFGEgSSOIvT5jvnPf/T2sxPjItInWK8NnCBGhmEFnqNImPAAYUYB84LYjQJO4LKFLGXxoNdM6sLqUkUk1DJ6/X6VQ1YxK9njfjGTymiSwMPMVF4vFQOOcxmyvJCGVOZQMaNpMmax57r+9s7R06evJ7brUVoo5fWE7PmAA3AyGA56XQhA6AdJPa1lMkPLAghvrK+DIFqbnstLOnSi9dWN3b1DlM7m+oPRaDBSJZkTCC8KgkCi0M3qSq/bSuULVJAjxGMO/OgH399+9VQmmEaxnis41QtVFDTCJzVtZEwYBLwkxohiHkdxPByPB73+xfnpeDxWFCWdSsmiNLFMIvCKJCc1nYiCG/ij4TAMAkQ4hmBMKYWAyKITBa7r5tIZ13UBACLhRVEUFNmnket7GGPLshADLKaj0ZjjRVGWKYKYJ7Is67reHfSjKNJ1HVIWen4cx6IoXk4AYRiGoU8IcTzP8VyI8GQyueSeU0oJIYokcxjm0hnDGA/6XQRhzEAYU16QCCGUUgAhxtD37CgMtIRqmgYALIiimLE4jnme53k+AkyWZT8MJUURJEkUxcu9YbFYrFarmVw2iiLLsq5fv+77/s2bNzc2Nr7++svl5UVR5Hd2du6/+64sy4PBoDI3yxgTCX/v3XdPz88BghtXr3S6LcuyCCHtdltVFGM8bLWaZ2dngiD4vt/t9wRRohS8evXqxq2bOzs7hUIhkdCarfry8vLL1y8Ma/Lu/Xu/+Zu/WSqVjo9PS6Wp8/Mqo/D69evvv/8wCD3G2J07d3Z3dyGEgiDs7uzwhCQSCUmSpqenb9y+lU6nL9k1nU7n4PhoZW1tNBplMpm5ubnXW28GxhhxuNqoy4pimhNRFl+/fj2ZTHzHVVV1Znp6b2+nen6Rzxc1TVtdW2GMXea+z8/PGWM8z/eHA47jopBeckjG47EiyfPz81svXtEwSsjK6empruvPnz/vdrubG1d91z09Op4qls7OzvL5fDKhOaaztLjs+/5lWpEXBTWRWFpZdhxnYhpzc/OUUt91N9fXWUyLxSIR+Gq1mkjq/eFgZ2cnlUlblnW5vekNB6ZpiqKIGMhms5cTzOUvTL5QiAA7OjlO6cmEog77g2KxaFnWxcWFnk693d0pl8s3b95st7qF4nQqndvfP5ibm69W69lsVlLkP/jBD65du1Is5v/wD/9waWkhpiGE4ObNG9bE4CC6unllPB5vrm+oirK1tXXnzp1cLre7uzs3N2ea9pMnT773M99zXVcWxEIu//rlq/n5+b29vXq9ns3n292OqiUy+VzEKC8Kluu4rsvzvCRJ9995t9NpKYryJ37+j4/HRrPZzmbytm2vr68mk8l6vaqrie3tXcbY5uZVAGgURYVcZjgYOI63trHZG3Q5nviu1262LhG0luvkisXlxZXxcPRHfuq73W5vZWWlUCg0m83FxfnzsyqjMJ3Ufc9hDAq8BCGLooBSChEXRtFlsC1mAPDEdJ0gjmgcC4TnOY6FcWC7kMaiKFLAAEQI4SiiRCREEBgECHGKmnz2cifGGPAcgDiKWRQFoe8pAg9BFMch5KAfB6lUivru+lzlp9+791Pvric5lpGxwkUCARhGEIKUngQU0hhAiCFkHI8Rz/sQ6+VsywG//cX+r/zhV5/utfoo2bTg2MMUC6brXP7LoZTKshwEkef6ju2KvJBO6sZ4YJgDTZfzhRQDgeOMR/1WKZUw+53QNTlEQ99y7DFjVJZlzwsFXgYxcCfj+aniwnReJRhRFoegXuscHJ71R+Pv/dzPTmWA0RgKvmcN+/1u67JxU6pMl2cqru+cnZ+XSzPdzgBE8LOffPF2e392cQVNTAtBMldZsAzX80LH9lJ6UgAABv5kOMjMFOuTkZLMZzP6zvbrn/r2B9l06rOvHq3fvS7dWPuXv/arST0RRXHgR0EYdTrtjKqA0B+a4+54yPGEw7wkiglJDmw3cD1FTfCSRDCBDEY05iQhiiKMsRP6dugLqkxk0QsDRZQyospFDEIIOBxFEQcRF7PQdh3LTsiKggUBcqEXgJiyOHYt1/HciWtbnjuZTNKJFA/JeDxGhMtls81mk1dlKJB+pyNhElhOWk4IEGMAAQC260wmEz2VxIQTZbk/7AHEMsV8e9ADAHAc50Wx6XoUYYQ46oee7TDGREnSFNU1jbSqabKCBQJ4nhekIKaG7wOOuFEgKbLrezGjlulgIgiieOlZRgB2W+0r6xsfffQRBOBk/7jb6ZQLxWanPTAnxUpp69XrXrN94+ati1rVc9zlpYVPv/zCp5E5MT7/+JMPHr4nifxwOHz48OHx3sFoMNy8emU4GYa+SzDu9vqWYy8tLIIY7O3s5nPFKKRBEPQ6XVWV7927s7iy+OjR5++++65lORDixYXVTLpQzOaM0fji4kKSJIHw1fPz4lRZ0RJvXr8u5QuN82ounclks4+fP3v95k3E6Oryii6r+Xx+fmGh0WqJqsrL0nmjtr65WSgUKILrGxunF6ccQglJ/vYH3yxkcwlFzekpXVRoxAqFQiaT2tl5e3pyhjH50Y9+tLSyCCEDCPKS7AWhoimCxLd7XT8KAUYMgvOzM13T7IkRR1E6l621m7OLC6lshgMQhHEhV0SIE2Wl2WyNe6NrK+vt8xpm2PaDAEE7igJIt/Z31UwqhGx3f3++MtupNk52DyTE1c7Oh/3B8sriaDLUUklJUwfjgaIp2WwWAZhP5QQiSryEEFdr1FPZjOd529vb+VLxol6bOMb80rxljD3LnC4Xq7XzbrcNMLAd59r162EQmGOzVJjq9YZ+yJKpXK/dW11Zb/cH3f5gY23Nc9yxMZpZmHn79g3PwTByX75+wQncZDI5PjpKqonpXEGXlPXF5TfPX8ZBKAjCxDQ4nl9aWHz98lXoeiLHu6Zx9cpGf9CdWBPTttLpNC9LgMO1Wu2StH56ep5KZSDEe7sHtVqNQ6B2fjboDfudvuv62Wz+1atXnudIPHnx9JkkSUtLKwBhVVWfvXwRBN58ZVbi/h0vemvrtaYquXTu+OD42s1rWkZ/u7uXzRVarZaqJRgEoqadt+qF6fLi6tJ5rbq0ugIwsl3H8zwMMYaQAETDyLYsBCFPiCKLBPMBjQ0WooQEYpqUVNezg8BTRUETReaHNIo5UQoB9WxHkeTL1Q0kCCBUyJYcOzBsD/y7OiGMOYhFzvOchCTHfiBJgiALfhwIqhZBLKr62ur8f/i995cyKAEYNZ3Q8wqFvKAobhSoqiryHKOBbU1iFkuKPBgaip7BmnY6jn7148N/+Ltff3JkHE9wx4oBUSZOYHthHMfj8RAzzGNF5FQQIxoxRVGUhGx6lu3bmqZxmKeBj2GUkKCuKQmFT8qgklPmi6mCrolIYhEhWNRV3TPHhIaL06XV2ZnADKxx4NrgzfZBjKK/+pf/3Lszqn8xQK6JQDyZjGqNaq/XkWQ+l09qKW378GjihTYFqalS33aebL1FoqQgwgURJbwoCrIkSdT1RYRViW80zxevrBihvbOzbRnmyf7+5vLyxsJC4+gQ+0EllxcF3Om1GGMcRwTCpzTdtSccopfRF0lRcrmcJIi+5STVxNLCouM4nudxPIEYRYxGccwRktA0XhAkRY4Bc10XIwQpYzENfT8E1KeR5To0ikFMJY4XBGFimRDCKI6xyEtKQhJkRZaJwMuqEkRhpVIJbDd0PIxxEASWbSczaUopAECWZcuyRMLTMEIMRF5ECJEkSRTFOAogQ77rQhCnUjokXAxYEIVBEFx+tAIAQhorWuJSGRMFUeh6MpGGvcFoYiBeiBmNaagoEmUMEy6OY8eyGWMYoksLqCAIhULBtu1EInFlc/Pk+DiTTPU7XVEQXMsGAERRGIQe4jiEEBH4s3q10Wm3Wq2trS2R8BzHDQajtdWNg4ODKIpSqdTp6Wk+m0snk91+X5blfCpzfHioaRoAqFVvFfMFhBDGuN1umxNLVxMsinf2d1+9elmpVGzLUkTJmFhnZ+eiKH716OvA8yfGKI5DAIAkSRjAi4uLhYWF0Wi0ubmpKMrbt2+/853vYIzDMMQYv3z9ajKZKLK8srLSbDYnk8l777336PPPE4lEFEUvXr9aWFhqNBqu7Q77/csjrTmxjhWyZR0AAQAASURBVI+Py+VyHMfDYX9xaWE8Nniev3HjxqUJq1wuN1utS0F5IpGYnZ09Oj7GHI95YlpWfzDIFvKUsW6/AzgcsLjd6UAOQwiJwA8mY8yT6ZmKbdtHBwepVEbVErwoMAh5QUjoqYfvf9BpdyVBvnbt2sXFxfz8fBSGzWbzmx98Y2lhoVar3bp1i/A8gLBULp+dn9u2PTc3Z04mpULBMIx2u/3+N78xGA57vd6dO3devHihaglZluv1+mVv45IpNpyMeUHQ06ndg/1cLifywt7ewc0bt2u1OqXg1p17x8fHHMfduHG92+2qquo4TrPVKBTzzU6TUnr16ubBwUE6nZ6fn283WycnJ+Px+PjgcLpcNk1zPB4Pev3A8ziOC4LA8zzXdQuFwtHRESEkmUwihI6Pjxlj3W73MmrSarUWFhfjOG51O4IkBlGo63pCkuvV6srSKkGk2+0mk0lCSDaXvn3nJkKIECIrifOLmqol4jg2JxOeI57j12q1O3fuTCYTY2xubl6dTCYT01jbuDKaGIZhuK5LIer2etlc7tHjrx3H03X96OQQYlQulzmO63d7mpqgERN4PpFQfN8NPJ/FFAAQRLHPYkg4nhchYyLhVVVlcUwwJphDDPhRyBGB4zjP8wkRDNvywyCIQowxQPjNwWGIOZ8GAMYcx0GIdV2P45gQwnOcLIgxYDEFgqyEYSjzfEET/r1vPfjeNzavzuq8D3AQy6Kg63oUBwghTU9xPM8gwkTkRdn1g4CCiJMcjE5G4W98vP/P/+Drtw3/oBMZVLNi3nQCzwsEWbWd0DAdP6CEVxjlLMuRZQlx0PFsURSLhXLoOzT2fGfM44jngsAeEOoJKE6pkibznue4gQsx8sNwbEwwxhsbG+V8KfBChZd3t153qsd/+c/9R//ln/lOkoTDRjeeDHQeB47T6XQigADHLa2tu5ROLOei0YVEFpQkClnI8aTe6VHEiRIPo1BGEvRjTeEtZ3D73o1COes7o7RCUERff/2cmc7JV08//T9+DbUH1nDoxb7hWo7n8oSbDAeI0TgMRMQVMjk/cG3PwhTkkhmRF62JqWuaoiiW59q+xwsCQigE1A48Sqlj2QwAQRBkzMtE8Dyf8QRymCGopPSYURBTSRR5WXHCCElShPHQcQJAA8+PghBDLgpiTLjj42NFEAXMCTzvel5IYwoYz5HQ8zlBEGWJQhDSEADE8zxmAIRU4UUYURjGmUSC+l6hmKcY9gyDEczLkm2bHASMRgBQOwwjhHhOghGCERf5QNC0RCbj+z7G0DYNGgUMxGHoKzwvclhXVJ7jGI0AjS6rmJ7jpDV9+9WblflFSZLm5uampqaWFhZT6SSGUEScY5hIEWOBHztWvliYnqlc27xSSGXev/futz78sNpqeJ7HY+7ps2eFQkHk+aODw1u3bxjG+Pjw6MG79y8uLgRBKBaLXz/68tq1a77vh0GQz+ffvn0bBIGmaZlM5vJ6yRibm5uTJOlSpNBqNO7eul0qlbr9HhEEPaEtzsw1Go2V9bXt3Z2j05ON9fVXz19ghDKZzOudt6quEY6jYTTpDT587wOFSI8+/vTO9Zu10/Nerzc7PXu4d7i5cTVXyD96/DiVywcsdgM/Vyi83dkmAs/z/NHJ8YP3HnYH/ddbbypzs34UtrqdxcXF84uLbr+XSOqPvvxyc3NTFMVev58qF4AinDer+amSnkln8zlJlikEnUG/NDezcGV9aml+Z3+v2+1eu3YtoPHEdxLpZOC4vmkTjqtfVM3R2LNsantGbzAyJ3unR5DDw8n46Ohof3cvncr++Ec/QYQTZOns7KxQKMRxfHZ2xvP8q1evms3m3NzcJx9/JosKYqjd7MzNL56fVT3HLWRzj589xYQL/WB3e+fmrVv90XA4HM9MzTx98rzRamu63u12Iz+gUfz85YtCqWhPjMloVJmdffn2lSzLCUU1LJPj+ZhS3/PmZ2d917MNM45jTpGUlO6FgSiKc3NzpUJRlxQRcolEYnqmcnR2misV6p0WROj6jRuj0UgWpXKxuLezM1UqdertbqOTkJUXz58XSyVN103P+d6//++dn596jv2zP/0zf/j73y8VSlc3rzUarbW1NWM8efXi5U/91E+93dmuNuob168eHh9NVWZW19dfvXr1zv13Z2dnH331paIkeJ7f3d7R9ZRpmjt7u4IkSYp8mb/q9XojY3L16tWL0zPHMIvFvO+7GBPCCZ7nQsgQ4hiFpmOLssQYi2MWRVTXdRDFURiOJhMKAGTANa0wjr0gAADFMYMQuoFPAeJ5HlEo8RKlNGYRQ7GS0t+eN3tOICg8jV0BcQlRMw0XcbwoyoBCEAMQxhhjUeQJhxgNIIuL2cxCsfTg6vVrpXyeE5Icj6JIk2UviEwvEvWCIOqTsQMZ4xBEMeMgSulJxEs+QTtt8D//+vavf3Ky3YaTOAWEJOYVx2eimogQ8mPW6dthSHKZMgAQ4Rjh2LDMsW0JiqwlVYxcBMykzssSJBwVuLiQwpoS5XIi5sHEY0hJB1i0Y9ofDlIp7f27t1Iimc/lvGF/b/f5zLT2S//FL/71v/TdCmbm+TCYWFGIz2vdoeELopgvFgqlSi4/ZRpevzNGII5DGqazGT8KHMfxXQ8AKkqCqsqHe9vOoP1H7t28sTi1VCk8erRlWM6HH7xfPz7qnJ4QLwhMy7UdiJEbeFHgJRSJAUoIkSSZEAIAwBCUCgUIgB8Eju+Z44lt24jDSkKdTCaU0pDGnudFYZhJp8MgoGFkT4zI8wMWU8ZoFEd+MB6NfN8XBEEQhNFohDhMAZMUWZIkz3FEwieVROT6LKYspoqiEMzJogQAUFUV8pwgS8ZwJAsiQghgFMYREXiEEASAYD6Xy11ODARhGFFVEvPZzHg85nhCKY2DUCQ8o5QgrKoqBxEAIKYgiGI/inlJ5ohgWY5j2QRzuq5bjkNppGkaYIyFEYwoAvDya5fjOCIKi8vLe3t7K0vLrus+f/48iKNkQtve2lJV1XXdVqvx4MH9mIL+ZLS6vjkYTcbjsTkxjg4OT49Pdnb2OMyXSwWe5/L5fErTh4Px5ubm8ydPJ5OJpmnj4ejBgweFQqnZbD548KDVaiEMUqmUxJPLV3IyHK+srG5tbWdzBVmW281m4LmlQiGTSX3rW99oNBpHR0fZbPbw8NBxnOFwKEnS6xcvGWOU0oODg0wmE0ZRvV6/VOoIgpBQVAjh06dPfcddWVw6Pj5dWl2rTM1Uq9X52bnjw5N6rXnv3r3t7e2rV657njccDpeWlmrNRrvX/8YH3/z66y8zmcwHH3ywtbU1MzufSqcvLi5u3Lx2eno6Ho/v3bvXbrdHo9Hy8vJ59aIyOzM9PX3pPxr2B5eXgNFooKeSgiz5YbCyuGQZ5mUSptlpt1otxpjvujSMioVCr93JptIsptl0JpVKKYmEqKjpdDqbzS+vroxGo/fee28yHEHKbt++fX5+fufOnZWVlXqz+c477+RyuUsf1uW7Pz09vbW1tba6mtaT7VZL0/XBeGTb9trG+ueffy7L8sri0osXL65du5bL5UbjQTqTZDROpfSNjY1Go7G2tqKq6tHxwYMH74VhOBqOH9x/mEymwjD0/NCwLIiR6diEFzDhBv1RPp8/OzurNur94SiKmRdGrXanPxxWZmfOa1VFVZPp1KutN4uLixBCy7JEWXY8L51O5/N5SRBN03z96lUYhsl0+vD4OI5j3/Nev3w1XZ6qVi/a7fbCwsLbtzuapl0iz27cuNHtdyIa86LcbLd29/fS2cyLF88EWUplc0pSS6ZTFLB+p1vIFX3fhxBmMhkWg6P9I13TlhcWm/Xqf/If/2Kr3lhbXWWMUQBt24GIGYbB8zwFUJAlwEEi8BBCSJlrWJqs8BCn9ITjOHEcMwglRf53f6cQCpIoSCLicBjEcRxHUcRLIiJcFAXJdNpw/YvukEsoUCBRRBHEEGPGoCypjMIwDEVR5AQuorEgkGyugCWt3hlgXlqYX/rTv/AL779zTyJcUkmoqppOJwfjCWXQ8xhHJFlSCOZknucJF4ahIMmUl7Gmh5Lw5e7wH//61z95XdttRTUDnvVNiwLAC4AjaiI5GJnVRgtCjBHBmPA8jzEyXHPijgURC4Sx0PFcQ+BYOiE4ZjepIRFHmsxrsiQJsqooEMJsPjOeDEfjweLs9NJsRZd5gYbxqNc8eDuX0X75b//Xf/qnb4ieHfb7Eg2Q77XrLWNkO6GvZnQs8EpKR7Ik2KYRMi+I3bSexATHfOxAn5OEvK7/vb/yX8L62Y1S9i/8wn/gI/Djp1//g1/+793Iiglr9du6IBe11GQ01JMKCz0CARHEbKkUQhYBVsrndDXhOE4Qhm7gIw4zBOM4llQlYlQUBC2RIAiHYRgH4WQ4IghHQUgIzxHesu3hcJgWlTQnZRVNFqVWrxsxKmAuKasYImMwSomKSsQw8AxzLGAkEU6VFYxxzKhpWwRh13VjwGzHUYgQGnYURZgniYRiGabIC5eWANd1Q0QdGmKR96NQlZWMngwdTyUy80OCsGd7l4Zb13AUjmO+zyAlmgR10aReFIYZOZFKaKHrUcCUhOoFvmmMeQ6LPIlCHzGaSulBHDAIXM/7+ItPcuV8DFkqk17cWOt0u69evL5142a9XudEoTI/t3O479j2bHn6/Pz8MqcPIQQcimisJ7TQ9S5Nb67j1Ot1jPHjr59OTU0VsjnbswGCT75+fHBwcO3atd3dXVEUR6PRoNu5evXq/tFho92Zm1v6zf/zt27cvP3s2bNer1csZq3xAEQ+ROzzLx8JglQqTdXr9Q8//FBQ5RBQjHEmlX7n9t1SvnD9xo23+7uFcml6enpva3uhMosg9/GnnwiKDAnmCDIMozg9dXp2ZrvO1c1rF6cXPMfNzszs7x4sVmYbZxcoonpCq9VqqqLlsoVPPv9idWM9jIPPH321urZxeHj08uVLwuPj46PFhWXX9be2tlaWlyPHs0eTxfmFrddveq22JsoHb3dubKwzLzja3v3Zb/9U9fRsd+vtTGmK+WEhky2VSmdnZ9evXNV1vdXp5IvFwWBw2ebf3d1VtYQoK91WFzLmOM7xycnEMJ4/exkEQfXiQhQEYzLZevNG17THT7/u9juV2ZkfffRREEWXAPe7t2+fn58fHx8vzc2fHB4NesPAj05OTsIoohi+3d0p5ksohqfHJ3du3W60mqZjzkxPO5bFi2Q46p8c7OuaOhyPDk8OJV7q1JuUgmQ6c7B32Kp3XDcMInrRaDY67Zn5OSLwU6WyKAj7hwflmcpgOKw2G1fv3TZDrz8ZraytAYBu3rxJCOn3+3du3trf2RUEIV8sP37+Yv3qtf541Gy3RFFUROndu/dEQdg/2AUImhMLMHT96qZpjO/ff0cUxXqtee3qLdNwOY6/du3K06dfra2t2J6dTKWzxZKoJi6a9enZmZ2dnc5gKGv62JpQSOdnF04OT9ZWlzc21z755LP5+cWFmbmT3QMOQHM4fvzoK4kIw+EwnU5jjBFCgigbhhXHMUAoRsD0vJhSQgiMIxhTAXEcgMALE5LMeIxkwQ5DxAtO4Idx7Ps+gNALPUEVEeGJIELEAQAQZQSCRELfOTjAWmoYhjGEcRxfEpsDx8MQY47jJEFOyJCDSCBn7fZJz5Cnljbf/ebs5pXKxurM8iJBoqZomKD+sFfMpgGLY8B4WWEI+74PQSQQyPHQDVyAoGnbgOPlTHYE+d94dPxPf7zztMc1Q/GwNZg4gesHQRzKuiIocqs7jELCIiwQXpS4QjkjSMRxHM/xRUKo77TrF645SiZ46ptpjdcFVlSIxgIVBNB34jBKaEmAwdgeiQraWJ5byWVpb6QA/njv4MXTL3/uu/f/9n/283/uw9V84MadodebOKZr+matf0FlyhSIHGusaSqETNcTURRwHMcQ40VCw2BuaiqBodFuXhwd6pr67T/y7u/83u/ak4lMoOsY2XRKwMgejWSRtw3TNOwoilRV9YPACwPEYc8LAIW2bWOeXE5hl/vryWQShmHkB77jCoSnUXypGAcAuL43sS3Tc4IoVBTF8zzbts2J4TiOrCqO7wWeH9iu77iE44zJBFEmcCSl6YokQwgNy3R977Kc+e+u2HGEEBJEMZ/Ph57v2k7g+3pCGQx7hJBMUndtx7KsQqFgm1YchILIiyLPGHMchwjCJYoLMRD6AYYs8gNFEkSRpyAOWEghRQhFUcRiGnih5wWu7xWLRY7jPMdhjAmiyBC8ZHcUCoWJaSyvrgAOv3zzOl8unZ6fpbKZ2dnZly9ea8mk77tfPPpcFsTxePzkybP79++7rtusN7LZ7GgwjML4+vWbx6cnzVarUC65rjs3t1CcKl8qnjmO88NQTyXn5uaWFhb39/dT2Yyu62Hop1Kpg4ODQqFUKk7tHRzduffuixevllZXVFVt1OuaplIaWZZ169atXq9Xqzay2eyv/+ZvKIri+/7h3v7c3Nzx8fFoMDy9OP/Od77T7XYPDg8fPnzYbDb7/f7m5ma9XtN13XVtnud5QhYXFzOp9Hg8LhdLnu2fn5xfv3K1N+j7oVcsFkej0fLSytnFueP5ST1drzcP9o9SqdTbt9ulUqkyPX0pD2q3277vqqq6s7OTLxb0VLJ6Vn3w7v0oCAeD3vXr1z/+9FPTtm/fvv3ZZ5/xHKfK0uOvvlZVFULY7LTT6fTe3s7h4eHy8vLlD7dZq5sTY21lnWC+WW9IkiQKkp5Mr21eEQRJluXK9HQmkzFMU08m9WSSI6RUKvW63dFg+P7772uaxvF8qVR6u7OTTqfv33tnf3fv6vpGNp2GEG5sbARxJCjy6tpau92uTE0BSl+9evXOO+8QQs7Pz5MpjeOQqsqKIrmWCSGrlKdYTEulkiBI3r9DIfKSKKtqQlaVpdWVGAEvDE6Pjj3P27h21QuimZm5paWl09qF6Tqlcnlra8vxvN2dfUGSq9Xqzs5OIpFotTq269x78G532BcTSkLXU5l0p9Np1hs85jDGPM+n02lN0yzL6nRaz54988Og0W47jnN0dCISvtNqK6pse24qlWo0GrqWDKLQ9/2VxQVRFHmej+IYQmiapu/7uUx2a2vrzdaWIid2d/Y5jsMYD4dDSmmxlA+CQJZUy/FSqSTHYUEQLccWRREBGnph6Pm+52EARVEUCOeYJowoYuBSSxADFtI4ZlRL6rwoMMYgAJcJBdu2Hcdx/CBk4PIAVigUzuvG3mldTOVFNREzCiGAEEqSJAgCx5M4jjmEMYCOF5VnFzZv3ZtZ3QCCFGLODjxRFkqFTLmYwwjlczkBI8JoPpsMPAsDyHEEEj5k1I0CLwyIwBNBABBbnk95OZKSJ0P2f/zus1/7ydOJkD012CAWPUzGjkMR1rSUZfrGxHUcz3Gs0WiAENLTKVFSJoaZzmSyuVQQ2mHksNjTBE5gERe5IvBTMtFlbjxoxZEnijyD4KxWG4/HxVTqztWrhXRSE3BSZAcvvgDjxk/f2/z7v/R/+fbNiugYfrcnRZ4AAIaMYHbZ0GR+4HqehwhmjGGKYRiLGNLYpSBujQcta/Lk7VvIuGIiLdg+Ni0FI4mHse+JPCdxImKkVKrk8hXbdofD4dRUkcXxeGggRJSEhgkXRREiHGXM832R8KHjcRwHADBGY1WSEUIMQcf3AhpDSbSCIJ3MOJbbN8YOjJnAGa5t2JbjOJ5jWeZEFARKqSRJGGMah416VZIF13coB1L5NIIscJ3LIyqPOcuxfUidwJeJkBRkAWEMoZ5MhmFoG2ZCkhVOCG1XEyQQ00pliiFmWGaMAFEkwPM0pDACsigyxrCIgziEMARRIGAoCAQR5NMAY3Ip7SOEWJYpyiLmuYhGXujxIsEYS5LU7bYTCQVhDDG6fuPG27dvFxcX+/1+p9cVJFFWlVQq9e33PsAMFIpFKaE+/vJxpTjNc9zjr74u5wqFXP6rrx+vXbmiZ9OtTluS1aOTk2a7Va5MR1HkRz7i4Pbedr3ZqNZrvCQdHR0zEAuCYHsupXRn661pmlJCPTo7W1xefvPy1cx0ZXp6evtgD/Ok3WhqiooQ0tM64PD777/fbrfzmeyHH3746s3rIAplVcEAfvXoS4RQOp3ePTziRUkWxdFguL66PB50gsATRf5kf796ejYYDFqtliBJAEFFks9r1YE1jhCod9vJdGr/6PDGjZuyLBrWREskRVGFEGYyaWM8zuVyupayTfva1U0EaBgEqXS2PR6+PT4oTU89ffw4nU4LkvjV86ecrvTdydvjAyIKvV4PMgYgPWlWeU1OpvXzWnVlc51hVK9WM6lUo1aHEC4tLD9//CRy/Xt33jm7qKYLuZFptFotDkDHsCdjEyJuMJ5EAHICz0tiv9fT1QSMo9j3RoOe69mNTjtTzI/HxunxWT6b9Rx3bBh+GERBnEvnDNO+aDQzmcxoNBIEQdf1N6+2OMglNGXQ7dVrtdDzL+3PgDFVVlzfu2R7qbLC8/xgMFBVTZXUcX9QyhdEUazWa5W52fFkcnh6cuXGtZ232+fHJ0vzC41Gw3PdXC53cHQ4u7RwcnKiqQlVURqNxrVr105OTr74+qv5tRXTsUeWcXJyggCcmS63W41MMsUYG03GlyFRWZY3Nzcvg55aKuk77iVuLAxDCKHnea7rG2MTxpRj8PTomPohxpgQMhwO52dmC/nS6dnF4tLK3PKiGwU//yf/RKPT4kT+3v13/ShUtETIwmcvXlJKM9mU53me7wu8GMehwBEBcTiCmqREnktpFMexzAs8IYBgQZF9N6AhJYQ4vtcdDXhRIJhjMR0bE47j8noKU+RHkUepS6npepRGus6/2j2TtSJlMKYBpRGE7HKdeIlLikxn0p1UZhfypZliKc9A7NMgjMPQ8zKaXM4ruQQ/VyonJa2QyRbTekbG5azq26am6yFERhBFHM8nEjFDHMcxEEEUAeYhGANEfI5724n+7r/69NefnL/s0aOB7yOh1exZE8M2XE3PUAaGhgkhZzl+xDAQRCqIHkTpYlbSRYSj8ajXqF8oMi/JGJMIID+d5KfSKh87NAoDnyKiAS5RbQ/Gxmh+Jvfw5pISmErg8IA9+/KLxvHeX/xTP/vLv/SLv/jebFC1gvOx0xhEExcBAv3IB2FMMA8QQjwfhDFPRIlwMk+IwEHCh7z6+bOtVq8XBQGMg6ymsTCIgzCMfJ7nHceSZVmRVcdx4yCuTE+Ph32ImCypPJG73X6r2xFFEVI2HA7DIOAQFgkPKCOYG49Grut6YWA6tu25AYtt16WUjoejRCLBy5LpOm7gi5IkiqKiKLwkKgkVAJBOp4MwDGksimI2m+2NhoIsua5rGAbgsKprgiBQSjFEhVwe88QJfd/zQEwxA6ZperEfQ3AZ2pVFkYdYESRNUUVR9AI/DEPfC+IghjT2fZfwOAp8jCFjDGLgui5PMAGIBmEQBBAhTuBiwILAQwhCyPr9bhCGECMGgWEYGGNzNM6k01oiUavV5ubmPvnkk3KpxDHoO+5UuRIxajk2x5PPPvk0l8pc2vLSyVS71TInxi/8qT91enLy6uXL1Ssbbhj1+v2Zubl6vb6wtJzJ5r9+8jiVzUiS1Gi33v/gg0tgwI2bNxHGw+EwkUgMBiNBVpLJNEKoUqkcHh8Oh8Mbt28dHB1Ksry6st7u9C6/kRFBlmNGUXRyfpbJZKampt5sv52fn3c8t16vVyoV13U5jpuenr6MeURRlMvlGo1aaaooiPzZ6cnde3cyqXQYhpIknZycOI5DBD6TSWVyWTfyLNfiBCLKkmVZg/HIcZzhcJxIJC41QLZt24Y56A0JEXq9rior2Ww2CALHcxU5EQdhpVJxfY/j+Zu3b2WSKUhZNpuOaEQpbTdbiiSXCoWLi4s4jlOpVKfTUVWVMXZwcJDP5xFCp+dn3/zwQ5GXnj59ms/nLdPhOYFFbDgY57M513UvLi6uXL3a6/UQh4fDYT6fTyQSlFKMsa7rl+zSi4uLcrFUzOUBZTGjl+O/rCr1eh1CODU1NZqMR5NJoVQaj8elUqnb7vQ7fUJIPpNlMZ2dntU0TSR8r9djjDmem0klMcZxHJbKhcB3B4NeIZf/+svHtWqDMXZxejE1NSUI4uvXr8vFYmVq5ujopJgv1i7qrUY7jumlD11WlUqlwvPi7tvtfD6fyedevXldbzVTqdTU1BSGKA4jGkbdQR9g8PrtFhF4WVWr9QZA2HLcmFKE0HDUhwwszM8f7h1OFUuu49umdePalVqtlk6n7z94rz8ceJ6Xz+drtRohJI7jdrut6loYxPV6vVqtzs0u5PPF0HMrlUq1Wp2aqczMz9TrdVWSwzAM4giLfL8/lCQJMiBL0mVSm0IgyAIDwPN903UAgjxHFEESEIcY0BQ1iqIgCBiL0+k0hNCwLFVVCeY4iC7DbBSRVL58Vu/2xq7l+QAAGoQ0jCzXDOMwjmPXdk5Pz+bn5jjAItcM7BELRpIEJRFmMirP0dnp3LXNpZVKcWkqP5vLFNOpQjZbzhUK5ZLnBRwR1ERSEERKGWMMQQ5jDCEUJQ5hRkEsammb8h0XPDkd/OoPH//e492tuhlK+ZGLfSg02z3LCQgn2ZZvG7bjOH7oS7JKKY0B43gUxWEioagJudmsT4y+npQRDcb9toijhIAhCHgBEoQFQSAScSP37HR/1OvcvLI2M5WnvrVQKVF7vP/8kdU6+PN//Dv/6Jf+wt2ZjGjSsDlCmOMApTk1xYUACzLjRZzQPAocyxYwDnx3YrtAzIdCxowjB7lYxUjCgiAQzAtEVBRF0uRkXkcIQQpnijPO2JUEUVXVMAxbrZaqqsVcHjJw6axRJTnyfNMwrInhWraua37gUUAlVWYcxBgrRPAmDs/zo8mYYD6p6L7jEohAGIOYMoLt0Lcdp9fvx4C5YTC2zIDGlu9GcVzIZGkYTVzbjPwYgjiK3InpWXYcM16QREHGDGAGCCGcLKKERDlERCEIgjiKEEOyJKlaIggCSRA1QQosJ3A9WeFD6kNEAaOAxRgikZd8J4ycMCGqsiBHjE58K8QxEpDjWAgDSRa0ZCIIPMJzHMGqKOiKzFEQWE5CVva2tj948NAZGWZvmFISPEcualWO593A50XBdd1hf6QIcj6Xi70AY7y7u5vOZJZXVvq94Wg0SqVSx8fHmWK+0+uajj03N2da1tud7Uqlsru7W5wqp7PZ/f39cmUaItTt9ouFchTSmYV5BkGjWbt797ae0nr9vqprhyenr169WVpYHhkTXuJzuUwQesmkJklSs9nc3tvt9XqX4iRFUQ4ODnRdt2334qLmOQ7HcZqmeb6TnyqcXpxhHpenSy9evXQjTxTFWq22sbYu8kKr26KAddsdDBEvcPV6lVJqmqYiJ9LpbKGQ4zg0GAxkUY78cDAY9fqdwHVCL4j8qHZ+IREul0ibgxFGqNVoF0qlk5OT090D4PjzhbJAYSqVcn3P9/18OuOMjXI6x/zQmZgiLyiiJKtqZXZ2bJsxBJDnXm9vmZ5jOTaLKYEoIcmtRvPOrduO7bXb7evXr3/xyafXr10r5PKtekMUxd5w0Ol1OZ6cXpw7jrO6srQyvyhicrC37wdRfziOaOy67qU2XSJ8vVbDPJH1xNgwIcTjwRAxkM8UE4r++vUWY9AwrPPTi2y+iDhMMCoXS45niyK/vLx4cLAnioKmJy4uLjY2NmI/EBB3ZWNz3B8Yw9HGylroB57nLS0sthvNOzdvXd3YlDh+aX7hvF47Oj/tDwaXp9HQDwLXE3lBFsTmRa3X6qiyjCFKJpOD8SBitDxdGo4HqqqlktkwoIVCqdGsj8fD+fmKokjD4XBtbaNabXIcSaX1Wq2mqupFs/6Djz9S0slas9btd8rFqUajBTmQK+Zev3kbBXFWT8GIbr/ZUgRp9+32+dHJ+ura9pttVRLTSS0OaS6XF2XRj4IgCqOYUUpFWZgYBhIIBcxwbCqRkDBZVaIgtAwj9gPmxwlewRBhiCBGECPPceM4JooSAsr8UIGICyPXdMamh9VU3wyOLy4w4XiOYIhi3xMEQmHYbrbb7d7YsiWF9yZd5PUkZnqjC6d/gsI+Bo6q8BzHCRyYzcnzOWmpmLy6sqTImqxlZ+ZXZhdXZFGBMUUxFLEgYuJ7HoIYQhzGEcQUcMz3XUVIIMiperLvg892rX/x8f5vfX1eCzQLCFBUXSfyrRADATKEmE99K7RtAXH9div0A4BwGFPLdhO6RllYPT3EIFiamdFEicY+YzYhvi4BGYeVSiadSywur/gxq3a65ZnKxpXFqaw6nVIKCTkOgy8//4ia1b/6n3zvv/0LH35jgUcQ83HEQBiLHA79wHIdPwqjKAjDMIqiVDqrZ0s+4J0QEFmUZTmIKGUMIS6OKURctz8sFAqe53U6HQ5hy7AhBXHMLMuSFFnSZMpYfzT0PC8KQs92WEwJIZr67wLjFDBZVS8NSoHnx0FIo4AnmBc4WRLGk6FpTVRZ4TnCAWhbpsQLCCGEwWV4nOd5VVIjP7hc0YwHQ0hZIpG4LI8lk8lUKhWF9JItBxCEhAMcllUFAOD7PseLEaPJZBJDGARmQhEJwbbrOI4XBXFSTfi2NZlMOJ4DHI4hCIIAxCDyQ57j4jiOwjCKAh5zURS6ni0SPqGqccQoQF4Y+VFMKU1qum3bccQUUVpeWOx3uhzHHR0cFovFy8Gi1W2uri53Op1eu5NL5yzLhizWNHU0GuUK+dnZWUVREgnN8dx8IVuvV1vN5je+8Q1KKSZkZW213mxSSmdnZ7e3d3O5wvlFbWv7bSafe/z4seN4qqo+f/wkoyePD4+q1Wo+n9/f35ckaTgcWpY1MzMzv7jc6rSnypXZ2flOp63rWgRZKp3OZDKqJGez2XK5bFhmu9e9ceNGv9+HlPE8H1GaSCTa3a7tON1uN5VODwfjTm+QymV39/fCOLpx48bp6cll8tJ3vWI+DyiN4ziZTp2cnCRTmUsNlu2YCLLBYMAYQwipqlqpVEajkef4sizzHKGUtpv1TCoV+QEAYDIx5+cXLcOAMXMst9PqEkxYzNLprDGxPMcNPT+rpwTM9Vs90zBEUWx12r7vDyfjMIpULelFXq6QD6K4Wq2Px+P5+fnD4yPIgUql8ubNm4WFhSePvnr17Hkum62enSdkpVKpnF6cz87NpdJpxlin2zo7O5udn+MEvjRVzGQyvu+rsqJIMsFcQlZkReH+/xyV2YXZfKloWZbjON/61rfCmPK8uLKy9ubNm8sg9tbb13JCHY5Ho8k4n8/3eh3f9RACJycnAIByeXo46o+Ho0Im02o0Jqbhh8FlemQ8HtZqF3EUBp4/mUx4IiqablmWqMjn1Yv5ykwhk+51utduXM/ksrbrrq+vf/XVV7du3hRF0fGCtStXX73eooCWy8Uvv/xibW2t0Wg8e/mqPF1x7KBeb26srW+/2aKUbl67Uq3XK5VKpTL7wx/+eGpqShLkL7/8en5+vtms9vqdX/hT/+FgMIzDKJ1KXVycxYwur6z4vm+a9oN37zu2HQXhqD+UeAEhLCkygyiMI0VRJpMJ5onlOGEci4psug7jSEAZ4nme5yGEGMAoDD3PM6wJAwATDmPs+74feo7jJJUEDShjkPA8Jrxtu8Vy6e3uHpbkECIlofI871hG6Po0CGFEeZ7neC6TUlKaIGFP5eLI7IfmsFs78e0hAZGIGYrtos5nEzgpgLlsejaf52BcLuWubl6ZnZpNKgmBIxxBWjLBcTwRxChmmOM5IsRxzCAQeMl2PFFJUJE/G4EfvWj8s+9/+dXp4Ggc+mIqxAknoIIoua4vimIURf1+v5AtWBOTg4hCwEsiwFAQiCjxjUajenHmWBYHAGZxQuQKSZFRu9trhKEXh0FlqqwnEhNjQDDI57WZgp5PirqMl2ZL9rBR3Xu+Wkz+lT/zH6AwgIRIXuRijpq9Nh+FCsGAxbwoOH5EeUVJpZ3AC3xXRiQlaTwTIBWIICAOhTGbnVmM3Dj2oqSuiRIfRD4nCW4YOIFv+VZEQ58GaiJBAbt8yy79EqZteWHAOBRz2A58GsZpVdeIlFYTvCSICmePJwLCoihgjDAEoeuKPK+KgmtZmqIgBAWeAzSiQQgZ4BAhgFN4GSEkcATHTOXF2Itsyw0hTOWzIi8IhEcC9kDsghhykGcgJYiQQgaQYdkUxAIfF4sa5iCjECIMMRqPx8lkkud5P4wGrs2pMuZ4AXMEIJ4jnMTZocMgxRyUeUFgSOIlQGHEgCipI9MGHMdhYlp2hJCoaYqiPPn6q+WlpXK5rGfTUCQRBiPHnFucOT8/5QC6feWWMbYBAHJCrrfq07Mz6Xzu448/LhQKtuufnV0IPL6ytrI0u2iOjFaz44fB/v7+ZZCUBfTWtdsY8ZKkVCqVXq979eoVVdUCN/jg3XujTkvCWJWVk+Oz9fXNyWRSLOQCxz4/OW50Gk7gu7bdqNZGk8nbvT1IuNdbb8zxZGa6kkgkGILJTPqy+JPPZIuFXK16rqeSvdFQlCXbdqkfa5I2HBkrqxtjwy5PzwAAtrZe33nndqff7nf6siCbQyMhqrqWare79965L8jSaDIuFou9dueyXOYFvqpr3U6v2eqkc3ld189Pz1Kp1HA4VHQtiAOEkCiKw/GoPxiVSiXTclq9fkBZ87wjYXVsutV2m0gShWB3d1cWlbmp2dAJeASLhVwhm0tpKd8NBEEgguB43mhiLq6uMYyQgOcW57qDbgTj/FQBQnb75vX5mdnxYDg3MzsaDOMwWpyb7w0HkMO5fL5aq6VLeY/FvUHXtu1qtbqwMAeiuNdqj8dGJpN7+2Z7qjTtunbMIojB2/0timmxXHz68lkqm2IA7e0f3r9///NPPzPGw9u3b3c7w8F4Ypp2s9ncWFvjOZTLpO69cydXKtZ7nXQmIymiqqrpZKo3GBiW1R0OOIkPQZzOpxKykklmCOSSWioOwu6g3zeGelq3+oNwbEEA3uxu1fo9H4IXb97k88VxZ9hutOut7sHJuaTJo3HPtIbLizOWZVVm5nW9sHtwLif00XBCw2hleYnjuYtaFQCQSqUSklzJFznKKZK6vLo0GPQUUWS+/+UXn22srnU6fdtyNjc3u/3Oo1cvxFQSIfT5548wIrqekgTRc3yeE0IvFok4Ho4R4jRNE0URY4wBNg2H43kvigARPEoh4f0gwoQ3TDNmVFBlyHOOFzAIRFGENBYwwpCLI0B5EhHEI8bFYVJOTCbucaMdivzYdQkhOGbIC8OxieNYkcTBoGe6hu2ZceTxGOmSIlCYFjhgjd1+05t0BY4lVC6tcbM5saLEFTXeqGRFEGJGr1y5srm5mU4nIcEUgRhRnhcx4g0r5IjMEGEwkhXCAebZHs+LSBBCUag56DeenP7Kl9ufnfbbTBjHuDu2Y0B6/bEXUUGSh4Mxi2kcRsmEijkGURzHsSDKpfI0ZdAwDBqEEkMJRCaDripzWlLCXBy5JgrdpAQVwuLIgyiWVVap6FldSErcdCGbS+v104Pzwx1EiAgxRwSO0iCpJ3zbjH0PxDQMw8XllcHYMb04iCKOQ4gBDiIIcRBEPMchhBCHwziKQ4oBVrVEFMecwA+HQ56XMCa8JCHChXHk+75t2wwC1/coYCGjjutijBljpmkyxjJakgURpiAOo8BzRJEXeQ5ACiEEGCEAKaWuZ0dBmFaU2HFiPwCAibwgijzkuVQm7TvuZDws5QtRFCEGEGUcgBhA23Qsy4m9ICErtuf6cYQ4HEVUAAiH1Pd9jEk2m02mNMD8hCp4nscQ9v0QIQ5yOAxDgnnGIK8oA8tw/YBBBABwHIcxRgjhOAwoxQDyvMgiapk2gjikLJnQBE6wbTcMY1EUbdu8uLjIZvPHB4cX5+e2bTcajWw2u7q21ul0MpmMJAjPnz/P5/NxxPqd7urqsu/7nU5v8+qV5y9f8iLJ5DPHx4f9Xq/ZbLaanZmZmcX5hcsWoiAIoqK2Gm1ZVCaj8SUk5+zi/KJWT+rpH/zgB8V8DiFkmvb0dOWrx1/7vj8eDCVeWJ5fkAQRQuiHYUjjVDJz/fr1o6OTubk5WZaPjo58xw2C4OTk5MnTpwCA6sXF7u7u3bt3a7VaKpsZGZOIUUVRzs/PS6Wpvb2DfL7YaLR831/bWH/69Onc3Nz09PR4bCQSehjGxXx+Zmbm/PRM4MjVq1dfvnw+uzDL83y338nms/V6s1AobG5e3ds9AABk87n+cLC8tmrYVi6fd/0wjKiiahGNiSAVyyUk8oAjy8srrVZ7NBrPLcyfnp8BBNO5vB8EACNZVZrtFqX05ZvXluvMzs81261au3kJDR0bhiCJtVqt3+8TnvejsD8YjEajVqu1t7c3W5k5PT2N4/jyiuOHwWgyrlarmXTaMMa+a3uep6kJBGCn0WYxTWp6EASmYxaKuU6rIWBO5PD5+fns7Czi4Hn9Ym1zo90fjCxD0RKDwejBgwccL07GJgWMFyRO4DmBPzo5cQNXSyWPT04mlt3t9kej0Xg08hy30+lwBGVy6c2rV0ajkSQJZ2dnju9BCC3TgRRqamJ9fT2ZTu0fHFRK5fnKTLvd/CPf/W6713V8b2Fpqdls3r55M6Umbcu59867MaOCLEWh/9mnn966dct1XVGRv/ntD1+/3RJlifD4i88+8Tyv2W47nnvpMlMkWVO1drvdarXml+YPDg48z11eXPr93//9hcXFuYX57d3/H03/2Sxblp93Ystu79Ln8ed6W76rqoEmPBpDcYZmYoIERzEzpEYhfhtJbxQMKkbkMEgIBAkSbIAigAYK3V1V6Ory9966/niX56Tdub1bTi8S8wkyM3LtzPV//s/ze57ff+uB1269Ptg3bWd3e2c+mUvGHctuysqxPQ2T3nAQxXFZllKAsq4gxEqpTqdjGnael2mRG4ZFKeWc53nearVs05JcKCFtw1RM8IZpBEsp65pBTACCQgidEMC5holE9NH+UQExooTxWjBZFyVGiDOmhCCE1IzHcVoWNZCSYgQk41XmGmjYsjVViyaNl2OTSCwL34Q9iwwsdGfYGfpGPL3Y3uj/2q/+yvpw6FiWadhSKd10LdutGdNMTTe0lSkOIZLmpWa7DTZKYidQ+3Kv/ncf7f35t3uPx1lmdBdMp8G6QDpXSEKEqZbmWZIkSinGmBv4lGqMMdPQDIMAxcosL/Os7TtFEhHFbJM4JqmLZV3HnqVVWaIhkKTLLF0Oe+31QUfXMMGqHdgUS0QRLMuiYhWgsKwLgKBksswrohlXs7BkginIoSrrCgMcxylXnImKVaVByWCty0S1WM4NwwiTpBLMMIxuu5PMUx1beVFJBQWTRVaajh3GUcEbRbHbDtzAxwACJlxqVFEqhWjqumzqsq4E45CDoqgo0ZMkJYTmea6UgogUeQWZQI1YDexcipzV42RZQKFbZsvzF4tFnmeUUl3XLd0gEmx0u66uUwV5VQ+6PQRhk5dlkkGAbNu9ef3GxvoQIRQvI1M3fMeN43gRRoZtAYQQxg0TUCqDakBwTdMaIWvBAYaEYKRgHqeUkCxP0rygupEVhd8KFES8bhSrWVnYtmvbbhaGXdf1/RbnstfqvPvgzcuTE1vT6qz4L//5R9d2b5UN44Dfe/Pe6ei80+lc373x8V/8JJqGsmEff/zxnXu3c5afnh8PBmtJmk/ms51r2+fHJ2dHx+2gRSmVEGRlMZ1PWF22A880tKLMNE0zDCOMo3c/fP9yMQ+TZHt7GwDQabW7nY6u6xST4/3DlmlrCKdVYQXe5WT84tnzluMd7h8sk9gwjMViUcQp5/zu3bvdbrfb6/lBsLe31w5ayTIilPaGg3kUt7o9DJGtG3mUbK9trELwa2trYRienp9v7+6eji4UhOPLq8NXex0/ePL42+fPvnvj7Tcux1cVr3dv3ZiH4a1bt/I8n09n77zzTpxmZ5dXQbf37OUrjRpXV5OLyysBUV7WluPFafLs9UvdtohlfPf6udv2Lct8/vS7nZ3t0WQc10VD4d7pwTxaOEFrMp13ez3NoCWvAMH9wZpuWEEnMExNCYkUsiyLMcGYUAqeX15hqt+9//Dp8xf3HjwECE/ni7sP33j07RPDtC3XO3i99+bNW7IosyR96613JhcT1/AcNzg5v7h+54ZpG2cXpw8e3FvOZsk8/JVf+lvPnjyjlK5vbnz51aMP3v9+f32wSELHcb755pETtGrJdZ2urw/b/UFSlJvbW0LK71696A7Xrq6u1voDLJRn2qPzU12nrZZPNZxnieTCMs1uq71Yhnldb+5sQ4LH4+livqxrtrGxtb+/f3l5yaTaPzre2domEJVl2e/3P/30U8dxTMO4GJ3HWTwcDlklukH/059/djWdXF6cHh/umbae16mmoaATmKa5vb3DpAhaHYLwYjbvdLuT2TSKojwv3SBogEqSpN1qRWmSl0W723n06JGU8oMPvv/ll196ntfygyhc9ocDzdIQkKtfbc00LMdGBJesWUVesiybTqe+7xsES94kUeI5nmuavCo1hJu0cnVTA4AI5dsORkQ3rKysNN3UkYYVWOXppJRe0Hp1tigBBRjVZaWUwtSslKykpLqNgGaZQdDqD/rblukTDSvcOG2raJIqXQwDG4vSxKrIorquddsMAn/dNjYtsOPDG+v2/OoUiea3vv8r7z98r9/rIZ0CiqmlWR6pywgK5dpeXjJAKNH1pCg5JRwRDZm2iQQFf/l4+vuf7f/R4/HLhOzP2VVUVhIo3VC6abU6HEAloaE7Vc0lBJ6pV3lEYOP6ZlQkcZGlaTxo+6iqQJ6ahnRbhgAVB03H9wxE2n6rqTlrRFGW6+v96ze2dAv7bRthgmzHVBAoBau6Nk0zywqgsBu0F0l2MZkrpCkAu70BY8KyrIo11NDTNOWcV6xJyrzb7XY6HU3TGsHzsijLcjDs1XWNMeZSGIaR5lmapqZpOo7DOY+iCFOSlUUj+Grqn83nhmn6vg8AMDWzrutOr1c1tWVZdV07nmva1ioKwRizHLsWXBFUc0Y0SnVtFs7qukyTxDGMYX9QNjXVNITQoNOFUomqIYTkeXp4eEgR7gatN+4/tCxHKXV+cbq/v79CuhuEWobJJdA0TSlV13VelQKoijWccyAVlsA0jKKouBSYkqIoLMta4R5X5UeGZTZNI7mgmGAABeMIoaooZcPu37lLKbVdhylxen62u3P96OB4GUe/+hu/Hsfx4d4+xjBOloiQ0/Ozs7Ozu3fvbm1tIYR+8IMfPHvxUil1//79k8OjzfWNt95668d//ueGYVBKX7165beC1wf75+fn13Z3T05OmqZJkkRIefv2TV3XfN8/v7hodzrbu7tVU3MplAJcqPk8XPHiD/cPru9eaxjbe32glPJsz9D1Xq+3yhz0Ol0I4ebaelkUiJL5Mtze3t7Y3Dw/PzcMozfon5yfbW5uxuFSMW5pRtvzlRBRFG1sbIynU80wqK4dnp2sb2+VvNk72N/Z2j4+PtxYW3NM4/G333ZarePj48V0trO19ezZd5qmSQW++ubb/nBd182L0WUraC+Xy0bw67euv3j1UjcNx3GuZtMHb73JOV8ul/cfPlwsw6Zp3nzzzfF00ul1u+vD88nVzs1bw82tyWSytbM9mU0N017b2i5Y3R8OirKKk4wSPQrDpqpns5mEoN/v27bt+X6Yxpqhu63gajadLOa14KcX5zdu3cyyzPf9tY2tbx49cl2/3e381cc/29jYmE6nF5ejm/fuzBcL1oiNjY1nz545jiMF+Pmnf33/7r3JZDKZzDqd7o9//ONwsfA87+DgYGdn5/z8PAzDTqdzcXFxcnw26K8dHh6mRY4wLqrSMa1uu9Nut1eylVJqNg9n08X8YpKH0fxqEkWRZpmN4q/39yzHbnXaKyD+/v7+7vVr0/kMEjxcX3v86NHG2vp8Pj+7uFgh6Zuq7rTaB4fHUqlep5skye3bt4brw26//8YbDzCS167tKATrpinLsq4rIUTTNKOz8/lkenJ6fP/uPU0zkiQRCGCM20FQZnmv3YnTfBnH777zvcvTUcvzv/fue1VedDqdNM8ePXlsWlZZloZhSKyIRi8uLxFCrusvk9iwTIhRELQt3dA0Tam/aVko6wITCBVwLKsuyrIoVkSdpmk0wzJdNykKKQGQgBDKhRQQBN2eQuDl3hkDSGBMsFYUJdb0RiohpWlZSZIUWR4uEggwJohqEBHY63UxRZejUx0DgyLXNJDigpc6Zr4BfE35utjqum/d2gZZ1CwXD25cf+P+vZs3b3q2A4EEQHZ7bYQBAsCwLE03lIIYYyGEUgIAQImJDLfR6aTW/uzro//0s8f/9YuXz0b5SSTOwqLBRhhnuunWjIdhCABgrOGi2tla0w2CMLh+YzfLEi6qui4ck1oGmc+upKoRxVVVVUUumtoyzH6nKwWjEApWYsS31rqtwEJFXaz8PRBgjJDkwtBMy/aTSmZcEcuMi6yqRRxllFLRCA0bnAGISHcwrBvV1MrxgtPzc8EYlLIRvJY8zeOqiW3HaJoqryvHcwVXSKJimbGs1hHJkhRqhEFVE5CLhkMFMMrznGICEEKURHkaF5lSStf1UorZcokhQgBAXZ9maU1gKbjteBrSTIxdwzIMw/c9CrGByPpwUPMGUzRdTOsiB0okdbZx89rb7727s7PtOM7x0dF8GWZVqQDgkhVFUaRZJ2g1VYUU4JzXdY0wME2TCS4wrARTTFIFdYgtjNM0wxqVCAoICKWMS4iArutAqqaqqVKKNRAQ1wswlBoBvhu8eLHntoIGgcOri87m2uHxyc6NG06/89FnnwYt7+7NW0VSIUWFELpjDbc3R/PpaHKp28bp6OLBm28YxPzFJ794542HZRw9/uKL/+l//D9rhj6ez9798L2rq5Fn2rvrm0f7B7u7uwCjZZqsra2dnJxApTzPWUVOrq6u4jS1HUcimCRJu9cVCMySpdMOFsuo7QSeYQ06XQBA0zTLMLx57brnuEdHRyu+eVEUeZ4Ph8Pz8/M0TVdgAITQte2d87Oz7fUNi+pZFC8XoaZpnU5nPp8TQhaLRc0ZJPjF4V7Bm/tvviGQvH3nFkEgS9O//Vs/LON0uzcMLOv185f3bt9pBA/j6MEbb5xdnBuG0e900yR578P3tna3JrNxr9NO4uX5xSnR6NnF+ej0jEiw/2pvbWOzYuzl/oHvBVEUffzppydnFz/+q5/89ONPOG9+9Ef/aTabnZxd/Iv/9//2zbeP/8N/+g//+7/+V8+ev/jJz372o//8x6PR6A/+4A/+9E//9MmTJz/5+Gev9l4vwvD3/t3v51UpIag5CzptAOFkPnN8bzafX84mP/y7fy9qamzbv/k7v3kxubB88+333nr9+nW/P4yXSTrP3rr/dl6xqMxu3rx+fnq8vbVlGWaR5d975708SqLZ4tatW0qBbtC6e+PWy5cvPc9TjB/t7W+srbc8v25Ky6RRuKQYCyUXcdIfbgBIpAQ7G9sW0j58671b1647vhPmUdzkxNDPR5fzMDQMQzZ8fbh2OZ31NjbDRRSHybvvfO/yclxVjRBKSWiapu/7ruvZtrtM0nk4sWxCMQzni4o3jRTj0WVdlLWQ3718fvfuzSwJwzDstjuu4zx88OD2tRtf/OIXjmXfvHE7nC8DN/BdTzGBIOR1s1gsOGPvv/O9x189SqK01eq8ev1iuDG8mI3TpgAUZnUugDJdByG0yi16XlCWJaW0Ksr5bAYVAABwIA3XRjoumhogiAhRBAkMK6UERJTqvGkEUIZj14wBiIUEkJJKsKyqhp2N58/2Ug4zgErGEEKWbgDBu/0OIIIgPhmdjy8vJuNLwRogVZ4Wi0XY29wcbG1RSh3TAkIEttazIWjmebEwHdppBaZSgaxud5yuLqene4HjvHnvwQdvvXVn+5prOIIrKbkAAmNUlSWRwKIE1LlOFERSUZLWouY4K4QVtE9i9tl+9NHrxc9ehy/H1d5oCYlZ1w0A0nb0LAtZnRdVdTWbQkqapgnni7u3biskR7OLWTLXLN1v+U3TEIgJ0TDGWVlE0bKqC1vXAKuqdAmaQkfSNykhlDaMrVDaZVEsk4QSO6ul0tzRPElKaFq+BrS6qBopKaVAQSml7hhZVWKMXdddhQtWZStI18MwdGzLC4IiyyUXUZz4vq8TahpG3TRFUTS8MQyDscawLaakgmBtazNNEmoZomFNVXktryhrk2gO0fOqVBBgjUKIECFMSUVQXhSDwaDOK8ewqqIxDMN2LME4UQQAMJnN8jzHALVbLd9y4iLzW/5iuYjmKpwvsAKabuiel9W1YRBIMOccQjgYDJIoFUIRQi2Kr6YzAKHjOMsoMg3DMsxoEbY9twDKdV3WcEKwlEIC1XCBoMIQSUR1XTcMI8uyJMls16qqyjIMWfLBztrF1eUiWqxtbhweH+1c26W6fja6uH//fhTH8/l8c+f6sDcYXZw2TaOQEkp2eu2zszPO+Ww8w1h/440H8/k8y5J2++GzZ88ms9mte3dPLs57vZ5tWcfHx//t3/nbs3ABINzd3f35Z5+bhqFp2ng83hh0m6axXacW7OmLl3fu3JpOp1JKzXaoaTGh0tlsPB6/885bF2dnjm332h1IcBQuDU27deP66dmZEAIhlGVZkiStVkvTtE/++tO/99//g9fHhy9fvtxcW3/15OlkMtnY3Hx5sEc0DWA0WF8TVVMVhWNarG421tallKvB6/XB66Iuev3+t99+3esPGGOTyaQ7gIcnR0KI8XRyenoKlfrkk0+youz2e7//+7//+vVL23SqqmKCP/r2SafTCcPw2XdP8zTb39//yac/K8vy62++AlJFUSIUaHV8IeTR/iGGEmP83XfPTNchmtZut27dupUn+enVOQa4Mk0puFBgNBqVgtU1UxzYtoYhWl0SGWP94eD04ryoq36//9HVRwiRf/cf/2Nn2I+i5aNn32mO1Rr2p8sFxHg6nW5tbNV1vb9/MNhcMxw7K/KiKJJl1Gm1HcfZ39//wYcfYghXfUxpFB+8PqBUD8Pow/cHVV5Qqs/nIW9EELSn6fj506f/5H/9J4+fPW9qsbO7+/PP9h7cuqNjcvh6b+v27mQ0hhivb248/eLx/YcPXu/vIYR2trY+/fjjazduzJaLjY2NcD5PF8vhcAgAGI/Hf/uHv/P0xQvfb0kBHMvJkmWn3yvqEkDeavsnHx9NJg+wpk8mk52mWV9ff/z48RtvvIEQ2tvb++V3vvfFF18wAG/fvPXoyeNJON/Y3PzJX/3VrRu3meBhGHa63d3r16SCv/js87/7d//eN599werK9uwoitrdztHFma9RC1qGYSgh53keeJ5p2GEYEqAE4wiqwPNZXZu6wUWT5qmhUcMyy9V/QMutG6hpWpamCutSAKTTLE8pxVwCCFXDhASgLMuO37qMp0dX4bWBFy2uOrazghUL2ZRl7tlav99NszoMw0pUrZZv6gaDanQ5NU3TbbWVUsSwo3BZ54nvmLqO5tFcAH2905JCMQ7mSWO5g3GcixJdGwz67aDX6ZxdXiTRUggFIGoavlxEvGkc02RViQE2DZ0pIBijlERxjpGm2caLy/RoFO33yfdub8Rps913QZ17Omo5huJsGS17vV5WlkIIQowsj6/duL6IFskyKpqmqkpd1yVXRZ6bmt7qtLO0UEIA0hg6ArWAgs3nU9M0kVAAYqRkoyQ3qUOgWVUVNY0C4HnODSfARIvCuZAcU5JkqQJcgKa3scahiOIFwooryaXAlFZVXde1aVtVzYuSx1GmQToIOlWUNnUNIFxkMbR0w7G5kr12BwOIuKQATSYTCUENZAUlIYTnDWqkS01NYp9YDjED2y15E/R7Qoh20Op6QZ5lXrulMLQsS6ca1EjGqpQXJ1dnGOM7d+7cfnC31evMk2gRhidHRxeHJ+ky6nVaru/lTZUWpcJoFi8N22ya2jJNXdcxxoJxqIBSalWBlIaJTqjkIkky23NrIPO6qhqOEIFCIqk4k4ZlK4gaKRhjEqgwSyretH1Pp1peFrppxmmapXGVZ/fv3D15eWBg/eTkaDobd1y/6wU1ZzWQTdNMxmPPcfr9/nQ6J4REUXQ1HmElDU0HArKaSyiDXktC4LWC9fV1y7JarvfZz39umubNO7e//uYbBUBRFF99/sXdm7fmk2mSZY7nHuzvm6ZZc0Y07cHDh0KIXqe7jKKTqwtBUZYkvGHDzY39k6OyLFf5bw1r4Xy+/3rvq6++6Q2Gs8V8PB53Op0sy+JlgiEZrK+dnp3labG5vvXi5Uu/02ZInY5Hg92taRRO5rPTszOdUseyWV66uom4rJKMIFwUxXDYL+oqKXPddcdxlHJmeN7OrRvj6WTvYO+7Z999/e3XX3/7zefffv3tsyd//uMf//TPfzK7mE8m88ur6XwaAikns6mCYJGE7X47LdM0jZFkVHKLgl7LWe93Vc1attNrt13LDdxgd3tn2O2s9XpFlh8fnV6Ox2VZpmlc8qbiDGMsuBr2hmvDtQ8+eG84XMdU0wwzaHeG6xuLZXw1mZVV8+L5qxfPXu7t7YVp9NNPfnY2uvjzP/uLb797ej6e/OVPfloLNo+W03CmmzoxtTjPgnb32rUbWVx2u30hRJbEd27fUggpiTaGa65t7t7YvX7rpk5N23QppVu7O4qLdrvrue00Kbe2dkzNevH0mefaEgmmmrfefnA5Ps/LtOHN0f4RhsS1PIK0Vcz1V3/11yaT6cvvnt+4dsNrteI8n0ymG+ubdd1MZ3OMsW3bB8dHXIhwGU8ms7OT41s3r6dlsX39+r17d58+efT+W2+tDzcev9h74933ZcMMRN57773D0zPdMv/Wb/zaz7/8XCJ47dq16XTaWRsYnvNXn/zszt27EsFllnQG/YPTw+9evLQCTxH047/4i3a3Y9jWdD5DCNm61vZcogCCYDqeMMY6ne48DGvJddOwDFNxgaQSdYUELLMcI2holHPJaw4ktwyNNwwoNZvNEMYQU0Qpb2rHsaiuYZ1WUuq2o1MjsH2Jgd1qPdk7ZdAGmhYXSckarFHbMKHgUsqsyG3f2thahxItptHo5AIJqGrIa5CUTQMhNo21zU3bDZI4E008bGkdmyfzozSdAsi2NttdB91pWZtUyXhJuRp2+2/ef+vWzYe207q4nM7TlNsa9hyENSQxVKCuSpMgDISGIYRQKZXXnGEibedgif7zL07/y7PFz8/rETNDhqO8Lhu2trsdpjEmREipKIO6PDu7EBwpYDUMM8YlFxAqjLHpOVGW67oR+L5h4qwMHd8EGPSGPd3SEcSkLGvX9THAnEsIMUKIUO1ykUa1DJM8K4sgCBACTDSu7zSioZRmWcY5d12b8wZCyJWcLRau7wnObdOhmpGlla6bUgIpRLvdRgidjS68IKhZo1ZqBmNASNY0lFLbtm3bzspCKqWbZsOZZugAwVkUSgy5EJJJjHFRFJqm8boxDaPX7qxuWECqPM9Hl5cVZ0ijuzdvdHrdqqmPTk+evHh2NZ9mdVmW5bDfL7I0jRPDshzfy6tytpgHnXbZ1HWZ24auaVqcJIwJKBVjLI6XQKFVbyohxNI1xUWaZ67vKQiSJFIKIISlEARhCKFpWoTqUkFEsO26kMA4jof9tVXYYbi+truz8/LpszcevDnoDYMgGPb6G/31cBYigoWUSRr/8ocfXl5eLqazmzdvz2Yzzti1nd2maTRCdY0cnRw7rs+EiOI4SZLz83PHsmez2Ycffkg1jQuxe+O6Usq2rN/6zd+cTCa+77/55ptFUbz19tvn5+ftTocQenhw0JRVluV5XuZleXx8vLm5CQA4PD3WLdP1/dlsFrS7x8fHpm7cunHzt3/7tz3Pc3zv2vXrR/sHD+7eq+t6Hi46nY5hGFLKpmkePHjw5NlTrGuD7c2Lq0tsaFu7O03ThFEUx7HjOE1REgV9y5lPp1LKsiyrqrqaTUeT8bePnvz0k0+Pzk9///d//8/+7M+KLD84ODBN03YdjHGSJJZtttqBZVlSym63uwqLrlx0u7u7nHNIsGboBGOd0FWUsS5LyzCwko5lWKaNVujzsiyy/NrOzvvvf/DgwcP33nn7wcP7g+Gw4dzxvLffflsp1et0kiRZhfIfPnz4K7/yK1TXu/3ejVs3V3H8v/FZN2Wv12nKanI1vroaV3l5Obq6Gl2Zuh7OF+tb69PF9ONPP3n6/Nmf/eVHhWgURFmRV1VlWGZV140U5+fnNWNCyYOT49HlZbfXS/NssZy3e73ZdF5kOVTg0aNH6+vrN65dT5LEME3P91dATYWRFwTvvvtuHC7Xun0Nk8lksrm5uQoxrK2tnZ+fr6+vM8YczzVNk3N+88YNoSTEaGNjoyzLosiCwOt3e0dHR2VZj67G3z5+cvfu3RfPny8Wi8H62rPnLyzToYh+++13hGgIkcdPnt57cD8IPClEXden5+eMyzffePvi4lLTNN/3ijJr97rrmxt5WVu2fevuvb29PappTKisLpdJzDkHCFJda7VaCMK1tTUhpKUbkinGJSEEEUgIgRAihIRgQggCEVAIAVhXheKKQGrbNqW0YZxxThBCUOVV3ghOKa2qSkHAgcrLotXrX4bp0WiGdYtQyhVQAM3mC8FxxQCkelU3ZVmbhosU9Twvy5IsjeM4rsuyqqokT8umcgO31+sJztMkdDS01nVNLLNklkfTlkk6Nhm4ZKtjtzTgAtXS9Z7v37x+4/3339/a2R4MBpgQ0/WIZrU7XUIIq0vPsQGUQgiEtbrhCiCBtBwYpdn9fD/8g588/+o42YvwmDu0tztPGivoCIwlhjVjs+nC8zwI8WrbF7TbaZ4VRYEJTNNYQVnXdVnmURJblpVXOeN1kkaEIsTrRteMPKuJZlZNU5a5Rg1EjJIjBig2aVkXAACMqZScSbYyUbm2U2W5oemO42CMbdehtllJDoSUNSuLotPrJllasabiTGEklWq3255jISkUF5ZpCqAARgKoJEs551VRDjpdJWUhWYUVckymY9T1ch0S317EERVAl7DT6lCqmaZZ5UU4GqezEEk17PduXL9+7/adbrc7m80Ojg+ev3o+X4ZE17x2C1ECEKw4I6bOobqajE3L6nQ6vcHganJFCNIo3txY1yyTA6CUghADjJZZIpSUEEiIiqpkTcOr2tasJMkaXlNTkwhzCRirNQQd3WZFw6SAGlIKQo1M4tD07TIrVSOhhjJWvn61/9ab7xydHze85o2oyybLMs00wmVaNezWrWuffPqTwWAAMV3Mlzvb1xAk49F4MFjbO3gdRovd3e0sK2bzSDMNBcDd27cP9vYYY6/39kZXlxKoy/MLx7YVF59//nmr25ks5h999JFpmmcXo1v3749GV8v54s71m/t7e4hgz/Nubmw/vHU3iiLdtqljzZJoNB23+73pdGqa5mIZXl1dff2LL8L5opHi8OT4ww8+ODk4LOvKsK0oilpB4HleURSL+fydd95p97pRHK+treV5fn5yev369WUc3b5/jwl+en62NhjUda2UiqKoLOvxePrnf/4XP/vZJ7PZ4sWLVxeno8nlxLctneLAc5uqpAgSCIbdDoTC9AyOGmzgeTQLAjdPk6YuEVTzq9liFtYVK4pKAFILwKHmtrpU14UQZZ6XeSEQgBSXZbm5tt5vtzzLubi4uLi4MKleprmEYLi5IYR4/vz5w/sPMMYY48FgUJfV6fHJ06dPVxrUy5cvF4uFZRiO4+hUowABLrACW2vrLMmfffMons6Xkxms+ej07OmL55KgNx48fO/97/3iybdfPnvyi2+++tO/+HGYxEdnp1AjiqD2oOd329Nw0R32vU7r+d6rijNIyeHpSavbjsJpU6bvv/+ehGJ//3BtuFELJRHx3O7RwanheOP5fD6f3rxxbTq5nM/Gg8Hg7OxMKZXnOZPC9bzDw2PBlYTA8b0qL66urpZhxKXClEAE3v/eO1zUz169+uVf+Y2j/aNottjevv7No6dvvvUOgBJJ/vZbD09HV5N5fOf2/U6rgxW6de36+fFpXZaQyzLJNgZrZZqdHh7du3N3dH6uYfL2G28+ffxkY229yvLXr15JxlftJX6nLYA6vxwRU5cECQg451mcWLrhmJYoG0PTMCVAIxyoOM+YktTQddOAGEkIVqBvhJDkgiBEIJISKIIUBISgIk1MTYdSmFTTKeVQ5LzWLJNLRC334GJEdLPhCkAIED46HX/x7fOvHr+KMtEwUhYCKU1wkBW5ZlLNRECVTZZEkysgKwGqOA0BVuvDnY4/LNJG1mLY8tY8UxVxODn3fG1tw+t5fDMQm5b0Zb7dd7fXOvfu3Li1szNwgs3+uuP5ZsurhYSUKkyYkHXFbdsBEBOsKyYF58SmNRJG4CUM/tGnp//iT5/90ZPFX75cTipdIjvlLC0rCl0d2rPJFCJuBVotS6ST3rBnmDYAAEARRzMm6rKuGINYt4VQQnHHs9M8QRgS3/OYEFES265HNSPPsqbh5+NpKSSA1HX9siwZq1epzk5vYJlOOJ3ZpgUhTpKsKassy5gUZVkihIo8xxBEyxkAQAKBKSnrClHScDafz1delDxJRcPyPB/2BxgiClEWxVWSObpZ17VuWwKBOM9K1lBDz6qyNxx0Ox3PdqbTKaubeB7mUdLrdjc3N3d3dyGE8+lsf2/v9fMXURjWdU11jVK6IgcJKZ12oCgmhi4gCDrtyXhcpBlvGsdx0ixuuc7G2nC+jPKmQRBDgsu6cv0W0qhCsBGcahoAgFLa1DUC0HWdsiwhhABBKSUTvEgzXjer2UK3zPliSS2jrBsMkRDCdJ2KNabtnJ+PLNuWWEVF1ur1G84m49ntW3copRejsx/84JeWcXx2MdrZ2Tk9PTd0fXd3968/++zajeu6Tp8+fRIEQa/bzbLMMIyjo6Pr169fv3Xz9u3btudmWcYZm16N2+02RAgTsrm56TkOQRhr9Oj4OE8zQ9fPz8/ffPPNxWJRFIVpGAf7+4jQoq5WpawNY/tHh51OZzwev/HGG9duXPfbLcbY9sam7/svXrzorw09zwNCQqnm83mZF5Zlba1vPHvy3WQycWz7anQpa+bazsuXL++/8fCLb78GBN++f+/rx48gRmESf/r5Zz/5+GcHx0dRnCqAFASaprmua9u2UmqV9AvaLQWAlBxASTCu6lpAIJRc2d2qqoIY1azJsqwsS8MyS9aE0dJwbKLRFaLH9f1f/+0f/uDXfn2wsZaXpRe4KzjobD4Z9ntvPLjfNI1lWUmSpGly/+GDH/7wh/1uL/D8Mi/SONF1/f79+zeuXV8sFlJKx/c21ta3trY++OCDX/nBD27s3Njqb9y6duPh3XtvPHxw587td955e2tr0zCM/nBIDHMZJX/yJ3/y05/+tN0JNne22902YwwQ/Grv9bfffmua5mQx/+TTT3XT0HX9YjR64803NUPnUq08Ode3d/qtzuNvvzVtmxj66eXFjRs3Xrx4sb+//973Pnj58iVA8MbtW69fv7x969a17Z3lcvlrv/ZrL549xxgTSsum7vf7rVYriqL5fO667gqy3e/35/N5q9U6PtxXkl+7tvvzn/98Y2NrY2NrPLpsef7KNjafzQ72Du/de5CXdZZlxwfHcRxPp1MmuGmaTDStVmsxm3Mm8rKezWatVuvVq1e/+PKLe/fuPX78xHW9fm+Y5FmapoO1tcfPvsMaHQwGi8VC13VCiGK8F7SllECp6XRqGAY19Hm8RJpOTR0bZBFHRV0hgoUQCoisyB3XBwgWVcUYA1JxziFBBauxRg2dAqWapoFQSaW4ZEJCAcBwc3MeRUktGNIAwpyJ4WAbQONg//Txo5e6GXClJXljugHRrMlslhYx1aBvW5ZGmyKfTcdlWXImF4tYChJ0B6ZjC1bKJh20Hd9GV1dHaTobdKydoe8Q1vewB0tH5m2sbvS6D29cu7Y+pBi1Wi3L860gQLpZNI1hGFAhABAhBEFcVyyOl3VdxkkGTY/r1kUCfv7i6s+/Of7pk/MX4zIjLX/zVpQzCYhEaDafr4qgpVR5UXMudV33fXfn2nbJcoWRALBhChPSNM0qPUBY2QjOCcG240RJVTV1r9vNiakQpqbDAakaZrtGkaVra1vT6TwOk97NfhlldVETalSNsHRdh7iqa4yI5ThZlmEJWMVMkyKEGt7EcdxqtZSUTAgohYFp0zSeptuaUWV52/NF1fiWQ3W9amrPsDAinAtXM9qtbhiGgWMDpeI4rqoqZ9XG2vp6q4s1GjX1LF4eHu43TWMYOsbYoYbTcU+nV47tNTVHCLGm0XU9bSpEcBQt2p5fVQVFCDIGEdA0AolWZUnLd8MskQgbpplkacm4hGCZJZqm1axqqjqwHAWgaVl1muRxxismmDJMkyNQcUYNHSEkG0axxkSDKaFAg0piigpWhmnstFrJZPnOW28dnO6/Othvbax/d7A3WYQ/+KVfPjs9dW3Hd4NffPmFIujm3TuPHj1aH665lv2T50/ffPst03bDKP7BD36QRUsKQeC4z54/7w0GeV2dnJ42nG0IaRvmoNNut9uvD/a3fHe1llzvr1FKX756/pu/+Zst14uXEdW16Wy2ubZeptnhyfFwezPJM4Xg7u6ubZqbm5tra2vffP3kvfffv7qcnJ1evPfBu2VZnu4f+7a3vrP75ddfBZPpta3tV3sH27vXLct6/fLVzd1r7735Ls/qIkyH7b6seJ0Xd+7cOb0437i2E9XFy5PDZRr/8Z/919FoVNQV1JAVeFICrOsKYYBw0zReENR55gVuIYTpuqws0kliGgbnXADieh0lRFZncZoBTVskqWVZAktkkqvZuNvruqYhqgYjPui37965P57Onh8eLJYLTOD2tY17d+5jgHWqBb4bRdGjR4+mszCKY8u1d3Z2dnd3y7y4vLxs+cGtH/4wy7Juu7O3t3d2cd7vdq2Wb1nW2w/emF2Njw+PDE0jAH7w7nuq4YtoPkkWhSheHu8RQlp+m7J6Es673e5v/84PZ5Px5dVFGoZZ0Xie1zQNkJJAwDnfP9yfTqd3bt3+kx/9sdNyn796cXJ8fHx0vDZY67fbi2nkvdtaH2z8/MvPf/k3fplJtvfq+XvvvHv88tno8nQ4HL5+9fL0/Mxy3GdPnhFqtILgm2++ef/99+fzeZ7nq4L4vCh8y1NcnY8vP/je+2EUWpbZbrf39vZ2dnaurq4UEEHLO7u8DNpd13ZeLaP3PnifEt2xfcOwv/nmm6Dl9brtPEuYkrptN4oRQxcI5E01XF/jnOu22d9Ya5pmuLF+7603/vhP/uTu3btZnkdFhjUKKD46Pen3+67ja0jreG1TM3jDW06QJSlrGs/3kyKP0xQSYBhGWVeWZeVFaliGkKCsma5jBBHnKqtqqmtA8aaCUEnVCERJTZCCqkpzHSEBBGOcaJgoDShSlLVugKapz+bL28OeZI1SIppFvUE/uGGO55N/83v//p33P1jrD1KWWzoOWoOqzvIqZxICAHRqaZpdpFVcZR2/m1f5YjGjGuz3Ag5qDEpXk7ZjZUV1eXHq+O3hZqvI6w6TNi+zshaamyDd1nuers3T7CqOFpHEttuyXQPIPMsYE0LhRipMQNv28jw3TbMocp3q3XbQNOXTw/B8DJ5chu/f2357Z/DW7o0yvGoY9v1AcJAnNZDCdz1AZBjOYcxs1+oM+vNwCQHhUjZVhQBO46zISsSKyrVsotFllmCKqKGXZdk0TRgnWVVXDUcIrfzaWZo6jkcQztOMMaabZtnUtm1DiCjCJtEgAItoWdV1VVU6oVKIFVOp1WpVVUUp1Q0DIsQEXwHedEJlzQATBtUwQrxuTF0nhGAAXcu2dENVjUm1aLlMlhGAsD8YPHjzDdO2irqazKaHh4dpnjGoXN/L81wJ4bpunKUr1o2uaZyxKIoIIUqIJEl6vV6a55Zh2KaFpNIAShdLyMTdWzc5q/O8JFQvqsJ23TTL4iy1HNuwTM00bNuGEEKEKtYwxjwvkBL4vltWeeC1moZThAXnCGMIIeOy3W67tsM5X7nRLd26HI13dnbSNA4X0c7mzqtXr4Tgw+Hw888/7w16aZrOF8sf/OBX9vb35/P5zes3rkbnrCxuX79xeXkxnc92r99+8XwPY4wxHI1G77///mQ2vZpOdnd3l8ulRuja2trFxUWSZ5gQAACQajUhhWH47ltvr1gWQRBAghEleZ57QaCZxmIZKgLDaMmqGkN0enJSluW1a9e+/vrrtMg12xxdXWmatrW1tb29vep1bHc75+fnbT9AACRJcuvWrfF4/Pz5c9/3NU07OTza2trCGF9eXra6nTBafvzpJ8enJ6Pp+OD4aEWXbfsBUAIT4vu+5waUUkqpAqLdCcIwhBCWZQkAchxH13XLslZK9zKO8jyvGVudxl6v1x8OFYRvvvOWYekrUMn9hw++//3vh9FydHVVsGr3xvXd7R3Xdvb29j7/6vM4Tc7Pz18/e4EU8H3vrXfffv+991hdPnr06OnTp2uDwa1bt06Ojp999/Tzzz6TnL//3vc2NjY6nQ4A4LPPPnvx4sV8PmeMlXUFABhfjcaXV367dffhg7ffe7vT7zElDUP3HXdjOFxfX3/48OG9O3f/L//LP/lvfus3N9fXh73BztY2hLAosu2N9XfffgdD5DoWKwuNIIzxb/zar5u6sbe3Z5rm6enp69d7Dx8+bLfb4/H4vbffYWXlt1uW5xZFcfvOHaJRCUC/N+j6rdH5BUIoiqLj4+OqqgghUikI4XwyLfNi99q1g+Mjz/M+/vjj+Xwuufjmm29c1724uEAItYNg9bl0g37+88903SBQkwK02u1erxOHIVJqvpjquh6GoWHpiyQMo2VdVhCAoigQQoSQJIkkkLfv3b28vLy8vLRtO4wjQun6xsYqmr5cLJqqjsMlr+o8zTrttpTyxo0bmqZhjE2iqYZTrMXLyKAaxhgAaJumFAxjzARnQuZNVTa1bpl5Xjq23VQ1IRqXABIMEYIQGoZBMWkqtkK3Iylsz31+cCYNZ7pcYoiAVLARkjX9Xu/mtZuffPLpn/74xydnl+eTRVIyzfY0w0G6zrnM0yINE51qBtUwgbZtb2xtWp6blZXu2lQnmoGArEwdbK11qzwKZyMMakcDA0/f7rguqFEeDkxtt9/rtfytteGwP+h2u7quY0hanq9hbeXwdl03jdKV93pVRZCXhYAE6EYk9S9el/+/X+z93o+//tHPn1fehjm8lktaN9KgRl0yxmSaV6btDYZbmuGGSQopaQ/783AhISKUWrZt2TYydKokty0HYyyA4LKOorCoK8sLINZWbnzAled4yzCeTscd303jEGAgADdNTSkupRBMUICBkEwKqBElSZGzquSSK8G4YJxismpIkVIams7rpipyJQXBCAiRJLFOqdfyiUbbvW5SplKKqiovJiOheNsPer1e0GkVrBpPJ8ej8+cnB9M0opQuF2GjRFzmvV7PsqxpGnEEqppBhSSXOqKdTkcKoSu03R2AhptUq+s6T1PPtZsib1t2Mp+3PBdjxBirGqYQzIus025buiW4ajiHUlFK87zMq7pqSoVVGCe253IpIISKcRNqoGaGpgugFMVNVc8mUyYahVS0XOZRJgr2wdvvnh7vS8Ecy5uPF5tB+8H1XcjEL/3SL6VVRmzNC9qf/vXnd+/c6XXbsmEd12d5Pr26XO8PRqOrZZgMNrZK1hyeHd+4c/PVq1d1XWuY7L3at3U7SZKXr17denBvuLlRlmWv3TGptjYcapaJNP3i4kI0TBLU31w/OTySjK9tbkyjMC8Lv9USQJVlAYU83T+0dKMTtM7PzweDgeHYECPXtgnCeydHL48ODKqFs/npxbnluVG4LOKUUprm2c6N64BiCcEiWrb63Wm4iLLcsOynj5/Ey+jRN99ejS6n44mGiWNaFiGaVMNWq+t7SRxTSldd2LZm1FmhQ+xRo1jGeZIiRJqmmS8WRZkl6ZJJBglo+S4Wom2YyWzGKtbp9ICElmF3Wt1Opzeehp989ov9/UM/cCUQaRpPLsbL8dKg2ubacD6dnJ2dAYB6nf5wfdCwajK5avlty7KGw+HV1dUXX3zBOX/77bffeuPNYX9gGAYAAEJYVKUQwrKsXq8HKapV/emXn9aCUV1L0vLJ4+d5VhtUcwydANVvufFiWtQZE7zb6l+eXmmUuqa9PVxHUiXLsCmr0dEJFqIdeN1u98Htu4NWa3tzaz6fL2fz9eGaGdhhFum69uTbR//5P/yRTc3PPvp0fD7e2t69ms0rwc8uLiSCmqFXeTIfjz3Hhko1TU0pUUrN5/PJZNI0ze76esf3FtGyv7GW5tnNmzc7nVZdl7/xa79ZF7XtBmubW+fHR997542Hbz4oq+bNh281eb0S/eI43nv1UqNKirLXaiVR7LruZDJpeX7L91Y3OZPqOqGzyTjwPMH45OrynTff6rbag0634wVlUUTL5WAw4JwrjLuDAUbUcjxMiZDSNq08ywyN8qpWNWs7gaqZpelVViKpQM3TZQKYQABahr1CBFJKszjxHBdKhSGWtRCMAYMCnQAhKUCCcYwxwVg2DErV7/Sihh9dzVvdQVEUjqEJURINxMnctcz/5ld/o+36n3/+xVffPHl5fHF4OlnmIkxqw+tAQOuizpNUyLqq06yMWN20/a5m+Ej3pWHpQavV9gjkdbpoGdSGkJfFPJwxJAWpTYPf6Dg0mVPJrm9urgetrU77l773zr07t7zAd/1ObzAwDANBBbjQqUapThBVCjCpANYA0fKyooS0e/bxTH15WvzLn77+f/zRz788inNkA6wtowXGMC+Ksmrymr8+OBeCDoZbtt8uedNdH7Z6fUi1vG4qLlCexFmcpEkMIaAaphR7vqWbxjyKaymllJDLNM6AQKZtQQiFEIamSwEQJJJzDKEQAiCIMZZSEvg3IVKia77vE0IMQhXjUEiDalWWW6aplGqqmiBMEO71eo7nDofDVQZ1NL46Pj5smmYWLjr93t27d/v9PiQ4LfLJbLpCf+R5DjBKs6wsyxW80HVdBQExdL8VOIHvuu7qcich0HRdSImEquKMl8yx7DRNg04gpQg8L5qHHT8Y9Dt5nmJMXddfKYNRuGyqGgEIJeRccs51w1jBcHRdRwisYnsrQVMnFEqlIdw0TdNwAAAlBCppUI0SXaO04wffffuNYRhRljecra+vAymPXu+7rnt0clwzlpflxejs2s52nudHhydN09R1HYbh+mAYL6Pd7R2M8cXFRXfQ39reefToyfc+/ODu3btCiN3d3Xm4qJq6N+g/fvx4NpttbW09fvxYSjmbLo6PTzRNwxo9G10wwT/77LN2u721tbV3eHDv3r0yLy7OzlZ2IFM33nv7Hcswz8/PgyAYXV3Ow4VS6vDwkEvRGw7KqoJStTx/OByenp52el1KaZnna4NhVuRZUaRp2uq0AUKQYifwf/Kzn3300UcHrw9MTbdte21trdvtdrtdHZNuEGgYGxrd3tiwLQMAIDknGJuGAQGYTCac8yTJZrMF49IwjKoqEFCWoSnBFRe2buiEQi7rvEjDqEwyyIFlOp7tddo923Lb7XaepK6usyy7f/vWgzu3iQSL6Yw11QrLHrRbjNW2bb779jsrDg/R6PWbN2/evLmxsQEA8DwvTdPRaHQ5Hk/nM9u28zyP0+RqMj48PIzicDab5HnKObc0fdjtJeFy2OvrmJRJfHZwVKTZV59/8Ytf/KLf7XVbbSjh5tY64w2Q4s0337x+7Zpp6lkSIYQsXbNMc2trp6lKyXhdVxeXozCO3HZw9/693/md3/n1v/UrSMDFLPz000//0x/+0dnZ2XQ2Mx2bMXZ8fNjqdBRSdV0bhiE5X/wfjHhK6XAwSOKYM5Zk6WQ+GwwGRVHM5/OyLH/yk49WubbXr/Z6g+GTx0+//vrb1cwULucVqzzfSeLlgwf3Om3/4PD12mCIMboaj7d2t7Msi5fLTrsdhiGv6jLN6roWgo1GIwDA08dPLN2IwiUAYHt7u2mai7PzdhAAALIsy/McSkUIubq6EkIopZIkKaoSIZTGCQDANE3dNDRNp5Raum5oJq9XJHellKqqxtDMuqzzvAQKUYyVkEqpJCsQInlaIEQMw6iLXCNYSVg3EBHrfDwFhHIpEFUQSwWEZRkEiDKN37x779e+/0txFP31p58t43Ielw00L2dLLv4mEhxF0dnorG7KNE3n8znBmkYNohlFWRV55fv+oNuxLdIPvK7n9lq+jiVvUigK35C7Q98njIWX67a+2/JAEjuUrA/XqK4hRNpByzEtXTN022UCKAgppRQTy7KapvG9oCzqsmC67dTYCqX50++ufv8vv/zLrw/PC1iQQJltaHqmFwBIqO7kJV8slhom3VaAEIiiiBq67/sKQdQIbpqG4xoEwzTJq6qSgBdVmhQll5DXTcv1kFCSC84lMSyEtYZJDCGFCHFUF4xiWrEmrgsJAVVQFXVVZUI1pkUtQ/NMu+14jmas9B+TaFgpz3M7nY5m6EmWJlkap8kyTfaOD3XT2N3evrl77f4bDxvOZpPp65evTk5Pz87P0zghAAamg4VydaPbaXd6bY3ijuu71PB9Py1yw7ShhJwxx3HSLNva3QnT2PW9drdr6vqwPyjz3AuCoioBUoZh+K67NVwnFOVFWlUVENI0zaasPMtuuYFiXHLxf1gzadWUGFPJVRZHukYkQFGSSQQLVkIIm7Jqeb6OkaXpJtHKOKcIAwQZ5xSC9f4gTHLD9eyWNw1nuu1g076cT7vrQwAJ59LUaFUkBtVu3rw5Wy6hrg83t6aLOSGEIMDqYjjoXl1Nzs5Hg+H606fPv/zyS9d1Ly5H3X7P9txXr15tr21NLievX+/v3rhZK5UUWeD50+nUCfzO2kAw3m210yJ/9vKF4zjjy0uLaAYko9GoYg0AYDQapXm2ubMtoGp3O45h6lTr9Hqn5+eEkN2dnbTICSGyZtubW+eXo4o1N65d/+qLL4BU6+vrSqksy65mk3/3h//hr7/4xTxcGJq+1u+Zms7rBmJUsYYLoRtGzSrfdyXjBOH1wfDGzu6g19/c3Gy1WkJKRRA1LaGA67pKqTKvdKJDDgiHDjZYUTVVHc5CVnOK4J0b113TUKypy3w2m1yenSrWmBrdWl/rm37P9uo8y5KI5fnb9x60bX9yeWUFziJd6tSAHDx+/NhyzFanLYRIksTzvBWv6tWrVxcXF69fv55MJmdnZ4+fPFEQMMaCIGi3Wl23PWz3DcO4cW23Y1s905RFdnpwFM8WySLWkUYU1ol+bXtnOh2HcVgzNl8suoO21/GFUAiS7evXhltr7XZgWdbVbD6eTSGBbstxfY9oGiaa5wbPXr+Ms7Tf7b3/vXff+/C9d99/7x/+/b/37hsPbccyTHM6nrTb3aQor9++gwhhdY0h0nWdQ+W4LgQyjZaU0rpppJSYkK+++aY36EspdV2/fv26aeo7axuba5sVk3cevHX39j2TahRDv+VajsFE7doGr/LpbHzv3r1vv/22qqrOsH96eaHrum3bR2enmmlQSg3DqFljee729rZlWXfu3zu/HOmmFrS845OTNE3feeudLMnaQaepGIWkSPKmaba2tgAAnPPdmzfm0VIiTCyjATwu87xuwiRGFFV1gTGVAhCCqqrQqEERhQoQiBTEiOAiy4e9fp0XOiYrlaauGCFE0wAmoGJKIc332+PxtKgboxVwCqCGESSsYhACjYK6iCwK/sHv/Nbd3d1/+29+8t13B1fzrBI4Z4xDRXSNaIYCpKxZXqR5nsbhuEqjKsl6fsd3OmlSSAxbnQCCpk6mDuY+Em0CXQ1WMpMg3fRhm0c9kG9oyhfMFsLQSBB4nZZvm+baxk4DkGZ7StMAoZgQ1jSsLCyNKsYBwI7jSVZRLCEGSsePjrL/8LOX//Ivn345Bk/H7CpVkyhjSuqWmZdFslhG0+nZ4V6TJ4HrpFFclaVGEDJdL0zjFUzRMZ12u8u5sFyHUooxbgetOI4Nw3L9QCKoEAjjaCUlSylXdtSGM0opF0IBILlwLJtgpOm0ruuiKBaLRRongnFbNyCXJqWO40gppZR1XadpWtZ10dQQozfeeGNzfaOu2NnF5ZMnT168fDmejolGFJB+4DWcRVGUpqlG6LA/QApolErGMUSDbi9Ps26rLYWoynKl2xKN7h0eEELyPK/ruuEiLwvDcqqmFhBkVX1xOZpcjSEAAAClFEFY07Q0y4q6StO0rkuEAICyyDLDsPI8BxBWdc0573Q6rG4k41JKIRiESiEICU6TBCtpaLSpCl3XuVCaaeRlwaWcz+eGYViWdX5x7AZ2VpRVw7Z2thezeeD7Xb9FgJpPZ+F8cXV1ZdpWnKaVYDUXNeN5ni+jxWwxHa6t3b//MIqiTqfz4MGDOEtr1lRNMxqNNtc3Li4uWq3WYDBYhGEQBI7jeJ6n6/rJ+cWr1/v9fl9wbhiGaZrJMjI03dYsx7DXB8MgCDjnnPNOpxVFkWkQ3tTtdtswjLIsHz58eHpwtJwvOu2egjjPy7qo2+12WZYnJyeDwYAJnlXF/vHRX3z00aNHjyaTCRCSYoIxtUynKGtdN03NIoQghHTLpIY5m4frW5tra2uji4vlItxYW3/69Ol0trBcb+WHaXVaiKziF26r1TIMAymgU22F8dnY2PBdtxO0ppPJ+PIKY0wIWczmjm1jhDBQVZbbur7eH26ub4ThnFIazhdpmr733nu6aUAIDc0MvFar1dre3JKcmabeavuLxYIxNp/Pw2iJCAzaLcPUqqLUNW06nU5mszhNBFeO5a71B53e4OTsNAxDQkin0wmXc8uyhBAKIN9v7WxubQzXtrY3qK4hAEzDOD490QwdY7qxte26LmvEeDz+6KOfnJ6cNzWPkwRjHCWJYZhFUVxcjkrGXu29/vTTT58/fz4aXxqGsVzMs2Xsux7C0G8F9x4+8AL/5f7r23fv6LruOM76cE1KwXizPhhCoKq6hgRlSYqkvH/37mg0Qgi5vrdIotcH+0KILMuyoryazrIshxByKfeP9q9f3yUEJfEyyzJKzPOLq+998H6cJovF4t7d+/P5vCiKra3Ng4MD23a6nb6mGdPpPF5G0/GkLIu6ro6Oji5Go263a5rmd98+0iCez+edTqeqiixLGsHLukjzzDQNgjCltKobCSCl+uoBEUIVTa279iKOy6bhUq0mlZU/DWsEYsAEpzqZz+eO5RKEVw2XhGh5nkOKalFDTCHVLMcRCr44OrfaHQ6UBIJLpWnGKnENpQJSJmG4u77+v/zj3zk5PPvDP/yve0cn07Q6nCwWadZwYduuhjUIVVkkRZ5G80WZFLPLKVQqcL35dCEYb7X8dseTvKiKqO1bLc9EoFYqr9Pp7c1OoAkXNfc3ureGrTXPbNvGZr+7sbZuGMZwsK6bVqfTMy2nrlmr3aZUL4qCGrrrumVVe573N5wGAczAF0b7L7+Z/d6Pv/wvX7w+WKoJ00epMNsDarnUMA3N1KnGmno+m1BCLMsoshxVUgqAi7xRCgqhppPQdVqSSQoBL6vlYmZYZlI3szQRSMZp6LhmXmRpkWJK0yJDlKxKohGEFGuYaFwopYASqsgr27YlBK1OWyfUNSzPsuqqqIUYL+bz2aTKUsd2bddtD3qaaZwfnVxdXB6cncdlIYS0LUu3dCZZ4Lk61Xrrw/Xr1/ob65jS0/PzIGgRIW9cu1ZUxfHFKQFwGHR0QAzNYDUbDtayJN0arq/aLPOmdNu+Zhqaoed1czUPJ8uI2paUfNjv6IZTN9yyDEJRLUQllMBQYtUoBiHUEZE1o5QKoAzbUgCUZc05V0ohIDWEBWcKSkSArmGKEKtyx7HDLBEERkksIZCI2F4rCILJ1ajtuTpGBMFO0Hr2+DsTUxbHPddVTPW7g1W7o+tauqlNZ4sbt+7YvpdVZXttYHlunCZ7e3vr/cHo5Ozo6KiqKkpxmsWdoMO5DLq9MEqUgsP+2uNvH0VRtEziMI5Nw6krfn4xElKKpiYQmaYpmKjyUgoFAJzNZjVrFILPnj0zdbq4urR0nObFaDK5c/POycGhp2k6xK9evc6LWmHsBsFyusAAEY0u0jjlzbcvn/34458wJAxDw0J4hKqy1jStbBpIKRMScAk5kFLGSTaZLBmHZd08efY4zdOrydXPPvm4KGuACMK467e6gW/ZBtU10zR7g2GSZ2meEZ0Ylu54dlHlTVP5vlvUFeMcG4RJlmWJ5zsAIy4EaxqTENM2Wp328ekJ1nQ78BTF995+MyuLqiiBhESjW1tbt27cPj05IRi2Aq/faUdxmKRRlCwXcRjl8Xh6NR5f6hQvZ9OyyAxTK6pKInBwcloIcXB0zDEmvjststenp5BqF9MxMjSpwYKVq/TQ8fmFbpmb6xvD3hBBMl/EWNcazsMwMTSbIH1n61o3aE2uro6PT09Ozzut1uHhwcXF6PjkdB7OJ4t5VtVpnlVVtZhNRqOrLCvC6aQui8l8dnR60h50JJQIgTt3bnXbwd27tx3bjJfh2elJWeYSAy9wLUJ3hxt1XnqOS6l+dnYxWN+w/IBDWTUlxBhgMA7n0NCSMu8PB3sH+3Vd93q9i4tLAciNOw8/+/JL07YCr5Ut80F3vcprwNmDe3fPz89Pzs/ffPPN3c2tKssHna5t27PF9O69OwjD8WzSbrduX7uehCGmiENhuVbBStOzFIEQgjzNRFkPWp00zizdkaXQJAYNd22L2lZcN4pirOlVw5REAADLMgAW1EBYg5qOFYI1F1woxlfxeNQ0FUCwAVBqGqBwuVwwXjtB6+XJlcC2plOpmrQsaiEY45yLiou8qKuSZcuUcvHf/+0fvnFn64//66M/+qtf1EYnE3CZJoCJKs0xRBAASzcoJoqrJM6urs6LPKmS7GT/GGm63WphS/M7Xi0bCdj2Wq9jG45FsnypRGHr3IJJm+Q3OmTLo4O2pxHQ7/hr3VbH9X3LcyzbdD2gGVLTdNeGFDMluZIAYYSpZTka1ihBVVN7gRUz+uffXvzLP//im5l8ldAvDi7jRrX660XT5IyZrot1I0nTJEmgVKhupON4ru2wstEINXSdc04p1XWKMMjzPCsKhRETXAlpmeZiGUJKTMsZXV1alpXmGSSYapppOQBBBiQD3DTNsixNnQIALMeSCLjtICuLosjiOJ5O554brDh8CCGl1Hg8Pjs7qYpipaNp1FBKahoBUHZ7bcswNI0gjBsl4iSjptXpDeI4VhI8ffLk9evXr5/vjUajg6NDRDDnHGE8Go2Gw/WLiwud0FVh8dnZ2enpaRzHSqmtrS1EzcUyXhv0NjfXo2UMCa2bvK6LuhZUN6imVU3JGMvzdNXpLoSSjOuEapoGMbIclxJk6sbKMyChKptScQaAlFKWTR30OhLhmomqZo7v1UpESdxq+bxhZV5QikeXF4PBYLlcllmehIu6rpMkqYsSCH50chi0fdOxj09Pzs7O/HZrGUUAwel02u/2wjC8fv36ar1uWVYaxWdnZ6vWSoBgVhTL5fLmzZudTqcoCsdxXNvr94eGbU3mszAMTcOYTCZcijCOVhuU9cFQ13Wl1M2bN6fjydpgoMRqsgKPnz52PFth8vTlK89vzeNlXpVhluzcumW6Tl5XRycnv/8Hv5/lOdFJ0zSaRnSN6LrueQ6ltOasqplpW0IIVjdN09i2Tanu+63FMi7rGmtYgr9pwRVKapphWZbjOFXTAAAE41fnI9swW92Obtklb66mM9u2Pc+TUHqeZ1h6s6IVNc3G5naSJGkS3b9z1zRNYurT5Uwh6Pu+wujemw/LqoII3bh2M/B8xhhAMI3j6zu7omHhfPrNN98IwRQAhmFADMLlMiszz/OyJCYIAgCSJGGMFUWBNBLFqeN5izB69npv//S0amqE8da1XcOxNV1HGK/4a6/39iBCewf7J2dn1LSiJEEIvXr1qsjLi9GlVEDT9dl8IQHo9QetTvvl69cIoZu3bu0dHpRl6fqe5bl+p9vutoqqunHrJhP8+PQkTdPxZHJwcDCbzdIoHl9dxXEMMQYIFkXxD//hP/zwg+8FQVCz6ujkuB14J0cHhqb1Op08zRAhr/f3a8YOj477g0FWxtP5hAuhAIIIMS4RQkzw589e/q2/9au6aT9/9uLBw4fRMr44vTAMi3NONAoAuLy8tB2n2+t9+snPfa+FAarL6uzk5OHDh+Px5WIx81tBVuSdXhcSXNU1xAhjjABMkljX9RWUexHOGGNVU4dRDABACq161sIkFhgCiCHBhmHVNavLUnJBKc3zPM9TSODfwDIR1Awd61qcZ7ppUEoBxFlWIoI1U0uzzHRcRfSLyVwz7bJmECLTdqWCCNPVjtA0bc4lFDycTN57643/6Xd/nVXNP//nP/ri22cNNE7m8bIWszDmTCZRyipGqW57biXYyifteUGcpYDgVUNWqxMYplbniWtgg4JWYEBY2xYEMsUiIs1ys0190tzZ6nUN4mK82el2W23bNC3LopQyITXdVACv/rSY4JwLJRQCSHHFuTRtBxDTaLmvx8t/+Z9++qNPH7+4Kp+O0r/88uVFxBtppZkk1KK6lcSFYTtIIwZUhJWVodG6KC3LYLKEmqolaxQLOoFla5gAU9dhwT3idHpDgZFEqL+2VjWNadh5UWGqSawKVtasQgTlZe44NoTAcU2kAaiDq9kkZ1XB6k63f+vWHcfxsqIOk2y5jJMo5U3l2xbGsKwyQ6O8qaQQhqZXeZFGseM7lNJwsYjDZdXURKPj8ZgzqRD2XP+tB/f/2T/7p2vb65loLpfzuCp023H9YDyZeUErjKPZbDabTBGQSjDe1Lyp0iQxDaNIi04rWD3qjDGq07opOZNZWhZFQRBueT5jLGVFJdjq8CkugFIN4KWoqWZwznVdJ5SWZW4YRiN4WVdY19TqCslqpKCpmyej86QpqWNdzecAINGIyWRiWJbp2kVTEULSNE3TtN1uL2YhQsQwrP2j46IofN/XECYQddcGo+l4fX09SRLOeVlXVVWZplnn9f3b9997593j4+PhcAghLFmdlnm4jDgT7Xb76uoqSZK6rg+Pj+7cuXP71q3Ly0vP8zRd99qtrCyqNE+iOInizc3Nuq5v3rwNKUWYFnGeR8nG7mamambqZq+dCY40Sn3jYjl+OTp6fXV2cHr4eu+lY9meYYGyQVyypuFKcigkQbXgAAG/40spGWsIRUmSnJ6c8Ko+OzoZnZ1T3YjitKwZpdhxLF2niMC8rJKs8LxAStn2fN92KMK+40ZpUje83epqmtE0VVFky+WCM7bylm3u7J6NLqACW+sb5+fngNKDk9PD0zMhRBzHw+F63fCryRgAMJ9O79y6tbGxcXZ2Vtf1xtp6WZYNlyux8eLi4vDwMIoSAJBOaLKM4jheceU0TeNNkyYZY2w8Hr969SqOYyb4qrcdAFCW5apNWym1d3iwd3gQBMHX335zcnn2bO/VyejccuzTk6Omqlag7Fk4Hy+mNVKcoKTMvn78KCmyrMilED/4/i/tbu9QSi+n46+/e3x4ftoAeX51WXFmt4NpFEoheu1OU1bhfJGG0c9+8tNne6+OLs6YFEdHRxjA7fWN3/2H/0jX9dFkDCl5vb8/n89nsxlC6Nq160qAGzduzefz+WysUeT5rmXZAODhcH0+D4uieP/9D6NlkmZLhMXo7MwxreFg7ezsTCAZ9FpRlgftTpTG08Usq6qTs5HjOJZu3Ny5tpyFgecPBoNGsbTOF1nSHQ76g16WJE1Rtj2/5XhpmpasSct889p1t9NKqwxrkGgkLdKKVVxy09ARgBKCqqo4k1CqlteybTeJM0J1rNGm5lBCXvO6YnndzLLU8LyqYUmc6Vg3qFlXjCsgMeYYUUt7vvfKDXpEdzCiRVHUTQmgJBRhjPOqxFSrqmZF2bMo/L/+g//uf/iNNz79ePT//dHPXoZsBp2kRkUpeQUI1JMslwj7vZ4glNhuUlWLxZLVnAK8mMyTRew5tqbDvAhtE0iWBIFWVEsJq84gwLACxfxGCw9wtes717ttC2PH0NfXer2WB0XlWaZkOE0ajbr9/nrTNAhBKaVlOoIDJVQQeIznAEivFRDTPJ1V//HjFx89C5fG7lFkzFNdN9cbbrTaa47bDsOMxFmlacRECHIphaIIY11fVsXOxvB8b7K5uT6dzoVkTSMDy2mqWte1NE40TROcC6BYXVRlTRDGFJVJZpqmRjSMIIEIQxTOlmmRiJayXQdCqBs0ibOrSSgaZlta0dQQct0wdM2EGDDZGKYd5alt27pOkyTaXN9I0zSOY6oZtmklaWZ5flFVQgHX9YVgx2enb99/cDW6XMRRq9sreWM7znK5LPO8LMuVx4522liCpi51RBzDnEdLRXjL98sFoQRrBDEmCNU45zUTgnEMEZeSSZnnZRAECqI4TT3dRBgrgJZR5LYDQmgcRrpGhBCGoZdFVpalpptCSSlBXZYQIstw0iKtqma4PqCUJEmkU9pUrNXqOI5zMTpDRIcACaBa3U6v1x9dXQIEAQBRFGmahhBaLBaapuVpJms2aHc1Qi3LqgWfLcMwia+urhzDfP78uW3bhqabVEMAKgjTPDc1YzQatVqtVZ2slHJzc3Mym47H4+3t7dFkvH1tdzabW44NISzz/Pbt22VZcsb29vYaqcIocz374ZsPzkcX2zev+72u6XqWYQIo0zTWTe3TT37R1LVFiE4oYCJKI4uQTquVs5JolCmJFQdKSSnzPHMtO0kiBJVtGZqm5UmmaXq720KUUEQoxnWeVXXBGKuSxrZdgmCeZINef3Y+2lxfExgu42hrfYNzPhtdAakhBIiurW8O0jRFCOnUWC6XQRCIqhmNrgb97vhktnPzem/Qn12OO52OaVlCiO9///sHe/v9Ye/i4sILgpX3+enTp44fHB8fxnF8PrrIs1LX9dX6NMtKjKFhGEKIumacSce0hRB1XS+XSwyRaVscKNM0m7rRbU0IMR6PTdva3983Vt0ReSaEAADopi2Baqo6yzMu5d7B/vrGBiAoK1K/1Y2iKIqWrVaLVzXA8PXr10EQuK5TNXXD+fr2VpGnURQBqYIgiNPEcZxVXmk6mxmG4ZrWcrm8mE90y+wErSSKx+MxUCqKol/+wQ/+0T/+3cePH5um+Wx//+233y4Zm0/mfrs1m83a7eD69eu6rp+cnFRVRbD26NvHG1tbg8Ha/vOXkjOlRJYnO7t3TNOcjMe//uu/dnR0cHp+9qu//ltffPXl2+++o2la0/B7d65pmhaGoVJquQiF4lG4RAoUdbU8Otzd3S2KwrVtHWvz+VyzjM3NzSLLEUKz2awsS0JwuJwNe30uJCEaxChPYkophlg3DAQhEKiumVLCMB2hlG5YUAEhxApEAzBsu25ZFo5lc0IYkwAgSimUhMsmLUq3FSzmV+eLyCBGGS0sUzeCoGSNSSlECilU10XQ6lRNbVsWkiq8utwZ9P5v//Tv/MlHP/29f//X339/48OHd4lvq4Zhxk3LjKLIxS0FaSOF43tl1USLSFaF7ToSgiRJCIQEAF4VLc+axnm75Qkh6qZwTAQBorDuGAg6tkZKww5ixqZRpGOcpSkA+mk87g62syJJytryfFYXmqZjBGzfF4LVdSkB0ImOEJEUp0mOiP3jz1692Dv6lbfvkTu76UlikXILBZrmCq4QN/RSwSTPlJC2aXHOq6wkEnRN3UFqdnWFEPJdz7ZtQBAxNMxZxzR1BBzHpJQQgge+7xFqSbLZGqwFPSwgrFUeZqOjS89wd9d3TWIACeJlNJvNyipXrApaNoecWpQBjjQECSlqzgBKV2sTCOJl5NkO51wpUVVVVVWLxYIQkiTJdDoFADx59nQyXwwGa5yL8Xja7w8X0bIqyslolCdLDKSGYRqFijGb6Kpm8XTZ9tomMUzNrKuqzLPAsTY2B3VdN02TpKUEuKoaU9cEqxAGVVXqmilrsZyGOiZ5U2BXXxSpHQQapKLihm0AiqqqzLNM1yyIaJwmUkpRNW03wBKIumFMmIYNmWgZtgnweqffanUuLseGpl/b3C6z3HX9tCiTskKavrazUwNZcXb39h0dkVar5ft+UVVVUbhUb6J0djXlXDIFcsaG6xsaNbI42V7bkGW9HE9lzZuyTpbx5vZ2WuReKxAIFHVlGMZkMgnjKC2LRbyEGFKEj/YPVuD1dq/reN7+4aFuGL12p+23u72hZprLPLmYXPT6w8DtjI5Pe34rCReqYU8fPf7sZ5/wMjMxFlWjEV02zLfdfr9fc5ZXZcmbJEsZY6rhZZowVk/mk7QqEMaEkDReYoPotm4ZVp2VomCT89EynHPeQKjWhv14GQkmOl47SdI3330LGBhpuoIYKBEt5t1um1Js2E6n26dEb7e6lmUpyX3TdHRTCGU4ruG4t+/dNTRdI5QaZn9tXQihhDw7Od3Z3t7e3d3a2nIsy9T1VqcNMfruu+8uLkfhMrZMhxCtalielyuYHUJEMikaYRu2Rkie50KIsq65lIZlCqUQgHmaKQSjKEriuMjzKIqKqlxZs4VQTcN1SFVZzy+uzo6Ok7RgUrb73awuJotFVlRZuMwWcyTV9PKqLqvlYp5kyTycX11dLZfLRRKdXY2ytEjzUiAwXS4gwasylrppgiBACC3TRGG0ub5RZ0UULjElR6OzyXLBOf/3f/AH33z19b179z74/oe/9cMfJkny4sULz/NWZz4IgiKvXjx/1Wl3OedcirW1jTwvsjS3XGs6n+im0e33wySSQK2tr58dn7R833e9JEmGw+FXX301n8+H/UG0CGezWVYWBasVBGu9frmMKcL9TtfxXAWBlLIqyuVyiTHmXIzOR6LhomaQybVOz9ZJXaaUIEPTFVdlXmGFPcetWZMWKSIwShMJlCIE6KZEpChKTCmkVECIEMYSqkZSRARE2DSBphGqQYAgQAjhminDdJKqOZzMgGlphLqmUTNeMi4RqqoKSWHZetZkeVMAADDASENlnYss/p//u7/zv/6d9158Nfp//e9/9eX+RHiDiyhbJhGGSlUA1qSp67IsEYAKYq/XBhrJijyKktnlXFSqyZssSoftjk0pkdKiKPBNhAXnTZEnulF3W6BrFi2Sr7vmwPFaXlczgw9//YdOf60AQJnGoqyAYVJKyioreakILBnHyLStQMdOVTLdNCUEumNexc2//+jJ//0P/+rnl0XtDV6NZpdpHQmMZlFaNgwjihRsmkZCxLFe1vz69oYBhBTCshxTM5VSAoK8LIuy1CjVCRVFZRDsGObqSSiralWOk+e5ZlitTvfGrZt5UYRhmOf55fgyTpPFckkI6XTaTdOYtgUp7g16AMKiLBFCeVmVdbVqjicUlWUZhvN2u13X9fnpGSIYANDyfcdxFACu7y0WiyiOwzhiSsZpgjEu00wn1NB0xtjK7b5chL7rBkHg+34URZZlQQV8y/Esc9Dv3ti9EaWJYVtu4OZ1AxG5GJ3xhtV1jTGpylwK0fI9pVTNWd00tm03TSOltC2jabjg0jQtTOhKUtSpZpuWYzlQAdM0q6ZmnFuWlWVZEsUUa5PJ7OjoGCGUl0WS5d1uvyiKfr9/eXmZJMne3h7COC+Lg4ODXq93eXm56seYz+fT8WTVQTNbzCmlGxsbCKGyLLvd7qtXrzzHbbVal5Ox7bmO41yNLlfe59XuRCjJpOj3+1maOo6zWCzCOPJ9P0mSi9GoruuqqrY2NtIkuby8tF13f+9Y182G13VTziZTVtdQws9//tnL588uzk72Xr2uy8qx7BWfnVJqWRZjDGMMIbRdZ6UehmE4vpwGnm9peuB5a4N+0zR1XfYGA8u2Dduah3PJWRIuqqIYDAaB59uapbi4du3a1tZmv98tywIbGsDk6uqqHXh12di2zZV0fA8AkGeZrusr82J/ONA0I4ljKeXNmzc3NjcBgrdv355Op4zzhrFOp+O6LmdsuVweHh6+ePXq6uoqjuOrq6swSoSUtm37vh+GIcSorlf9XwxC3DTNanNeVdVKcK+q6m/KJZoGIWS7Tl3XdVHOZrOTkxMpZZIkUsq6aUzL4lIChJZxvFwuLUOjmFRVpZRaJmFRF2EcpXl+cXYiGReMl2WZ5klZVwAhBWEYLcMoIoRohKZ5BiGsmjrP8zxJq6oqimL1QmVdCSUdz12d7bqqyrJsOA+TOMvzra2tg4ODf/X/+Zef/Ozj7e3tB2+8sbWzNRqd24ZOCZJCzMbz3a1rOjVW6lNRFHXNNE27uLi4c+fOZDKZTuetVmsZRbNw4Qb+KrvuWjarm3t37hqGeXFxoRCM0gQR3HDOOS/zgjEWuF68jLIsgxAG7TYHikkhITBta217sxEcQmgb5mIy1TSNCZlmmVCS6IZp2oHrlUlBCDFta7kIbUNHCK3M8mXVuH4rTdOqqgzD4FwSSKqy5EzWdV01jAtRS6kg0nUTYwoAzOq6t7715NU+R5pAYL5YCAl03SzLWiiIMa7rUgLR6raRgnmaZWnhOg5k9eT0cKcb/NPf/dX7N1o/+q9f/qt/+4eTmM0KdhVmaVYjSIHCVVU1VZVEYVZUAGmu3+l2hlRzJMCW1y4rPh1PPctsO7Zn6LzMN/odz6G+g5tiDtlyvavtdM3tljG00Ns3tu5sr9kaphS//d47frfd6vUNx6Wa3go6AMGqqXXdbBrGmECISAWVhHXTFIwzTKSl7c2z/+2PPvlXP/rJSYrPcrIAAfK8II4z2/YFgIyxtCxJey0jpulZGxu9pmnKtNKxsbOxpRu0gVxoNG+YhjTQyMXZGHCVlsXJfCws3FgIe8Zga62BfBxPn56/ntdxzDKk4+6gq9m6YTtpWRVVRQhVABmWU2RFU9dJkpycHCGgbMvs9TpCCNswXdc1LPvp8xd5km9sbGCMqU6rIjcItmwDIdBq+a1eCzn02eHzWjS9docClIaxQUzLdMuGe34bAJimKTF0q+UQm3DI1tYHt2/esAwdA6hrFmcgTMKqKdI8K8vSNE2EkOCQEtPzPM4rgaQEQic6YBBwoSPEmiKKQtdwJAPUtBTFDAjHNSlEomZ1XWOiZXXtBD7jNSZQ13UJVCO5YZmDwSBKkmWeF7wxbEvTtOPDo267txjP227L0SxWsk6vfzWZEqyVZX1ycuZ4AbZtu9OpBQcANEU5PrvACpq6vn9y9Mb33qkEH8/n1+/dmUYhJBghlKap7/sm0eaTqeW57U5nMZ62LTfLMtO2iUY4lJbnKgQJIevDoYlpL2hv7myfXZx3Op3xeJrHuWs73U4nCpfXdnYP9w8O9vajcEkgAgjGSUFNS7edMF2uX9u0PCuOYwghVOjq4gpC/OGHv/SDH3zfsW3IpY21fBk7plULNk+SouInp+dhHGZlpFl4a2stz2rPbu9uXC+TChPltUwO6u//0vfmsyVQeGdroywyBRAmhkJUAKgTvL2+Jqoyi5YPHr6ZF40ftA3dMg2NEqTreprkn332+eXleGtrq9frJUkyn88JpYTSOE2apqmaBhFSVNVoNIrjeDKdK4iEAhCgVtDOs2LlyNR1k1IdQiyFUhIEQVCxhksBMcIQVlV1ORlHSXx5MYIASKXyouCNWBuse14gASyqMivyGjBJAEcIU6IhVKRplddSgrquNI1WXERZmlWlZusMSM02lVJxmtRACiWTxRIwAQCIkhgJqBrJmOBcMsEBgleTsUKQSzCZLUZXlzVruBDLRSg5J4SkVTEaXxFC+r3e8eHhv/nX//pHP/rRm2+/9X/6O7/TsFIpPp/PNtY2y7xZLJbtVlcIBoAEgs/mU8exTi/OiWa98/b7e6/3d3Z2qK5xoCTCpu3kWUqlujy78B1/bWOoCCCWYTku5xxjzIFqd7tX04mU8r333qlYNZ3PKsGEhmukhIazssjqkgux+iKoZdQNqATL6yqvciaaFbcDY5qmebfVFbVoqprXjW3ohk55U3meRynVqUYpZVJQSiEAvOambimEs7rJOS8a1tTctdzZPNQsJ8nZaBoyCO3AQwhJJinVTdNcxBFAilJcl3ld17puGJoZRZFSChNQ1PMmv/zHv/P9f/b3v395WvyL3/v4Z98cF1r7Mk4ny6WQRDFQVwmCIouLqlTLMJvOIqfd1fyO3u6t7d4crG+kaZrHkWrKwHTSxdwh9UZH86isosV8dAZZ2jHY9Ta64fBds94whY9qqppuu7W1vtbvdCE2qeG4TuBYDkLINA3Ht/MmY0wQTS+FABqBhlFzFng20dFPn8z/+Y+++Ld/9eKzkxpRrC/jtCgbIQFCQLfs40Xyo49//mc/+UnQ7dy9e7flty3TEUIEQRAEXsNZwZq8qgzLNi0bEawZeqfTsVxnhY08OzsLw3mSpZQSohOIEZcMYhQEAcZYN0zDtohmaJpe100URZeXl6Zl3Lh5s9Ntt9vtoihs19E0gzE2mS06vQEAIJzPIUJVVbGmRhC6ttXrdZRgWZbWghuOXZZlFiebw7WOH6RJ5DhOXVZ5llzf2c3zPAzDVqe9ubntOJ6GtYODgyJLdY3YnqsAghhDQhRA4+m0LquqzCnVi6qsqkIoWVVFWZYEYSU5r0rPdV3fo5QqyS1TX84XGCKd0DTJNU0TQmCqCaAQpnmeQwiFYFTThFTtbjcvy6qpgnZgWw6X6tX+q2WyDILAtm3DtpgUAABD09I0bbVaRVVWVTUcDn3fbzhjgisIAACbm5sUk8uLCwAANY29o8Mwib1W8Isvv6hZU6SZpRtBEIzH4xUZriiK5XJ5Y/faZDIxTdO0rbqud3Z20jRdveHj4+MgCCZXV7Ppot/vS8k319ccyzI1cz6fj8fjjz76KCtSTdMc1+JK6rrZ6fc5U4Zh9Pv9cL7QNM2wLalUGIYffPDBP/offtd3XIxQXdcYodlsRiApy9J2fcOyGePddrfbDqSUhCCE0Pb2ditoSyn/7j/4+/cfPlAQmqZ+eXnJOe92uxBCjPFwOCSEeJ7nOe7m5jaEsGmaW7fuHB0cSymBVGmabm5u2rZ9dHhICWm124Zpmobhex4AII5jy7IsywrniziO4zSJ0+TTn/91VuSb21utVmu5XK5iGUopAEBV1lKolaZf17WmaZzzJEkMw1jx/eu6Xl11CSFBEBBCNEoRQp1OhxBSVdWqtVkAJYRoBF+p5FBJqATEOM2zFScLEKxZNiaQNyzLssVioZRa5UiyskAALGazlYSy4t+yuhGME0IawQkhRVFURdE0TdBpZ2XBpTAsUwnZFCWEkBr6YhnmRdE0TavV4pz/lz/+0TfffOXY+offf7/MizCc27adZVmelxjjOIn8wMMYl1Xjuv6gv/aXf/nRL3//B19/9VXT1ELJV3v7pmnqOp0vw63NHUrpxdUl1WmWFpxzwZiUsjfor5jSOzs7X3/5VZFmlmWtFs4AwcV0Nh6Pe+2OEEK3zHa3yxoFNaiUorqGNcx5QzCUQjDOV+dT13UAANZwXuVKCSHECnmfpmlR5eBvEPACQ1QmWVmWGOOaMw6UpmlRmlDTisuyv7b9av+EOm4lpaGZCMCqqiRQXuAriIQQhmFACMuiLopCSVixBhLIeaMhkC2mfdf4n3/3tx/cCH72i+P/5z//o2cnl/MazjORcyKhnheVrhmUUk3TiE7CJKmVupzOo6xM81JKiSmJoihPY6x4k0V1vOh71s2tgSY5S6M6mhoy72j8RlvfMPmHt9Y2HW0rcDZabd90N9c2MaIIQNM0EaSaYayQmZblJHHGWY0wqKoKa3oU5QASZRgVCb47nf/xT79GZV61W31IjFpIDCGk2pevj85ythDqfLFYLBZSsChZTqdTVlQ21SlATdMog6aKXWSLXDZB4IOGReeXfJmAqum4nqzqgd9yAZFZKRiHEDqW7dpOu93udDpKKcbrk5OzyeUVQmg4HHR7PYgQhCrLEk0zlIKT2WI8nbt+sIyj1cgveEMwJAhIxgDn1iqnjLCBDZva+TJN52EWRp5lECCbMun6tkNJlix919vY2FASlWlVZXWeVYHnA8XW1rtVladVhqlWN5IJYOhW0HJ1gxRFQinGmCJENKyZRNOQMinSNG0ZRVnR1EICIKFgnq7pCsqKGZrBlEQGhRRVDTOwThSuqhJjtOKOnZ6edbtdTOnl5SVSQNRNzZpKsDBaxmnCMIjropFCSnl1PqqKkglODH2VmOVFVcZplmWe552enqZpKgFI8xwTOhiuaZpW13Xr/8/UfwXbtp35fdjIY+Y5V9pr55PPjbj3Amig0Q2iGx2AZmg32WawJVdRKlsvZvlV9rurWLJll8oqVtmiRLIkihKDmqnZBMmWSKKBbuQGLu7FjefcE/beZ4eVZx55+GGiYZ+nk2qvUGt94xv/7//9/llOATJSIQj7vrfetUJq540xXdta4IvJWBmtlIoYbza7k6NDDAGlZH5w8OiTT4IkRgidPXs+GxW676w2IY/Oz1688867m3qrrK7bShiNMZ1O9jjm+9OZ1+ZwupewgCAMEYmy/Fd+9dePD04+/Mn7i4vr50+ei15ZAFkYjGZTHkRJklltAkZDzmQjRtlkNN4L04wF9Ozq7ODOUau69WbX9/rs6QVF9OhgPwo4xexzP/fz9+7dIdhP0vgzb3xq7/CoN/6l1988f3GVx9EkTTH0t26ftE3fteL+nfunR8c3ywXG+PbRCXbgnR//uBiPwjj64Q9/6K2dz+eA4Kv18o233szzfMh5t95DTIxxSpkgiIbgiCROrbXT6bTv+2FRCEIIIey6zgHAGJtkhRJSKNkL4Y0vkhwgeHZxPtTo9XptpNJC204jAAnCAxnbAu8B6JXstSKMWmtF2zVlNU0LZL02UvRtXW7jkCNCIMYR49Vm2/b9ttwh5xGAbds3dUcI00I7Yzml5a7GhDkM6651UluhnDZ1XXPOCaOIUWH0zc1NwPn52dPnz59dX1/vHx5kabzbLtu23e12223ZdwIAt9tt5vtHHuCPPvr45OTk6ZMnWZIOuWP379x9fn6+rksU0YvVzbatx9NJU9XH+wfT0fjo4JAQRAMOMIqDcLtcTYuR6gVxYBynFELbi1la7GWjzc06DpOr64XRLg5jZKFqJTBadV0UBt6aATCOMO2EshAqb4VVkALtlVY9p8Rb44DzCHSyK+tdGHIEHPQWOauV6LtGyK7pagcsDojUyjp0frn2PNaQeGAZhXme1m3T9ZLyGACEAEYIEopYEGBKwyTtpIYgQDA0Fm62K6BWv/qFV776hSMGwD/4vbf/h9/75rsvti9ap+moU/RysZSyYxwyDlnAlsslcA44AwmgYQAjbjD0BFBCqENI+Wq91E11+2B/GsVeiHa76atdhORxZE5Y/0pCXk3Tkacn4/3jvb3bJ8f7+/uURSSIIYmiuPAOA22dFAmjqukp8F6ZgAfWYwQgZ2Q6GgMPEWSBQ6TuhbLOe8/DuPfofGEuVt31uq6qqqxW693yydnzy8tLp433fjIZG2eyIjs8Orq6uT579nx1fUMhlr1QSimj9/bn3oP9vf27t+6eHp9MRlOr9W6z8c5tNuurm6tdtQsCdnp6enR0GKfJtip7JRkLptM9zvmjx4+tdywId7sdpTxOc0opcB4BWBSFNaKudoxiBBwEDhgTEh5RTghZb7dlUxfjEQDgcL5/fHh0586dg6PDTvSt6I13Wuvhe2uEGOeZspIw3PayaXsAQCfFbrdTSsVJ6L3vREsIUlIGnHrrjFYYgjAIoLMckyExK41jBjGGSAmBEBpcLhgPf6MQ9MAZY3Xbtgihy8tL4Nzh/gGwrshGURQFQXD37t2qqgZ1khDCguD1118feHt1XS/XK8yolDLP84ODgxdXV1EQjvI8TdN8VHjvHz99UndtkmeMMQghxGixWPRNOxmNq6pq25ZzXte1Mabve0LIkEH80ycDwCjPjVL7hwcOIOvd/YcPyqZq+jZK4nff+0lZVovlOs9zAF2cxVEUYUSdAXmcQOOmxSjA9Dd/489Mi9F4XPzyl76khKqqui6rpm57KSHFddtOZvOmaeI4ppx5CITotBQEkOP9Y++hUPLs7CzN4h/86AfvvPcThJBq5YN7Dwmi0Lu63ImuX96sLi4u7j98MJlM7ty513VdmmdFUTjnpuPJvTt30yLP8tFoNM6yHAH/+PHjvu/zPK/L8ifvvPPqq68OV5ksy1566aXNZvPi6nI635NaK6M9glVVSak458NARUqJEGGMCSEAAHVdE0IGMoGRimLCKeOcO2Pquh5I9MMyxK6qyrIcTSYY42EvWkoJjNNSWqms1g7Apu5k1yEIpRAEUdG0sm1CHmVJ2vc9pww6741Ng8gZa4wZLPZJkiCE0jT1EAohlJCMECUkI9RaizEGzhmlAQCYkjAMhwM+4oFzrmka690wzerqZrfdXl1dPXr06J133s7y+I03XkvTGABQFMXNzY33cDKZvfvuewSz6XQ65CINWQXWWhby4+PjwSRTFNlw3WFBsNmshiUSZ2xd18/PzuI4NsZcLW7m8znGmHMOPYjDCHvQ1w0j1EMwP9jHAavbZvBxQI+iKG6bDhEahmGSJAA4iDwAjlIcBIEwAmHAORddO7zhFGFMUBRFUkoLrNYSQhgxyhGB3ilrEMXWORZEcZ7TMPr46fl4fmCNJxB1XRcEAUJEK8NZKIRgjHVd27UCAmy8w5Rq44XUPAwYw8jLdrN8cHLwf/irf+5zr508Py//0T/7d3/80bOPXpS4OBKQrap6s9n0TeusztNMia6qduPxuKwrYdzRrbvFdKa9E8b3Wjkrve7FbpVH7HB/ksQEGNFsb3R9OePq5f30/ji6N4oLaALXn86Kh7eO7hzOR0mYhawpd7JtvFPQu3Lb5GlilPXeDzZcRpDs+qZptNZIEdpZrxyACFsPrXV3bz2wBtys+vPL7dnFc0jstlnl03yx2ghtMIG7cmVUV60XEYa39ubjLMUU0TgMsiTIUhwGndGQ806bupMUsbZqvQHHh0eyF1HI92aj09PDKOZSdUL11pusyMeT6bauH3/yyYsXLwaYQdf3cRiHPFBGY0oZIV4rb00QMCN61bVRyEXbAKPjiMdJaICPJ4ULApZmJ3fuGe2dg3Ur1uXOE4QZ4SGHBCGK6roMWXC4f1Q3XSeVcy4ImPOScYwgBh56C7QSUgrrFMYAQuA91MqlccyAjyAootBDUHVdIwSiOIrC6XTCEFaddMpyRLQ2Sqk8SpB3fd0iCCfFaFwUHGLdi76XXdfFLAgwefTRx1EQHk/nkzir6/pmubi4fNE0jej64+PjbV1drhY8jnZdc71dI4KxB+dPnjFClFJvv/vjvf09x8l7Tx4BilvRC6OPT04YpRHjFOE0TSGExWg0zotyvblzcIy967rGE5COChYEyIGDyezm6nq6NyMB31TlaDZabpdf/6N/v9qtnzx9DgAKGMXAd02VBOHB9EDUfUDJfJrPJuPpuLh6cR6F/HNvfhoZ96lXXvXarFarTV2mkzGP4/F0UpZlEqXz+cF2U/bakCDU2h7vz6/Oz87OnmGMA84//OBjQClPotXNIsR4fXN9dLi/LSsp5Xw2iQI2ne4Vo9lotv+Dt3+8P5/dvXW6ur4KGLm4vLperWd7B7tdWVXl008+fv78OSEo4DTgtOu6t9566+6t2z95591nT546577xjW8Mto0XL150XSeUuri4iOMYWOe0GUrSIIBIIQLOGaHeOuct43SgqQwzVSsVACCMI6HksLnlECzGI+iBllJq1batMVb20hg77LsbYzAiQRAGCJm6mYRZty73szwhrKnquu0c8NpILaRou2pXAm2VUmEYxnneCal62TWdslZbGzAOtdVSYYwpZ2VZjvKCAOgsCHi0q0rr3KAzIA+csdpaoRQhRGvtHXLGtW2LMKjr8hvf/Prh0fzTn/lUVVWHh8dSmK4WWZYNvJ2uqU+Pj7I87YRcl7s4ja4XV30rDvf2GcZhQIWUNAyu14tiMgLAnZ+fQ4Cn073Hz5732hzeul0KgUPeaAkZMc5qrZMoBsjv6t2mrQ1ygCJPQKuE8bhulAesaaU2ru8a4DTGgDNolLRSME+MMMPcXlmFoHdWJwHvmgYggAjSwDpnZN8hZ9MwRAgorfuua5pms9vyNN00TdkKwpg0lrKgqTvGGAKgLEvKSVs3EEJKaa+k1MITJK3Rzm6rdS9qq0HI4q5qXVP91i+9+Zd//dWEgX/17979b//F7//ut98WtCg7CBzvK+GVg9bEcUgZLre7gCeMJmWtlHbj/cP06BAkEcSeQOV0reUaUxEkeDSLwwgLUV1ffdKWF1Bdv7RPX9tnL8/TWDcFVKcj/ubtyb29tLk+E3XVqxoHGHMklcUUEYyV0Qgh4LxWwmpHMUMGYkCIkDqOcu28sb5pul/8hZ9XAna9rur6nZ/8yGOXF+mrr7/WdxIgRCkdgHz1dkcxsd6lRW6BISE3zlhrdrtdGAeWQBLy4UiByD8/PxtmbkmSbDcbTlme53GaWGuXN4vLFxd900DkvfdDhNvgxIii2FrrgM/TrG/apqkQABjDvm0IIYcHBwThrqwnk8nDhw+nB/OHr7zsARBth/4E9ByGkZI6TRJjDGNMW5OmaRQG2APvbdf33vv1aqW1Rt554ALGKEZ5nuVFSghC0EdhSAgKAqaUDIIAAThwLl99+ZUgipTRxumuaxBCjJCA8aGBqkVLGBZKcs6DINhuNpvVum5bwphHkDLWdd3N1ZX3vu/7XVV+8vTJeDw+PDwkhAypoU+ePBnmgQ74IUByf3/fQ3ByctK2rTHm7t27dV1/8vRJnCYIodVqlebZ5c21kPL09HS9XkspLfCLxcJaO5/via5Lw4gRenR0lCRJHMdNVX34/gdBFL64uozTpNPyJx99gDiu6vpmuciKnBBUNmUURUkUG2OcN6+9/tL+4d6qXE7nI8LJ7Xu333jj9fVmeXx8KPt+oFMdHh3FaaKdRZSEYfjqq6/2beecT+MUY3x8fJglSRJFR0cn0INqvb1zcqyluLm63ixXjJDJaLpcLinFg+p9+/TW9eXVi/OLi8sXA7tqdbMYpJI0z70Hjx49GhCk2aj4zM99GmM8K8ZFmkGMF6vVs2fPDmZ7k/F4tVzO5vO9/Tn2AGjb9/1gq3XOIQyllH3f930/YCC99wOkKAzDwXpkrRVCYAgxhAOtcHC/KKUGvk3TNBjj3WbLKQuCQAlJCBm2VREhiJCyqZVSIY8oZuvVKmBsVzXaWsJ+ClyyxlNKD/bmIWUIIW9t23eDFTgKQ2utNBoiFHIOPRi0ZmNMFEXOWEJISFldVs5YKSVCqCzLruusMUapvm235a4Tqu+lMc45s14vz54+k137L/7FP//xuz++fe/ubrfrWyGlPj8/L0ZZ33cQQq31s7MzZfSde3dXq9XebOKtvXlxGVISBkxrXdd1MR61fbPb7Waz2XCDVEppa86vLzVwg0VysFd1Teucm02nYRxprdfrNWMIIk8praqaswgCEiU5Zhx6EEWBB9pDFwSMEAIBwZhq41bbDSe0KSsCUVc3g+xZdy0k2BgdRREP2HqzxAQ55yDA3kNIMGH40ZPnYZL12mhlZa8wpkLIVnRRyI3SEELOA21kLxoeBVIKABDBVLRCKROGoewFAT5ATlfLV4/nv/krP/+Zh7Ptynzt93/4t/7eP/nosny66EpDNCAeU4yp976pO4Ko6jQEhLO0LPtO2TDLgixrZK+cklI4I6zsqbfTIpvPRvvz8XZ1LqqbdvEY98vI1g+OR7fG8e1x/PmXb/25L33mP/5f/5mv/sKnmHa6UwFj2kjvvfWGUGCd1FYJIeq2Wm9XSCitrVutdlIbTzlgQd/3wNjT09MkTjUALAyAdVr0N1fXAMDNZucsmoznAYuzdAwQ9AT1Wkmto4AnYRBQcrS/RxlSXtSi7E3f9I3FPspizoMkSbeber53hABpyu7s2fNyuzFdJ8vSmc47xRnCGCot4ihQSm426zAMKSab5WpAbGOM9/b2IcTAIwRJEqXHx6cY4KsXl1qqm+srjjG01ipFEBKy75o25aFuFMXce0QwK8v6YDY1uq+b0iOPPMKAql4DgKzVAFgtOyV759xgA22rmiAfxZxHoVASMGIhCAASuzINQ0iwQ9h4QCmO46huG0+QsEpa2XkDOQXIIwQJwSzgleg6ozDDTVcDDEeTSdXUHsNsfxYU2bbcYYgwxpiSqmt5GBwfHk2Lke5FFiXnT55VVYUYfXT2DHNmvTNKYYhevnsfW5/GyXw6gx6st5swib//ox82so2SEDp/cnLCOR+yJuq65pisr26A87vNJk3T49OTTvTC6K7r2ralIVusV8/OLzgPpep29Xb/8CAbFfsHR/sHB5PZaNttGtDN7xzWtnnl0689u3qeTUdhGr334XuI4PF4dHh4iDxAzr384MHh4SEPg/PLS0zpveM7vlcnB/uTcXF1eeMdvH37Thpne9NxHoZ3Dg4OJpOH9196cXUjpAqDeLtZ8YBKKX/3d38XOQuMvrm6bPvm9Natsm4ns/mXfvlXbt26NZtND/f29iezsmkAJu9//Khs6iLPjdaAI89g27ZWm3qzm47GxXT83gfve22x9caYNE2rqhJtZ60d1pIxRFqqLEkHZYwQYq11zmmtGafO27Zt4zCilAIAMEQYIhbwsq4wRLIX1jvCqJaqqWpOqLVWWmUxrETfasU5s9ZsyhJzPj86IDETCLTQ90Y46JSQBGNgXdNUGGPR91JrYwzF5HC+L5XiQQCsgwB0QjgEvfdD8hwC0EPQtq2VOsTUGc95KLTCjAIAkAcJC/I4RQQjRjxAbdsPI4HVatO2PUJot9v803/6O0WRjUYj59z+wUyobjweb8vSeg8QnE4nsu/qamek4pT1fb9cLlUv0iTJkvjD999Ls5iEXBn90QcfdGUTMO6cG3qUg/k+p0y0XcD4yeFRyLgSkgJEIdobj6E1BLgsjJtdKVsJPdnuKg9Qmqaia6w3BtpWdb02yljjoIUAcwYACCn32mKMheq104wRYxQisNfCAAsxgM5yjCimRirrlEc+zPOPnp4RFvW9MMoELJDWhFFEEYYGhGEklFSqjyLmrczTWMoeA2wVTMKiakrC/XSUYmApJs12nen2f/fln/9PvvrGSQCevei/9q33/vCT5UelfbSqzlc7ZZE1mLFgt9tJKTHAu3UnKmOFcxbCMBkd3c73D4RxqlbTIIk8hkrt781GeZ7EAXHC9qUXW91dt8unanM5xiZUNdjdvLSX/JVf/4X/03/4G5+7P7G1gNaxgFjklFEeWcQgYgQy7AlEThlOg+lsrxW6V7bq+ju37724OH/tpfu/8qUvvfnmm/fuvnQwPwTORwELAn54eFgUhVIKU6KHqCOEoyCI4uDi7BlwxmoNIWSE5ml2cHCAADw6PgAIUh5QSrWQDJPlcrnZbM7OzrTWXdumYTTKs+l0enJyXBQZJQh6J6XEGIdhKLqeERKlyWg08tY65wghEJHRaFQU404J7X2UJFEUQW8Zgl4r6F0UB8NdOE9S2fVOG8ZYWTWIMucchsAqNYSHNE0DvYceBIwTBMKATCeTIs2cc13XQegZxRDCgDLRNlEUsTDAGI/HYwR801bWasYYJsR4Z7wJowgAAKHHGHoPjQMegratpVZt36d5Xta11hoioLWGGO3t7/daXa0WBnpMSVVV6/XaGIMpuVku33n7bQwRAvDpJ59QSq+vrwGC2tusyLXW6+UqCcJmW2qptqv1Bx9/hBCaTqfL9er09q2TkxOP4GRvNlwOXn74Eg3o4eFhkeXAOqMUY6zv+6ppZvM551w7e/f+vbZpFosFhFA7nSTR0fGB7LuDg4O9vf0kzzrZfPTxT37j13/pb/yX/0/n9ONnH29F/fHFExRSHnOh+vPz87quQ8YjFoimTXiY5/nJ8TEjFFp3vLd/sDfXQt69fecrv/bromkO9/fuPXxweHy0N5qcHh7NZpM8z3/tl395mqXTfHIw38/z/M0333z15Zf3ZpPJeBwF4Q9+8APKWZqm5+fnjx8/bprGWnt++WJvb2+53iy261t3bj9/8vTw8JBQevHiBaVUCZGn6ZMnT777/e+PJhMtZRbF1W7X9z0GcOivozgcRveDMjMYq5VSQ+vd933XdWmahjwYJhlt3QythnEuSuLBITMQRodzghDCMKGUeohIyClnEKM0TZMk67qurmuttXEWM8YC3rYtIcR7zxgZVs8IpWkcY4hU1589f44RYoPnz5ggCsM4oiEHCFJKh4fGGFutCULDGCCLE4qw0pqFAQBAdJ3WenCtGOut9Xk+ghACgIZboNJisbwhFNTNNghYVVVCiOl4vN1uh0vJZrMZrhfGqCxL0jS9vr7u+66qqr29vZBHTpuu6z7/+c+3oldGD5igNE2bbSnqNktSZ8xisdBab3ZbhBDyYGCCYIy9MxhC6K02PU+iodNP4sxDrK1nQTRMpBCEWush8IAwaqGHBHsEh02LMAwhwdLI3gjjXBRyzpgRMkkSHjIa8raXZ+eXQkqEacSjrmkxxlmW9a0gCA8pvpDgwRzVNE3AeFs3s9HYagM8Mt4hhJRS3sOAsiINFhePXz2Z/7X/6M8/PBnf3FR/8K0f/5N/8+9/+PHZSuOVhJ0nZduzIESU1HXtjR6Px5xQgpk0kMQpS7Lp/gFjQVvVBFpo1dNHHweMvnz/ThyRJKJJiAJioWrk7ub84/cevfPHzfJycfFk+eKTW5P0r/yZX/urv/Wlu7PENzpCICSAEdj1WigtldbGIuZh17StEr21xXg2ncyFEM6pD9/5wdknHwQo6DuDEWMsiNLEedPWpRJdNkotsEHEOaeiqXXbyrL71GtvjEaTME6LYgwhph7Xiy11qLzZNVW7WixfPDurNpssDUVXxXFUjPLJbIwIxBgmSSaNvVmvr64ujeoZwjEPhltnGsWD1nGzWnIaFNmIszCO46pqeq0UBjflulUCU0IRow4zxLQ0EJFOSkxZ03dxFMVR0PZN2VU3N1fb7Xo2GTFKm7pTylirjZHeOoIIxlAIYZVFAOd5WhRJnKWMc0II8D4KQ+8MAZ4x1mqpEJxOp0Waea3SJEIEO4iy0Xi322EEnDZ5GAcQnx6fYAL39/fu3bmDDcQGIu8JwtZaxvizs+eTyYQgfHl+cb1YeAzbth3awzSOx9Pp5fVVVe0mk9Gu3Bwe7V+vlmmeLxaLpqrTNKWUNmWZRdF4NqWcDfpMPh7VXVuWZcj48/OzzWYDnf/J2z/+7Fufvrm5EUKkaTqEe0VJEsdxW1azybRT8ve+9rXl+fU4SpMsDLNIKUU8pB6OsxwS/PT5syePH9/eP/jtr37527//e6/evvOXfvu3jx7e/qN3vnt2fRFnCSHo3r078/ke4/Tk4HB/PGUQ3D48Ppjv3z45zZL0jVdfG0fF7cPThw/uBRR++Qs/L0UnnV/W7fzgBAK8q7avvf7S8vKcGPvK/Zed9O/+6O3NZvOjd9959PQZBNg5d3Nzs7c3XW+WEMJXXnklzwuH0MntW0EQeO/iOPzJ++/uH+1HUXBxcTE/PNibTVaLm+1601R1FAQIgK5p+6ZFAGIIhzzCrusAAIPxboD3DvrSkMpNCMnzfOjuAQDQg77tBqFMa62ltNp4CMI4atsWAZgkKYRISSmE0FJjAAlEbdtWXdcJ4Z3hmHhtsEMBwkgbrex4OgnjiHGurAnjqNVSGQ2cZwAhAKMg9NZZbeIwQgC2fbfYrIXRLAggRFbbIXXEGGOcgxgJIepdaZVGCJVNvdptEaNJFFttQh6EYUhw0LVKKTMej72HP8Umr28+fvSe83K5XHadOHv2DAEYMQ6UiUKOMHj2/NxATwMUxoH1bjzds95op5uuVUoBY/M0u1ouSBQIY/aPjz/88MMiTm8dHZtOREFQVRWNuAYWcvr04uzug7vGmCBKrQHW2igOqnpNGGhFKY30xmPIgcUQMIoYw2RUpKKpYspnxRRSJhFkaeox0VoHlFGEtZBaS8ZYFIRRHAjVKyMDRqEH27pqpYp4Cj1hcTiaFtJICKG3oK67KEw7IZRRnDMrgdFYCid6Jdt2lMVZGhjVU8zGo/1eOcLjthMauN45jdCm3LTrm//VF9/8C198aYzBZmH/xb/78J9/491vvP98oeFWqk5JwghmyEJVdete9RAThlnIAogRj0KaxrxIHXbOy2meX58/B1YdH+1nRYwxjig/mExmeThOGHF2dXPZNTsh+u1mZZrtF146+Q9+7Rd+7fXjmQegBUhDRjAm2DmglEGNg+F030Tjjed/+MGzf/Cv/v2//fb3x7P9NM8YY70xxWi221UMM+SBc24yGgvRbcoNTwJhJQ34fD4fFZOjoyNr7dCoXl3fnD0/3yzXGGIp1PViuVltry9vgLHldrdZrrIkHbxlm9U6ieKu7sqy7NsOAxwEIXQQWR8wHoZhGIYQwu2uCuPo4OCoyMda2bIsMWWUMus95pyGISY0yzKlNCK4lwIgeLNesziUzjjgq6p6cXmJOcuKlHHMA+oAch44B6rtDmEAoAs5R84FjEdBQDGB3vMBOCMlpRxBiAEMCM7jiCJMCIEYIYS6Vigp0yTSskcIcc5l147yzBijtc6TdJQXu92u7YR29urmRmud5zkCYHmzANYzxqx1FxcXyPo0jF575ZWmafq+L7J8eXPDGIuSmDC2t7ef57lzLs/zIstvbm729vaGnVVvHQLAGXt5fVVMxiEP+rajCHtjjdIUE44Igr7I0lu3Tr7+za/zOIKYAET6vo+iYLlcVrtNFoXrm+u2qxfLy8//4qe/+MtfCNOg75swovMi+czpraBpP377RwyghLBX7t35N7//NczJX//P/+9f+uqvF/uz6Wz20fe/d/XDt2++/6NRr45GRZGlEQ9ML/dGk4BgrRXjdLY3TpLk4cOH1gFEMGJ0uVzeunUrzBIchh8/eWqcBZgsl8vr6+sgCnslWcD//G/99nw2C6IEYdr23eLmhmOyvLrGGFOG18slpSTLMiWFNfqt118LGd+bzhhjxrj5fL66WVjnIEZVU8dpstlsVquVtXZblcD9VFUfKGwYEW2kEKKua0qp955QyjmvmsZ7P9wjEYDDdBRjPBT6YQ1YSqm1RghxyoQQUsohPyjP8zAMvbPeWs55URRKa611nqYEEaM0cN5qxSnbLLdN1WqtpRo2RWUcx6Lrh58TBMHwiEaqwfc9hJlUVa2UCoKAMdZ1nTJm6LK99wQib+yA0WcBH6KFEIDKaK112/UI4TTPrxcLjLHWtqubtm27rtntNoeHh/v7+9O9vU70m80GU9I0DUZUKGkduFquPnr8SDkvlBzojFEQd62IkuR6cVPWVZKlxphPPvlk4J7KXlBK+77nYXizXlddxzmfzWZ//KMf8jCw1mohoUey6QJOtRLeW8KwsuangFjjnbFGaStUxCjFeDDUD1tjvRCcUARhVZacc+xBxLi3VkmDCDPGOKMBdFGcOoSjpHj67GLbiFpr4yxjjBPqjJVaeQiGCxAhZGDHjvLRMKWrmnrItwDWN13XS5kkSRQmTd+xMOCcA6vXLz55eDj+7a/+3MNDFiLw3vtXX/u33/2H//IPbnpwUYrLdW08pJxBiJW3m3K3Xm+FUEmce0wU8I4gT6EnKIyDLIk26xvZt/koy/IoTyOGfERxyNHhfJKGoZai3+2q9dJ01eLpo9R1v/jawz/3S5//3MPD0DpmbIhwiDEDANVp8f6m++bT9T/+4yd//9uPfrKFfHZbeAaDrHFoUTcvNqum7ptdgyFKo7jc1YwFJ6enhNLp/lwYu6iqUrS7tv7oyceb3Xq72y1Wy76XdV07DA3HJAkDTPfyEfSI09AoH9DICLu+Wh9P9g+KGYTYSJPRKMYsDTICWchCCsgoyYqiiIvs8NZJGKfj0bTvpdaWUu6sb9pWSWONt9p1nYCE4DDc9p2A3jKCQ76qdmVT1aKFnKTj/Gp104pG6c45nc/mm6pVfRcQrIzBlGJvI45jHhCEGSNS9gTAJIgDGhJEIsrTKMTOMQghhBRjCmgapBSiIk6xB2mQxGFCEebAx5Q6CBAlWkvR92laWAMQQhaY0f7kJ4/e9xAO/i0tlZbq9tFpzkPby7YqEfBJHALgMIac86qpm6bpuu758/MkjBZXS4oIxzSJU6n0nySH6IByzrnWmmMClAHGZFEUUvrKvfvImltHx+Nxsdgub927FWaRQtiTIAzDrq0P9/eSgIcYj5Kwr8tf/41f+o//j/+bk4eTX/vKl//cb/5GGMDQiSMtf+341lvz482jJ3fyAlv90c3Fh9WyAVSB+D/83/8nq8vrh0H8n/7an/6/fOGX76zKxGipRdM3HkHRtJM8W6yvcIgdtnERPru6WDf18+Xi+eKGRFFejHdl/fziPMiSxWa7vF5GYeYIudisP3nxrFNyuVm3vUzz5P7De84Z6Ozp/v44SaCzFxcXu91mt1nlcVxt1rNRgb176f6Do4Pjsqzrqg1ZmCe5UkZ74CiGlJzcuoUpXew227aOkriu6yCIrIeUsKqqlFKYQIhBLzvGedU2Vd0EPNDKAA+hh9ZaxpjUwnoTxoHVmlOqtcaUAACFkMZYjEnXCUxYnGTee60VAA57qDsBkeMxc95XTUM5pZwpoxEiqhMhZWmcIAg5YxQz6EG1K613mBKIofUWEUwYVdYABKGHqhPUo5gHSikHnJQSQhjGUdt3Axai7SXE1HvICB9clVEUdaJ3wDvgKaW97Juu9RAghBihwPmAhQiRq6ub3a5ijPVS9FqhgG/r2gK4XG8QIn0vHaT7J7etdzfLRd8KKwwAgDC6qMogiRGAfVkhjwLG8zxXWpRt7SludL/rGwUAIFQJgQCY7E2lkU71EcUhwAwQ2/s4SD3EreijJEYEMo4ZRQTBmAfEgZjHwEMEMYc4QERLDQEoogRqXaQZATALImy9tyBkoZEuZAnECBNSNq3zUHuHw+hq19YG+oAC4pUW0PlhAowQEkIAaAHUmABjNecBJNxhLpQnhDirMAHWqbZunLFZFGPvpFbKyvF4BKyIkf4Lv/LFr751f45BW4MPHpd/8x/923/34+fPd+L5ory+qaVwUjtIKAui9ap89vSirDsHYDadWEzDLCdxGMRByLgUXVPvGIdRACIOIoYJ8EoJTjFznnmPhXJNjbXUbeV1efcw/9NfeO3Pf+nNTx1PSaeI1KOAon/7w7f/8P0Pn+waEeQyGrt4TILcAmYAoWHoEcyLIityHocOQIRIFEVFPt6ud7vd7unzM2l127a7qmpFizBdrJZ1Xbdt65wZlErO+WQynu7NtNbKml1dSaXbptNKRYyvrldamul4cnBwQAhhLMCIZsU4y0d37twZGKqD7E4gbtsujpMoiinjxrt0NOZRiDGNosRJbaUFAChjwygSUiqllJBZlg2JkZ0UYRhSSr23RVFAhIUyEA5BTIAQUhTZ0It57x2w4/HYaZPwMKQ0CjglKOLs9PQUYHRwMI/SCGGfxEGapt66OEgoJtCagOBpkcu2Lbe7ohh77ymlBLMwjCGEnPNdUwOMoizppUiTxFqPMcYYLxYLDBEhxGg9jAp2u2q5XAIPpTFSKxYEl5eXRVEYrcttdX19naapc05LGUXR6enpeDzWQuZpZqQabvGckffefXecjrqy7Xt5fHy82Ww451LKJ08fJ0k0JGfu7e07Y58/eWqkHo1G/9f/x392/63X//r/7b/4i3/1f3tw7/SjJ4+hB9/5xh8yZf6jP/8XXj089mX9r/6nf/qvfuef/eaf/ZXnH/zgd/+r/0Z+dP1Lp3fizW6s1ISz9975UZiEQuk4jhEku91OKfHi+kVZb9/54N33PnpvsVler5bXy9XXv/Wtdz78cL1ev/LwpTAMnz17tr+3/+TRJ4+ePpPWraudJ6Duu+PTE4RQ33bz/dnt27frthpNCilEnsQPHz48mO/X5e7h3XtOSYxQW9V5ko6KyXw+Z5ggD777g+/3WjkIyrq6uLhYLpdpnoVxBAAIgmDAgTVNM0DGB0D/4BLBGPMoxIwSQgbODMZkgE5THlRVM/TLYRgOLTyEMOQBxjiMAw/derURQvR9N5vNrDUBZ1pIb6zSEhNS1tUAAdVaM8akVFJKjOgAkkzieEA2NU0z+OUH+H6cJowxbx3FhCAqBuSOMcMC+ZDoPTz5fFQM3nYAAGOsbVshBOWMMG49qJuKUko509ZIKdu2HY/HxhhnAcHs+fPn77zzzvxgv24bjxEJOCVsPJqEQTSb7uV50XX9er3e29sLedDWDUEYIIgIibIcALBdb5IwSIJgL8/Xy1W5Le/fuQchhNAPjxJy3tblpBh1TcMpHa68URh3nSh3FQZ40MoghARhHlBOyTBpoJwNb5dWJmBsNhlncdT1DWOMMYIRgM4HNMjirOtEGqeEEAt8JwWCGGMSxhEgeLWrR7P9ru+l0d67rm8Gk+tPZxJdk8aht07KfjKZaGtaJTxBTdMY7cIwhM5zzq3WGBAlNB7YlAhAazhEqtqdTvO//Ge/9ObtPeLApvT/87c/+sf/5pvLHq4UXUlkUCAV0No1dc953HWqE+ryegEpxTwEHnlAxuMp59waVW1XwMmAo/EkG41yyhAmPkvDvXF2fDDVfatkR6CLKW5WV9z0r9+a/9kvffYv/8bnb42pbxSyiPbWC2doGBwfHsRhsF3vvHFtVS8vrxPKjBCA4VL1V5v10/PzuimfPnnc1d315bKp6r5t+rYuskh2fVPVFDME4WwyjeM4SRLZdKpqOKGdltF0hLNgfLSnkG+V8N7M9saj0UgKLWXfipqlMY3DeJzzJFIYXJcbEvI4CIG0ozA1vYx4YK1lnAurWy0NBr3VTVdbLSEAu9U6ZyEHmGBMAOxWW26BrDsnDcU04qEVmhHSNE1W5IiSrhedVNpZiqhz4HKxpHGonHUIhmk23psxxoC3UYgCDgJKgiC4XC49wr3uOrnjOVawB9BMJhMAkFGaAJsEtCzLIAgjEmAPLYDWgSLPszj54P2PlHEE4yLNMKVBFC5XqzAM12XVWjXe35PeEsqtA4PEb41Lk8xB4KDPp1OP4WhvOpqMy7IsiuL09DTP81Fe5HkepzFmhGI8yvPRqOBRgAAUXd80zent29p6zsP5bL5dbV996U3Vaurd/VtHCNpxkW9W2+vFuhJiXTa7bXNxsXr0bPX/+fu/8wxU7yzOvvX+T1w2+vrT59sgkl3vduUXXnn9509fyq9V8PHV//krv/F3/tpfe/p3/4c/t5f9qcPb7XL18frqd3/wzRYZYVQSpwGPIeVVJ9M42W3W5y9eLDZrj+CuqoBzdV1bDL/34x9hBNY319/5w28GhO7WmyiKRC93VXlydJxk6ezoYLFZh0EwLUZ91Tx58uQPvvfdp5eX43QEpVteXQ/Ersvr67pskjABABkPtuWOh4Fz7vLy0nvPGBvqeJZljLGqqiCENzcLAECaZEdHx0NBHAy4ztphchgyPozvhBCcc4QQQohTjjFte0F50EsV8LCpW2/9EP7Qic57hyl23o7GGWc0DuOuE9oq6w3FXEsbBIEwvfXeGocAgh7ykGOKldI/LeLOSWWGZHYW8HxUBFEopEQYW+e2u50yGmI0mNmllJRxD0HTtcPBgwh2wNdtgylRRmNKOtGPJmOIMEY/hZfN53Pv/XCwDQtKdd1a65Mk2+2qJEmePXv2wQcffPWrXz0+PvLeKS3bph4V+Wq56OsaAX//7j2nDMPUabvblX3bK2PKuqYsnM/3iySSTZkGVFf1nfmJqDpsAIWIAD+fTbS1WTGWTZ/w0AFgIVi1VaU7xDEAwCiNrEcAIgCjIIh44ICnESu16KGP0ozSMJ9OlTHWdAiZKAkdBMYqa22cjjpptNZ5mrV9hynBhGCMA0JVJwghxXi0WK6Ncd77stwixorxtBdtWzdG6aaqp+NJQAKK6KSYdKJxSNMAAajH4wkAEDnqDGjKGkEshAYeOWeAswh4Z4EUpm17bwxz8lffevgXf+WtvRhoCR6/kP/Z3/43//j77729qF8s27ZRGIc8TPtOEIgGz6vsTFeLkGZxVFjMs9GMcz7KE6t6q3sSMhqx09sn+0fzYhIp20CkDo+m01khpezrJkLU1C0UbY7lg/3gL/zaZ//UpwpSC7E330fe9dpQiMuq3mzLfDK9f/9+2dQXF+cW+OVuY7Up4mw2GpebFSHk+voGEuyVqsrtOE2bqsaQFGnhIBgWpTDGVmhlbdu2PGKDKaXvOs5oURSc867aKSWMs+PphDJIKJXIV01LIW77jjKGEBocF8hBqQyj1HsfZWkneo9gSAMtlfcuieK+67hHAabldgeMXZ5fEkJmxVhKaa3XUnFCRdcrpbBARVoQQq31nZadksZ7aHSUJhBiqfW0yCGESZ5hjIuiMFpQhgkhJCLL5baYTBwEUrZhGDoEpdQhYaLrEEJhGCPsZK+SJOt226ZrMeWU8DAMF4sFQihK4izLkjB+8vgTq/X04HCz2sRBmMUJRhRznhT5EP4gpcyK0XZXt6IfvHqt6BMSy05FQWiomM9nHsGz8+d3jk4QgM65IAp3i4oRst5uIIS3b9+GyG93S+dNmid1XZflLk+zi/PnB3vzVx7cd1J3ZctwQAihlIzns/efPAri6Hvf+2GD5Plq+Rf/4l9qy4rxWAi/trq+OCsYe/df/zshRLVd/cbnf3G3vEqW7Z95+Y0oy6f56O0//N752UU4zl956YG+uLj693/05m/9JZIU1/V2tnf4vR99lxAirXTaQQpnk5ESvdTKAMgp7urm+uwMAIRC3Bq5Xm+kUQkAXSuePnn+OPhkPps5IerNLo6iOI55Eq3X6xhx6MH1ahGn6XK5pJS+8sorH3744arcUkq//OUvv//++03TFJPxs4tzSulyvdJaD5PSwds+GhVt23a+31Yl8oBSWpblMFrcbjZkmE8a45wbPgLOWiklRMg513cyH2dKKe/84KAY0qysdRBCKWUYhk3TBEGAERZCjEZ5VTVKGoxokoTbquSUQggHUOJut0uiWFnJGIMYAmistRAjDwHwYEibisJYa+29j6JISTl02ZTSIAybpiEUD5zhwXUTRdFwk6CUdl0Xx3HXdcPyLQ+Dvu93Ozu8Uoxx3/cDth5COOwSb7fbNE2/973vBUFweHiYxjG0zlvXt13fduFJyClr2zaNEwjheDQyxmybKgrCJIoJhhhAxhj3OApCCEAxyp6dnxmlozCmlNZ1zRFRXS+ERABb47U2cRg0qmeEN3Vd7E1UL2CcWOApD6rNhgfEY4QARhDrXnFCtVFxmgAjjVUsjBwQaRjVdV3WFQ+Dtm2TPIUYI0LG42kretP1CCGhpLXu+vraAGCMm41nWuteGyEEpZRjHichw6Rr2jBKrLZxFEktCKVWI2M0xniz3YYRJ5wBjJS2YRgaa61H3kJEsLSaBlxJ01VdkuUBAV/+8he//8Gj9x8tEAPfeuf5ex8+//NffPP120dXq2enB7MiDq3pojhBCGihm04CWxGMwpQjTJyHlFIYRcvFqjVuPNlruy4OaZbPxrNicbVwFillJ5NRua6cFUEYIqdt3yPoM0p/6XNvkPlsqpRuu35WTNfLjVIqmxWXy8sojzkPl6t13TYO+DTLlBRZkmRBWDW1x8QCm0SBMTFCJA4j0ao4Ta/XN73U88mU06DxvVJKWFm3bUwCzO1RnDgMLWLbqsriaJQXlFLVi4CyXnYQowCRKIiqtiEYN1WtfTsfTbQyWmsehrumbpRI84xYyzHt+xZi2ErFCHVaNV1NCKEEK+NCTtd1ube3d3V5OZ1OjTFt13EaEITLsj545VWltXIeco6s9has1+s4CKMwiFjIOHfO96qPwzCZjpfLRRgk1AeTGY/ieFvuoMMBzQzws700IuzmxaVxLgzjyxc3xXjSyw7zwDHmENBat6YNWBTEEYRICGFbmbAgDgMj5CQrkjBIWBAG0WqxDoIwDCgjJIwTa1yaJUEchmFslb3/4A6l1PYyQsSF0bYqMYAY4729PdG1UvackfG46Lrm9PTYO9O01auvvry6uaYUb8pydjgezydvv/v9N998AwF0udi8/tqbUTh69Oijk6M5wvoPv/MdGhFXY2EksWrOAvn+lZUy47Rrt44GPmGXjaaz8bfOnp9O8lh1D+/c+4N/+q95GG842nrz7qMPHx4fzzbrA0h/m05WiNY/+KG7c5plcbVTlFKIUR6mRT7ujbrZLBBC1iir3Ww8Ltcbiqj1kCXRo5sLjHHIadu2Xdcd7u/XdX1xdn4039vtdmkcaq1pwOMgPLu+PDo4BIhooRFC2rqvf+tbLOCzvb2+789eXNysln3fy+srSukQfp1l2WKxSNPUElJVlbduAEVQSr1zfd9b44MgcBYwGhBG27aN47Cu6yQv1ut1Ese9EA4ijPFsMum6bnDKay0thMqoATQ9MOaAs3EWG+2c9wBAo2zAeC8swGi32wWcW+27TmAMXd8zQoeNp7KukiRWRg+ICEJI3bYQ4yCKdlUVRZEcthAhRAhpo4E1vVSEEEpo23RBEIRBJJUYXH2MsYFw6YgHHg65jANDDULYtX0Yhn0nwoAPdk9KaVVVYRgOUMyiKL7xjW/cvXV67969IomTJDk4nH/y5MnJrdPHTz7pRD8ajQAAm80mCsPNZgOtS3lwuj/v00RqN53NHj39WHunkEmniT1zkPLlrp6OxpM069rGOSeUpIRM0lw1bc5pU3dRmgLnKWUKuFGeGWWiJEbeOOec1JQCDwzCNuZB14lRMqqqqqrbOIoI5NNRtNoteEwgilUvjbcUwl1VRUk8jJS1dfv7h5v19dnl+UEQiboMCM2ybOE9xAh5DyGI4xAAoK3TxkU8gIZ45yPKgyBYLBaYg970FhhnkfQWI6a1xiQn2GurAIWIIARwnI5pUpimG80Pvzi/NZo/+uCjx+Vmqw34J//Lj783/eC3/vQvwU0juq6IqbYdD7lQJs6zNE6rctfvKin7yd5Y9q0SbjSd8ThBCEJnjDaQhcVsEsbR1dWN7zTs9f7eqOvEYruFjIScU86cMowD1PdSSDXZmy+227woitEoSRLg/SdPHvWyG41Gs9ns5YcvUYRns9mwg8cIDgNWZJlSCmPaNa0xhhAEvSWEWGM2mx2EsCmrvu+893W57ao6IsxKIZuOMfbaa6/dvXtXa3l5deMA2u52EELofMB4VVWMMW1MMR4VRdF0nRACErzd7VjAScjrrnUQLBYLzKgxRihpnPUIGug1cJ2SxWzSKEGScNWUJOC7smy7ziOojHbaYus5IkbrbbmTRtMo6Lp+Pt2bT6aTrPDWib73zkVhGEdR03QhD0+Pb43GeRKGi8UiDiPOw4BHDLOmap8/fx6nqVLKWjfOx1pZoYzzMA7Cpmq892EcEULiON5W5bNnz155+eXj46OmaYwxUgkCkXdOdP3p6XFVVUkUHx0eRiwY53nIgyJJgdFZGjvnooAnQUg8jKMwy9KD/Tkj+Pr6cjweP7h3fzaeeO/TNLXGXL+4ePXVl7WWlOG33nrrzp1bVVUFlL35qbc+/uiDL33uMxMt95wJvT3cnxkrP370Ica4qdqu63SvsPLViyWoZOpZt95yTJz327aWFJo4WFt9Jbt3z8+/8/4H2eFpfnja8fjHV1d6Pv3+1fkHq8W6apgyb03mL3l4cL18KPRrJKRlA4W+e3Ln+OAYA3xyfEwZhhDGSdh2da9kL/VArhdaGGhfLC61d3XbPfrkSSeE1Gq5WR8cHXZS5OPRrVu3rLVBHDngby6vjDGbcvfs7MwBP5pMrm6ulVLPnj8fj8cAgO12O+wrDJ70Aes/2NiH8jdEsA56yCCtDA37z/41SZKyLIMgGGZugxe+rmvOWMB4Xddd24VB4K0btvabrmWcD9YLa621llJqtWvbnjGGMSaEiU5qrSeTEaIky7KfauUYRVGkrVFKMcaKojDOUUohhAO+vCiKwdU+COvW2gFrMyBxhr576NYHKs7wbLU23vsBUzOYcKy1SqnB8ck5xxgPcjMhJEkSIcTwGtu2LbL8+fPnVxfnq9XqxYsXwLqubT959PiNN96AHhzuHzx8+LDruv39/SRJCCFpml5eXAwu23w8woTk49HF5XnVlEma90Lcu3dvQDgYY5brFY+jtumdc6M0CglKk0gpqY3slRzMYIwGEKAhMixiPI3i0WiEEFCyjxgdsrwH0IK1tq3qoigG2QAhFIYhxnicjxhhA6rBOIs5pWF4tVwaD4wxIWHD0iwLA0QJwrjTUmhhrWGMMMYYJiEPMCRltZVaGGcBQpQFGjieRL0ULAiGd9h7DxCTHgscotH+DocqzK527fW6uvvSK2++9Zk79x84SH0SPd+pv/n3/5fvvP/08bK9bu3ltlMeF9N504uzqxceozQvAKVVI4QG+WTmIPUIS2eK8QghNGz2AgBGo/z+g9vzwxlEJkqDyXyc54kSjZWCIN9VJZLK56PZ9XJBQyasBNCNwuzn3/y545Nbm6oMoyCL4zSK0ijuenG9Xi3XC+fcpBhhiAiLlDIRj/qmVVasq1UaxxyHba+2u5oHZDJKsyie5qM4Ym2z42F6cHwShsHli4vlzYJAQhBp+z6IuAPWKEsgibMUYAwwLqt627U4DAAjypo4S3slRdsBACDBMGTLeqcwkFa1sqvaBjFqEBDYL1UrAwwYcQQZAg1FNEuT8XjYjIDSHIymQgjv7eBZTpKEUzaOMw7xOC+M0pOs4IhgiCZZNp9MLz75BHg1GsenR3tZHgdRBBCkBHgg4zSqu2qUjTCAwNsopJgiZ6zYlEfFhDBWdW3f91EQDmO3m5trjPHBybHFPskTSJBz5s6929bbMAzytGi29V6Rz/IMCgN6Pc3y/ek0CIKuE/v7e/fu36GUfvrNNwnCt2/d+sIXvmCd9lplafzgwT2EwKsPH/zmn/4zg1eSUxaH3Bl779bd/fFhW4o33/zczaPHf/b24a8fpp97ZerR9uj06N79l549vXQS2EZzR9pdlxWjICLbeoEx9g4iwuM8n84KgHVvxFZbPzkss9kft81TBs9kl+zNm1ZAHldxepGG7zP3cbs5UO4z2/7lD178YuV/9fDeb37xVz795mdmB0eUBy8urrIwpZQa4BdVpRHiafL48SdOqAkLVF3fvnvrZreEnAdZtpFdpYWQ8uOPP3YI7rrmar1kYSC7vlztsiRbrdbL7Xbv4KCr2stnZ01Vx2livVssl+vNRmm92eycAwCgtu1/lkoxbMMLIQYomNGOYDaEQgylcNCmMaZa22HLCSFkjSEeIusJIb0QiMAkDqMgANpSTK221nsWBK1USTECFhCEKMHWWucRgpRxbFSPEaGMU8acc97Ysiz9TxtrACEYzhittRCiqSpGCIQIAEgpvbm50UJyQgflB6FhV47GcTpUN8aY1tpo6x1AEHdt771njP7J64LGuMFVNZwZQzw9wsRYByAy1nW9oIxDCJ1zQRC0fUco/fDxJ8+fP5/NZpTyalu99ODh6nqRBOFqsfjwg/fzPFNGOQyUM5P5DBCqrX/y7HEQk/MXZz/9CCHGOccIXj1/vr65ro0kWRKNiuvlIhmPWym81VFI67bSTmFCBiMGG2QjjGvZo4jDkJEg6IV2DuVRMZtMeYCqbnvr9DhP4nGRJGnkPTQaRGEcRRHFDHqknbXWEkRDGnoMLXRJPm46FUUBDyhmDCLMCDZSSilpwAGCJCDCNh7put8B4puuHU6LLMswol3dS+28g21b8wAa20cBDhCQVa8Nmx69RvdfKsN5G01Nvmdo0kj76PzKUHZ65/bh7eNsNmsB6CH82rc/+du/90ff/Pi6xsmTxe7i8jrP8zCKqq7Zdc1kugcgRojcLEuHucbEItQZTcOQkSBm0fp6RTHpVT/bH99+6TaLcRgRgv1knHOGVtcvvJNIW2+Mi8PEO7febi6urz5+9Eg7e3l5WZflZFQEjEDnszjmNAgos9ZLraTsh7hx7+AwbddaQwivrl6ESUwJH2yq1lrZdFqaJEn2Do/iUV613Xq9Dhn33ktt4zjO00w7r4yLklQYrY2t69oKNcry4b8NC4G7qpRKYYyNVIv1ShpNKe2bNoljZ22eZkO+ZRAEBgENnPN++CZ7CPq+3+12cRjpvi+yZJRnq9WCMUYRrrY7zmjfNn3XeWMRQvfu3ZG96JqWQNS1LScUIbheL6OQZWm8P51YKdIwCAgtkpQQkmZZPs6FlskoFUb0SjZtxSnhlLaiF0pSistqiyleLBZREFIMIfSvv/769fX1bDLCGDd1PUi0wLkiSy4uzqB3jJA7d26HnK7Xyzu3Tu/evS1Vj4APGPnB97778O4dAsFqtdibTIHzs2L80UcffepTn1rdLK4vr26dHCVJcuvWKYboT33xi3VddV33qddejxjPokSJ8tnZB//sX/5P/+Rf/+5/9d/+Nx8/fVbX9Wa1nuSFlbrIxlXZrKsdZpRz7gFW1innV5uVdirIktYaSQMZp0uKvnv2tELoumyidEyjouPho1584HQ7KyyGYxocaNd+94efB9FX57fq9z967eS4SBLvvTM2DiNtHUCYcN51ghACnHPacEQW1zcY4816eX5+Xu9KCOF6u9k19abcPXv+XEoZx+n9+w+DOIIYQUqyUVE2NaZEWwMQXiwWxpjrxc1utxt6UsaYUuqnFyYph6LGORdCcc6HMjfU00Fkp5QOkvTQwltrtdZDiIQHVirRtB1E3kjVt51Vuqo755xSylvb972HP0U5UkystYSQthcQo6aqo4D3vdTOSyl35Tbg3Ds3BIn8rL8Ow5AQMox/rbVd3Qyny2Ae+9nJNHi9hk1X55y1dvDX/nSZE6FBRh82rYaXaf6EZPmzfdq+7392axma9+FrO6DtjTEIEc5DKfUPfvBDjPHP/dzPffjhh3fu3BluOW+88UbTVA8fPgTAhWGYFcW2rEaT8XQ6vrw4j0KepakzPuTherkiEDFOoiQECHaihxAGQVg2tbU2iYP5bEwJdM5oaygP6rIyyjrtAAAQYwtBGCd120VRVGT5wFzCCEzyTPdCCrHabgBCEU8iHlpr9/b28iQGAARRGKdJxAOjhLN2MMYsF+tWKguRtAZRYoEnhCRhRBGFmO6qJkpjZQQi3jhNKHLAamN6JRljYRharbtOJFGMIfIAGIR2FuLxPN2/7aPpi13vgvzuG5+Nxwe37r+0q7ogig+OT3rtPv+Lv/ypT//cq299jqQTAcHagH/8797+m//w9843QqDk/ccXZ4tNkk+Bx4vFkmLCw3hyeNgZ17S90r6XGhGapimBaDwey64H2i4WCx7xw5PD+XwymRYe2CxNTo4PoTUIWL9eLGPKVdUfHx69/uk3W2Svq+3t01ujON3tdmmeD1jUhHIOKCOsbfvz60uhO0YAochCxMPUKi86mY1HVbOdzkbAWqGMUmY0Gk2nUwNJa8zFetVYAQnWzrbKVH2XpKE2whqAcLBqasCI8jaJ4ggSIi0DqNmVaRRLKQHBvVFKSAxRxIOYBUDogoZEuxBgBlDCgoTwdlsygIAyphPTrEhZzD2xvaQe2l4yRBhjDgJCiDHaO5MnKfFwXIy01tJIYSXmVFtzenw4HuV5khhvi8n47r2H223plGLQFgGdpnFGotCH42zcdWK5XSaTrJRtWGRhEuOQKwphHEil/uQbVVkImq5ttxso1WxU3Fy++Pm33touV+MsoxiX23I8Hisrnpw9/cKf+tLRnTsoRIe3DmgYHBwfHR7tG90fz+dHB4d5mn3us591zgWc51l2cnTYNZVW4qtf+UpZlqPRKAzDIsuvLy+r3W40Gn372380mhTHtw/KaoO1+fwvfqE5GP/B6sW5tXw+/ePHH3zjB38UpRwi572z1pZliQjNxmPEAoLDIMosYSgIeBSygOOIGQSMs5CQFvoGwg7ig3sPpge3SDLqcSiCbMOj5wg/y0N1/0Bm/HQ0mvzkyZP/8m/fW22j5bXv6ySLkyQ52DsgkDppoHSml8po7d1ys23bniHqe4WNPRyNJ3FarbcOQxIH1piDvfnl8xfL65vFYrHYrDei7YEtm9o5J5F/dnNZ1vW2Kr2HXSeKYiyEqqpqqOlaa+RhyAIIkNG2adog4Fpb7+FgGgmCYDQaDXLHQAQDwCEEjDGDXa9peq3VZDZ+7VMv3To5eeXuvU+/+tqn3nrjc7/w2Vc/9XqSplmUIuOBAwBAKZR1fhBDBulgYMEHQUARHrbzuq4bVJeu64bnQDGhmHjrGKFGaYRQEARWadWLPEmdtXEUGa0hAIzSvusHyg0AKAiirusopUPpH57wsKUFIRZioJ8GwyLrwMwZSvmwh/X/b54JGB8YO3EYYYi9sn0n1qvN17/xTWGM9tYgwJLgenWjjOQhW95cx5xdPHsaMHLn3p2iKNqmYQivr1eqlVbrs2fP8ziFHgpnJgdzjphpFTSOYZLFqZYyi8KQIo4BQZAgTCk1xjEaNN1PbftDiEqR5cRbCIywPQ7IKCkiHAQwmBTzvb3jvJgwwmMeM4yV7LQRk9lo19QOQYxASBjQjiEKjKOUOog1wgpa4aV1zlrrtdW9tAbESeEsFFoDhI2zQ0yQMJ22wgOLMcIYQ+vSMAIAAR5tPL1ko3V+9NFWfHhxfevBvfXm5t0f/VA0rVJm//BgNpk645rWX9/UELA0zX77r/yl1z77aQ2RofBi4//rf/DNv/t7f2iyYx/OPnl2o6U7ms4xhGW9W7e7MIuAg1B75Em5q1erlbKKcBIEgVcuD9JqW2nRj4roYD46Pjxy2hKEA8ZREIdhGK6WG4wxRbTclEdHR7JvRdPOx9OI8b7p67rORsVycT1QVghjaZp/8snTm5vlIFl6CBBhUZQIoQhhm82GMXZ0dHR4fMqCcLUr67pRShPKOA8ZYwBB5Xw2Gm/LHYGIEBaFGaZkU5YQwr7rAsrqXYkAjJOkUWJZ75S3QRiGcdT3/QBLYZhQQkTXQw8ghEYqDGCAablcj9Nca922rWhaq3TEA6u0N1YpRSnhIRNKDOuvxiglurZusiJNxwVh1Dh3dHJYNvXQ3B0fH3ddZ7TGGGPst5vVeJTvVisIwGg0StM8iiIacKkF4rhTfSc7KWVd1wAASggCECDAw4Ai7JzJ8zRNU611yDjGeD6f9X07Ho8f3r93dDCfz+ef/sybq9VCyv6tT79RN+WnP/3mdDp+8eL8zq3Tq+tLLfv5bLpZLT2wYcjbqix3u9u3b99/8GCzWh8dHkIIZ7OZlPL1V1/b398/OzvL8zxOwl5219eXk1Hxt/7ef/8P/+ibu2L0Bz98H7MUAqytcQgyzrU1lJIoiZ1z222JEPEQJ0kexRmlPIvTNIi8deM8s730zu3qKs6L6WSvbxTClAdRlucaehNHz7T4seq/VW/OGcSTgnfqJRg87P33/87fG2/r/+Arv3Hn5PjmZrldbYo4HdJkuq6t2yaO0yzOrDBGGm9031bXFy/CMISUrcvdZlsuFgvv/cnJCeWcBEx5SxglEC0Wi/V6HUZxlMRN03zwwQcQwtVqNWjrw6ohAEAIMfxx8HQZY4YGdqiwGONBth64/IMkDSG01g+E0VdeefDWZ95866039vf3jTGc83ExUkI457z3d+7c+YVf+IXPfe5ze3t7WZZ1XVdVNQAQQNh1DYReKIkpQwhoLQeHBsIYIgQAiOMYETwIJkORdc7FcTxQ2ofzZrhzDI81qOpRFA7VfGi0h6Z7iIIa8P2DjWe4DXDOh0NuWLIdCMADFHpo3gfO1zBczfMcY9y2HYTYI4wQwRg/e/bs+fPnmDKE0Gw2Pzk5OTw5JoRILW7fvj2djSGEP/rjH56dnUmhN+sqjtPxeJqmqXNWKgUZifPiw48fG2M4Y7vdbngtCCFKADI6opgCoLoeGgAdHJ62lNI5wAiPeEQQns1m0+kUWDfKctHJPBmFYcgw0UoZrSHyHtjRaMQIDxgXXf/KK68MMXDe+9l4cnVxNZ/Py7rqrMU8GOJxWBgMF0eKCCMEQ6yUg4D2UmPKm74TWkGA0zSvq9YjRHlMo3RTy63wH7/YnDf27fPFe9e7lfI3dfeTjz/GjDrjr66uKaYAQONs3XSz+d56s10ul9q4m+Vi/+j405//+b3jE42xxOjb773463/j7/7P337Px7MX6+bj55et0IRxIURZlgGhSZQOudDW2mq7s0pTShGEjFIAgIOg65o0ibMsOTo9RhgSipH1hkYsnY1xFJV12+9aIhTu5W69Wi5vxnGBjKeICCUhJz2Q59fnYcSVtEU+Y2ESp3mchFHIhe57bfJ0tL93eOvO7STPCEXOuW3fJaNiNptmURSRoFyWTVlhjBHBZds4C+M4xRjWdQkc3J/MOaBpkChtozSrpdxJoUJCikRaAyEsm5oFHGOcpimgWHsXpLHytuwaYfVA9smjxCnNw8BBkKZxHAW9kiTgnhEa8sn+Xq967azUYrlbaWum0+l8Ps8nUxqElIfT6VRZc3hyXDeNR6itmoO9uejq+f4om+QAI8bYw4cP4yKiEVFKxHF8dHyKCGvrCkHrve+a9s7+UYKoVZpyZiFYbVcBZbKVZds8efGs6TsS8sV2eefB/dnhvocAAV+X1Ww0poDMR6OEMVn143SkujaPw7unt7773e++8tqrySh7/PjxfD6Pomg+n9+/e48QfHL71rOz56dHx946Sul6u9mbTNeL5ZAPd7R/sFqsb1br8XTywccfsaywxZ5JZlm+1y67FIVIe0CooUQ5q73TWsymRQAwBUgIiQmbj/dmcQGkQwb4RuSIzeM0tOD1ew+BlNMoPsiLiJBJkVGoxkXosNFJcBXyb2535YN7327r8NX79x48dD959pdmL/0Kn70KeXV5nRVFGqXeAs5Z29dBxKMoIIiITmBElFJJGKm+S7K0kX3dd8pZg2Er5Hq9/v3f//2BboYAVEK2TUMApAAZIbuuU9KMRqO2bVerNUIYANg0bd8JrX5a4ofAFs5C7+DQsFNKB97OkJ83ABcH+UJrfXi4/8UvfvHOndtHR0d1XS8W11L2RyeHjekb3cVhBIzlhLRN/dHjR46gk5OT+Xz+1huffv21Vxnn1hlGgTXSAVi3fds1wDsEASUsShNlDaPEWUMwhQgrpSmmhNAhh9IYK4UYvsMDNstaG8dxEARGaYzx8IR/proMMvVwLPV9/7OdJinlz+QmhOHQ3Q/rUd5DhIhSarBa9n0/0EOHEfSm3LWidwgiyoIg+t73vmeE3N/b++TR48l4TBGGEGOMl7vN0/OzTnYH85lz5uJ6UfZiXTUfPPqYBhhSv2jLq81G9iqkDFHiIMjznDHa973WllOSRmwc0Nh5poxtxTgryt3up1KYNjEPhnwxizEmbC8eZTi8dXq3EYIENEnD4+kk50y7PkholmUhCxiOAhourm9kL4yzYRKLvkMYAoC1h0IrAy3ykGFmvfHIG+d3263peg4RA4zjqO2kARBCHARBECTAYQiJlGDZ9JUnZO/OeUdUftzSPBjtR2nRSL1pOg1I2Smh7Xg0Xa+2wMKrmwVkiIQwyRnEbr1ba2uTPAvD8K1Pf/ZXfuMr0WQMA74R4Gvfeu9v/P3fe28tK54/u9r2nQpwcLy3762rd1sp+/Gk4BglYdS1rTWGMtaKPh4XcZpoaS7OziEGYcpffePl49uHSGvddK3QSmi1Wq2s1Upob4G1tmm6rusYIRhCq02SZ72U43GxXC6TJImjdCBfW6v7vj09uX3v3v08z4f5O4QQQdL2HWG8F2qzWiEAoYN5WjDGjNIE4SJJAQBXl9fOOUIRRqAuq6astNa9FBZ4IxVCaFvuhJKq60XT5nmOEGqqerVcamdX5dZ5Tyh13vdKOufaqmaMDSv7AePSaKU1gBBRYpzdVGWcRk3XYgyDOAqjaP/woG37waswHOBdJ7KsQJTNj4/Hk0mUZCcnp/loAgE3FhfTOcaUUV6MRmVTjyZTZfzF5c31zTIM47punXNJliIA+7bL00wIsVqtivGobVsEoZTy6OhotVlnWUYwOzs7G0KmllfXD27foogShBml+/P54f6Bt25U5EkUXZ1dfPlPfWkoNJ/+7FvT6TRJoizLsjQ5ODjgQRCG4UcffbTZbIx34/H4o48ezWaz7WZX17XopBbaabOrq9nh4dGdu4/Pb/7t17/tJeQ46BqZxoWRRggBPTDGAOiUEpzghAWMMQhcWze77TZgYRrEk7SgABGIZuORlPJo/yAi1Ctx+/DowZ3bWRJPZ5PT27dcwLYQbuPkh03VnuzLo+n7V+cH+Whf+lutXv/Bd75wdOskToi1BEPR9VYbKTpv3Xa7Fb3qOkEwFUJUVdP2vVamb1pj3HK92pQ7CzxjrKyrPM+no3Fb1YMnz2kzzUfeurZtB5PMsI47qNU/zRWAUErdtr2UCiHkIWCM7Xbl4DYZ+tafOUb6vqeUJkkynU611l3Xvbi6zCYjHoWb1Xq9XiutF5v1brery2p5cxNFEeL08fOn6/Vadr1UfRhFp6enX/7yl1//1KtHJ4e/+stffuWVl/M8ZQEdkp42mw1CCGIktarrGmM8dPR13QyEmSFHNMuyYSrws3XTYbNaax0EwWC0D4IgSZKhIg8/fLCLWGsHCQhj7L0dBgzDr2ERdJguDAP/QajhnA83Aw99lCZBlBhjEMbamMl49vu///t927/55pvf++4PgiBCCCllkiQ5Pb1NKV/vts/PX9x7+GBTlb02KGCt6GkUEBaEYWyUGPwtZVNTzpabNcWkSCLonRQthd4qoJSKolhqhQnz0HkEwyCmhKdpun94MJlMhgtHFMTOmCEualg6S5Jkf39/NpttNpvpdFpkSRKFRZaFPBj4PPmoiJKYsiCMkk1Z8SAihCmlEYAMMw9BGEXSSMKoMQbCoVXvprO9TjrhSONYC+IOJ9HBbRmOTTRhk8NaE4V404l1tWNxyNKg7oXSlsdJnBcff/x4tVk3TTdcIu/cv+cRTJKkE/3l1Q0k9PnFi7PLq9sPXs725p5HEuKPr3b/9e98/X/82tf53ulO0JtVu163hEY8SaTUq9WKYBZGSRgl682m7TuhBUReGh2G0WQyNc5eL6898pO9CXLSGqGuL8+M7j0yu7Y0EHXaVY1I89Fqt7pZ3xirsHeqFwENsnQ8Gc+0s9vdWvTdrVsnBwdHB4ennIbrm7X3XhlljHLOSKk4/+m2ZFZMpLKYE6mFcx4DRLSrl5uYc4+AAx4z6rSxUiCEkjQlnHVdFwdhDEhBQ6b9iEUBwKaXfd/nWRaGodaKBWwwnAHvszAWTRvHcaulIwh5wDHRWkdpwiK2XC/G4zFBOMkzY20vxWApq6pqNBqlacYor8rmzumdiAfIIwJJnCTJaFT17cViLTSezG5XjS8rNRrPdrvqcP9kNj0kKJ6MDz3gzpO6kZjwXqrNruqsapTYlbU1Xva9d3BysE9C7p1Tbc8g3q4369Vqu9mkQTTO8s+98dnNzbrIUufcar3dlfVmtxaq/+DDjz5878OIB5dnF6oTQ3Le4yefFOORtuYHP/zjQUDgUfjwlZcRQnGU/tG3v7u3f1g2rfFACHV4eOytK9erl+7f++FPftIquz86WD2/xh5st1sYxNJ6L3WKCbaeQkQIqcsqDkOrNAUOAQugzopMWgcIL/LpbP9AYVAaEaaJdnZx8+Ll+3ew9UD7z3z6C5QnyvggTAHhtBg9rjdPbPNhv4G3JvggV16kbVtcrT+Hk/hyiUXXNaW3OuVhREMtzWg0QgQLrdq23Za1Mi6gHBiLlMXKFkkaBEEQBHXXrjbrs7MzCEAcx6vdptdSafns+dP1euM9oIQ56621fd8PmrIxznsohBpc3kEQKqOHeWmUxg74pmsHV0lVNWEYxnGolEKU3Ll/L80SY3UYhqPJRBi9rRsAQNs0xlnrHeUEEdiIvu67VnRhGHRN7ay5WS+vFter9bJpKgPswfGRFGJ/OvvKV77y+V/4/N37915//fWX798f5ZkbvDiUEEIYY3XbcM6GrSitNaG07To7zF0JGcakGGMPwTDVHPKd+74f9JyyrAa25ZAW/TMn5TBEHWSQYe12QB0MW7iDVDUcDIO2Y4yy2hCInFGDWOSBtd5IY//xP/9nFy8u81Fxdna2WKwo5bpTzx49/fiDj1mcn9y7/+Lmomy2LGEe41rZvjMx4sxBxghA/maxOrlzt7GKcI6cc0LkWVIUGQ6QD0Bl9bpv1nXlMTLe9VJQyoIglFJCD7zQjNDZ7dNsPp6M01EU5kFKEc+nezCMqo1QrS+K4vzyTNrOOqU7gT1Iw7BtGxiws+trHrIgCJq6gxBzzhEAUDnRiarrO2hREnZWWGC8t9gBTnhVGxJNZTBBe/eDW6/Rk1d0drSD8SfXVTI6GI0mEeWc0CBgla7CIsryWGnx9Pzi6YsX9155KcnSPM9Fq8ud6FrrtJtOJ8Lo69VSGx8GyXRy4CF8+Oqrr3/2jeO7t1CAcQi+98HNf/qf/3f/5kdPSHF6vdOfXKxaCSBmwJNWmhc3i0ZIFscOOgDAi7Nzp40FBNGAEDIbjzbLhZECUYKKLBtlOfCeR6EGruwaQLH1XkppoHfQcUJDxhGARunteh1FEaX0+Pjo6OiorVrRaylVXdfj8bhpGuecliaN0uH31tqsyJfbnYUAYRDnMQ34tiqd1fPppO17xthqtSrLUkqR5mkUhqLveRDEWYoJMcZQD7F2FOG+aZ21wPmhFwuiEELIGYMAAPvT1owx5iGwEPAgcM4RRpfrlZByb3/eNM3QmEgpsyxjmADtbac5plpILdWnXnutqirOw1vHtwLG27ZfLJdxmuztHxyenG6qajbfh4hiGk73j777vR94Bx3wPOQnJ8fz+QElvO80hDiIwqppTk5OoihK05SHofHOaKeVublexmGCEHn+/HwynjV1d3bxoqqqsiwno/HZ2VmUxLP53ma3lVo9PXs+2Zvno7GU8uDgoBiPpnszbV0cxx9++OF7771XjEcXly/+4A/+4L333ltvN6vN+vL66hd/8Re/873vllVDKd1tq/c/+PDp06cnR8fvvP3jqm0+evS4rXpkMfVwEmdGOwAxxaSvGgBAHIc8CJIshwbItgtD3ve1kA0AZm8+BQAwhEMWzg73bz98eHLn7nQ2m+5PpRbXi6vFYrHebpIkS6OUUpoU+a5veD7i8/2Pu/rMyUfVdTxNZnH01sFR++77ny2mIyHHhBDjkUfAQWfM9fW1tRp5F8exhzBMkk25IwgHhKqmc1K3Vb3d7Zz3g9tXCEEw7rru/0f0hnBYL/qZTDFo64P0PFiSB+UaIRTH8c8ALJTSAQpICCKMbtbl/v7eyy+/vNlsBurvkMRU1W2QREmWxkmitaaYWOAb0cdp0kvhve+6hhDigNVaW2Awhh999FEQRFEUtW2z3W6+853vKGUQQl3fDvV3PB4nWToMXYe43WHsOXzRhiZdCNE03aCqD+4dhNAgtQ+OdSnlsJsahsHAsBwmCoMWX9f10LMPkKyhjkMI0zQFAEwmk8Em9DPNffBWIoTqunTOCCEg8sP15eBw/4MPPnj69Olms/UOFFluta2rLkuy2XS/Kpu2G04Ruz8/GBShvdm+UZoi7Ixdr9dBEKwXS044gmB4xIgHCPoooHkGlQGUsygItZQYY0ypAx5jOp8fdF1XFMXx8fF2uw2icJivxDx02tV9d+/hSwd7B86Y+Xye5HEch1qJgPG9yXSwzRhr5wf7690WQqiM6YQyxjDCKKYU4yhOKQ+F0gZAHicwiBqL+ORAkPSi0nhy9LwSgmd0clRDFk33ksmsFvLTn/s5iEEcBXmW3nvwYDydLG8WRZyeHh4o2S93m72To9F4mo/GSqm3//iHUsq6bjDEWZZ7760D2+3WGW+tPdg/irP89v2HJE41AT3Gv/v1H/+//s4/OC81yPd2Bl6uqt4AC9H04EAIsdvtgiDgnMdhslltKeXKGM756enpbDYjhCCAfN+31IJmW1dtBygr21JriTGiAbEe1r1Yr7dd0xqrbt0+eeONNyIehox578uywpjEYeIt4JwuVzeccwyxFEZKFQRcKUkY3rY1SwJh9fX1tdZaaGWBdxicL6+s90KpNI4ZgZDTsm+jKMIA7nY7i2FpRG1kgDl2qNMynY2RB5wxqRSipG4aKaXTJuFxwKObxSpMk5vNylrrIJDIV0Z2UozHY4xxtSul6CjDhFLjHXDeaUO1u71/5KQZF5PD/aPlYnVwcrp3sF/udrLp7xzeAgYwzHe7zWJzRUJQTNJklBqMHMaHp6e1qJ5ffcIT3Habptwg551ySppyW2VxdnF2AT2CEPMkbtpO9cob33RtNMo6oTCiHz9+1Bu1f3z0/R+/bZzlQfT2T94z3n3y7KmDIC1GQRg3bburq7ppLADLcruqS+3si+ursiyTJFmu19eLG8aY02az3fbaRFHy4x+/a7yzwK/W20efPK5km89GX/va14qigNDv6l3vTJRkgYdICqiEVgJS7AhKi2hTlwZTRENoIUEQclzrllM0mxRK1KeH8zyODvf233jtjbIsCSHj+SybTx9fned7o3Sc9X374vzsk48+xN4ZL403ouyk9C+s/Ui3/Hi2ko3yilj9ymz6Whx/6eDkLg64BV0rm1YQzBlCFDhGYNc2mAQQc0L5UMvSOGmqhnNuEfAEQQj39+YnR0eMkDyMjZBWK6VU27aTyeT6+rosS4woBFgKzVk4KOmUEu89pVxKrbXVWvMgcM78dETPmDEGYrBZb4tR+vJrr47H43v37k2n06qqhhI/39trm/7ZzbWnlCAKHOx6GSWp8cY6vVkt8jTurBRAp1mcRCGlOMuSm8ubd370TtOViHjG2HqxRsAHAdPWGGNWq7W3XmtlvE3SlAeBB0Aq1XYdwngox1rbMOQQIQ8A49w6J5UKwtBbp4QkCDNCEYTOWq0UJaQqS85YGARaqSSKoyC02sRR5J1L48RYneUpY2y1WnsEt1U5TFAHkyhj7E+M8CBOIsoIwmAQ9yfjcb0rZ+PJ00+evfTgZeiREBIj6hzAhBljxkXBECSAVtsuC5IHt+7cOTxA0FkEgiwTSgPnZ6Oxl9L3ImaBh0BoB4HLgyCGqOBhxgCUDrSaKoehd952Qggpb9++PZ5OGi0gRncPj2yvP3l2sX9yK8lSzkgUhpcX58U0Pb1zcnF9yYMo4MnDBy8HQdA0DUCoFX0SxxB42ckoSqQ1PAz6TtZ1DTFSxkAAkCcMBEpACYKdo5NXP0uOXzZ7d7MHb15K7+L844sXj549SUeFx3DTrRfVzX//O/8jZGS9WTz/5LGTliL2xmufgtpGzs3SWFn1/XffsRhbYJOUf/ozr+6NpzGLqEOTNN/sdkJ2h/M97AB1eHm9SrLJr37lN3/rt/7yq6+/RYMQcvTxTfdf/Hdf+3//o3/x4bIenT40OLzebKuqyrLscP9gvSpFq0bRaH+0r7reKGWMuV6uMKNRmiBIcN/3Sui9ydQ7iBDJklz2fZInm93OAZ9lRVEUJye37t66bbRbrNat6AmhxlilFGFB23XD5y8IgsFpSynVysZpEiax1to5UzclIjCIw65rh8EOIjjJUuMdDXgn2yCKrNMQwrYqKcXZOKvbRluT5/kwypfaKq2VUgyT6WTSt53WuiiKrhVK6yHUeCAvD8MlYRSgmAdB13UIQozQMEYbGFLe+/3Z3uH+wW6zGY1GvewIRbdv3+77Xkg9398PgqCqqgd3H4xGEwfg0fFxWqQOWGvttm7KriOMvvHWW7fv3V4sr3vRWquVNb0UjDFgHUaAUwKcC3mAAWCcQuCgB0Ioo91kNgMY/+qvf/X8/LyXYr6///zi4u13fvzgwUvnF5eLxUII8fzZ+Wq1WqxX2phtWV1cXS5W6/c/+PDps2dVVb04v7y+vHrvvffDOFFKXVxdI8ouLy/ffvvt+w8fGOufPX+ujJsfHpTlVhn9yhuv32w2wqjzFxfr7UYpobW22lCMCISbqkzHxXK1yrKk68X1YgERQYRa7/b254zQ5fUNIeTp06dNVedJ+q1vfUsIcfXi/O0//sHV9WJ6cCiMfn55cXVz3XRtMRm/uL56+uxZVddlXV0slzXDuyi4icIqDa/rum7K3XoVO/uVl17Jy3rsQYggR2QY6BljurqhmGAM27bFGPe9tNZ1XZdEiVKqE33TtYOw/vz58yiKCCFREGptGWNpFO92uyHGZGjGh9HiUJq998MO6p/sx+Cf9bOMsWHqGIUhIuBTb7x+fX0lRF+Xu+XNYmC0KSG6skYe3L17l0d8NBpNZ2NKKaW4KIokik+ODrXWxTiXStV12ZRVVVV1XXed4Jw7b7a7NQCoLqvr6+tqVw6PzjAB1gHnhBBDpN+AqBx86MOVNE3jn72Qn4njP5POfwbM+RMNygzJon3fp2k61OXh8oEQGhgDVVX1fc85g85jAH/mxRwGsMMbNdjq+76HEBCCGWO73W5vb885t9lsv/GNb+ztz5M0X23WQqshyvHqxcX5s6cHewe6U8NhqYX0WiEEuq4RQiCAm6YKI04x6qrSeMMC6I2NWJByzjxEDrRVSSAaQPlBwDxws/352++8s7d/mBWj23fvKKVefvBwOp/3RiGCwjhM0jTO4rZvnp09PT49hZiu11sAkBTCGYshzJNUtA10HmPoANjtdtpaQlEQBRBjhyhP8la5deMkSlw8mT/81JV0CwVANvn+B49+9MHH+WRKObtZLr77/e/94bf+aFfvOtlpZ/7oO9/am0ydc++//34SJdv1TgkJnPPWGa2Pjo60s9PZzEObpHGUhlrrMAwxYfuz/cPDw8Gg5T3EiMdJcXG9fP/jx03bHxweZ9OZZajD4J2n5d/6B//yb//O7zU4yY7uaZo8v1lVncyKEWfh9fWiabpiNGr7vmqb9Xq52+2MMUhp6yEWSlprvTMUopTwPMoIRdO92Xg0OZwfMEK11l0rpTSN7FkSSW0HKlYvOws8ZWwArS2WS+tcHITe+21db8odBLaIQwKcFK2yBlEi+i4KA2eclhpiaIEDhFrvCEAJpZRg523dNQ66KAiNkNb6IIqTJJNSMxpACEXbhZRmcaKExAFrRW+NgT/lnYKuazEECADZ94PRzUoLPIqjdLur6q7X1sQhxwj1UiBGgzQ6un2yqbb5uDg9Pj09PkWEnty7l08mJAgchG9++jMY0yBIimKSZsXJ0WkcpRbAR4+fVmUbR9l879Ah4AjwDG422yxJgTWEAGcFdDKmNGEsjkMAnVE6ipLr1brXpm6bsiwxRH3Xrf+/TP1Xs21Zlh6GTT+XX9vvffz1mfdmZmVlVVdVW7TgXcOIYBAUxWBIDAWlB+lFP0BSBN+EUARoJCAYwRAliIQEkIRAis3urgYaaHR1l81Kn3n98W675c20epi3MnCezo17zo519l5rzDG+8ZnN5vG77/7hH/6rNB4eHtz74x/8eDgcWmMuLy/3Dg9uVstsW0FIi6w+undfadvV3a9+71c54a9fn5AgaKW6ubk5ODgABH/5/Nnp+Zm2QANwdX179+heEqU/++RTbzQkQXxycckYAljNwgsCAAEAAElEQVQrDBVFWuumrRAGnehdUTBWQoJaYDoDi7qptlXfSqvw2cWtxaSV4rMvP0uHw9Pjk9XlJRAq2+SvX598+MmnTS/Kur1Zbz598WpVt2UrqqY929x+evxUhz7d3ftE6pc8vJEah3Fp7MF77+1j8u/88vf8bDNG2geSIUCJR3nk+yEAQLQ1RQYpwxHBGFsItbUAIAxIGg2stTfL2ze5oAhRz+uUanuBMQ645/wGjDFCKKeGJwh3TYchstpoLX2fu+0iBtAqiyGRvaKURnFQZPV4EGfZ6mB/CqDwOD5cLO7tH1Z1k8SpB+H7Dx8NuEeNoQRVVWGlwFp3RQGUbMrKalNus4BxDDCGBGOKMQXAFEVRlw3Dftu2AEEE8Op23bat1lq0rew6RijFWBoN8Bu8BQKgpIQWMeYJoVohASaIMgORlBohYpQVssMESi2Uka6vBxBaAIy1Xd9DhISUUgvCsNTCWKu0tgAiiIGFHuce58BqjIBWilEq+g4j6ORRbmAK/YBi4v4ZRZGQsigKrfVglFzeXK7LbSUaCXWURk1XM0Yev/3IpxQoKUUngQIU9Z30mB9yhoEdJMPxdFLKWkAZRUGappgSIe18d8f3/YDxNAgIBMbqxvSbujQAQAgDj1gtDu89uPvwrb6XZ2cXjZDHVxfpMIUYCmiT6TiIg/nOjtLgN/7Un/YoT8J4OpsbY+eT6f5skfDAQwQjRDCUqotiLwzStmo8HxsrFIDQCwsNc+hd9Oi8g0vr/fj56bY3lZSnZ2dGyp3x5PzV8TQZ7s4Wu9P5eDj2MBetQBaMB8OmE0rb/f2DoirLsry4uf3y9eu8qodeXF6vTN8XVS61WW63WVVIpNdNGY8Gjx8/Hg3GgFAJIfcCHgRt36+3q8P7d+4+fPTtb33vztGDt999f36wowlYNeC3//jV/+E//i/++aeva38ig+GqkXUvDURJmiKPPT87ZnHgBRwTVNVlXmSo6dqmabgfvjFRgoBTNp1Od+e7nHOj7GadKQDargcGuubXMYWVUhYAhCHlLCsLJzQwwEKMvt7mW2sRgFcXl0pIY97ga5wy2XRfy+qM1n4QWAspYlVeAQTrrtZKAamLzdoYgznLqnKbZxhTabSSpu/7KIraomqrmjEGKZbQYo9oCHopIIQUE9l2IfchhBrYXkmlVF23aTpwKTke45eX50EQLBYLbdXp+ckHH3zgnGCzLNPAxoN0Z28fUCqBWa5XjPHIDz0eLBaLO/fuPnn3nZ3dfQjhe++9H8fxsxfPR5OJtkrKPg5DKaWyqlc9piTPc2tM5AdX15dB4GfZ5vj41f7BQVFXJycncRBuVuvpdEq4l+XlfLaz2WwY4/P5vK7rvCg4xdlmFSeDoipVp60EV6eXXdNSTD775HMAEEYUQnx+fo4NyFebn/3sZ8PhkFN+fXn95MmT5Wr9xWefr9fr2WIny8sXr163XSf6ru2qTolNnrWihRBaCLXWlHLRiq6qfd+/XN3mbVNsCy0NRfx2taaEb/Li5OKs1/LZs2ejwXA2m9V1XZVlkZW7u/sQwuVytVjshIN0XeTWgrKoeq1w5H/27Nlpvn3Zd6sw7HdmecD5ID79+UfVzfW37tz/d//snwPrwjMGQQuhbbuubFuAIOecYiz61q0ZAQBCyrpqEABNVTm23zbPIIR7hwebPEMEO+F+27ZOOuT6vjchDABwzhwnBCH0dQvs+tO+b32fI4TKsl4sxnfv3p2NJ9ACq3UUBVmx/eSzjyfjEbBmZ2fn2bOvTk9eU0LiwLdKj8fDpmkIIVEQMkqRBZ7nNU0DMAIUOxI9xjgIAgRJXdfOIbUoCs/zgLGi64fDYdu2LgyWMcY5d804hJBz7njrboXAPU+7zp2z0A/cX+pQdUqpG0GcJ6W11gUYOEa/Q+Hdm0YIcb28c5F0T7SU0uH4bdu61wnD0PkZuAsAANwul2EYlmWJEEKUVG3zO9//vTCJZzsLl8fdNM35+XlRFHEQIKCbpiYMt/2bhx0hpDt1c3U7nk6avrlZ3TRN1XUNJOD88hxgA6mBSDEK+r7zw6CXQhndNE0SD+I4McY8ffr00aNHw+HQQhCnyXg8fvToEeUMc1o1TZbl3/rOL708fp1lmdMZuH6lE/14PPa5E9hroZUfRp2yApBaG+P5dDDbKpLu3zfxaHh0z4TDH3/+dNMp7IeXq9Xp1cW9hw8IoX3f325u/YD//Oc/9yg7Pj6hmDq5rHPWLIrixbPnlNIoiYnnV03HKbu3fwiUvHvnzs7O3nQ+V0aHSfT+N7/JuF8WxenpKef84M5R0/d10/zKr/3qfLHo+v7g6FACO93ZS4aDx+998zf/wl/UlOqArjr4n//Df/n3/sE/WUuGBzuGJZu6W5eFhWA8mXW9rHsVRaPxaCaFQVEQ+37IGC+qZn93MRhEBkGAMNCwWBeEkMF4xMOoU9pYBZSkAPZ1g+wbJlYvhIWAMKq0xhhP5rPVdqO17rtOdX3EvDqrk2AIILMAW2MYJj7z6roBCCCCfBZqYWWrsnVhNOJBuqkrBaCHGdaWWAiAsR4CPhFG1n1jIbAENaK/Xa8ng+F0OGrbVkMAQloBJYFR2gzS1HRiyELPIm1N3tYuWTQOfCul53nOXYQTCow9OjrS2v61v/Y3rLVHBwfOE3U6nVJK3TC1f3g4mo4P9/eHcXywv8s9ul4vz85OHjy4981vf+vy+or7HvH47frWGg21QsAoLe689aCHYNvUPIylsox5CFju4brJT89PWtl6gR9FcZIMuq5DCF7fXo2mIz/0XN7pdrsVVo9nU6P0bDK9ur0xEMzn82y1jrk/SUbr5Wo0GlFKL87OCICzdMghHkXJ3nRerrem60dJsr5dxlGUDodxnPzspx+2dWO1kV2rlLBaQWjn87njzzHmAYsQwB5lwyiByowm4zAMOSJNVpxfXipjttn66uoK+fz15fnuYuH7/s1mtXO4//jx48PdPWQs1HYyHDnKl+/7FCDOGKX05cuXnPPnr4+vmvwPXn7+U1lvp4Oqq776+Of3770VRcO/9av/k187OMry2uNYqQZA6cdRK7XWRgiJ8RvxJEQojuMgCNq+b/veRZoYY8qyzLJsPB5jiNzmsCgqJ9JBBEutXAXvuh4AIJR09EfO+WIxU0pooBEClCBgtbYKUrR7cOB7oWyUbvTR4qjrBBvGew+O4sgTTVl37Xhn5/1v/5KSZnmzGkSps06RxiptkyhVQkOI4+HozltvtVpjj2mty7Isy1Io2Ym+zAsMkYVAGQ0taKq6l8ILfN8POPOapsmyzLkCuHVZVZcAGFe7HTfMMVu+rtRuK+tSLBzW5OxAHCXGATWEEEap+YXjsTNB64VgnBNKm7aN4hgTAiCCCLv3TRlrwBuO/GAwcBfztSeP7PpBnNRFubq5vTq/OHn1OvT8siwhwft37lxdX+i+G/g+lFJqYRlslECIWGEOJjs+YciiIAiEVpxT5uG8ymtRAaoMaoIAMwS2tyutDQCg72VR1Mub1cXJScK9vqmHw+He/gGAqCpb34s8PxgORlEU+77//PkzjNFkMoEQp+lwOJqEaTKaTxCni/3dMIzv3rlPmVd2srFEsASPD1t/ukFhSeMN9G003LadhHYwmhHKP//q85vldWf087OT46sricHLi+Pz1dWd+w+UMqPRaDwdS6msBZQyo0yTVbEf3dzcWGsZJ71ozy/PDNC7Oztnr46bslrf3A6imEH80UcfEYKUUmkU9113eXY+HA8hQb//z7+/3W6DKBpOp7ebrTJ6PBgfHx+fZdnOO0/SvT2pLGbw5x+f/5//03/02z/47KyxfLIbDWfL2xVU1iO+AcEy6zabjtgQOX4oY979+w+01hjTruuNAdra8Xjs/KCVUkmSIEwBAJxSgnAcRUAb0fecMWMUIphwVjb11dWVizTEAGqpyryM49iFIvVtF0VRWZb5tgAGWgC6vpdd31cdNNYFcdRdK7QSWtdl5VHmcW6MyYqCBh73PUfwopQqaD3fJ4zeXN+6GdQY49z1KCZNVWMDGEAUEwyR7/tlmVNKb29uOH3TxzlyxWAwePbsGWPs9PhkNBju7+9jjBezOUJoPp+Px+PBYOAFPmce5x4lTHby6ODObLZ45xvvS2O5F6SDAWFeHKcYYIZp4IVayyDyAcSYcotw3XdVXXddRwjrhNIQLPZ2ZdeJrldKPX7nXQuBiwE6Pz//xS6LUIqt0j7jNzc3SpnpaByGYVUWk9E4CSOPMu57Qkmp9XQ+T9O0ruu+bY1Sq5vbnZ2dJEm22+12u92s1tlmQymZzac/+dlPb25uoLUcE859COFmsymKEkMUMs+nzBjFPc8oTRCQbUsQtBpYawlCjCDK2M7u7mabb7b5D374g7ZvEKZN24dR9MEHH4i6Pdrbl0LEYdRWNTTW9YZN02DCTs5Oz26uTm+vdBptR/H/+OLpl8vbB994r2rqj//4T6qb6//g3/7b90POhQwpsUY5U3KIESEEIGutxYRYa9frNfnFlxMcMcZGo9HR0VGSJEEQuHt1OEwdad1VIidT4vzNR+8sbY0xV1dXnPM4DBCGhNG+F0bZu4d3ESJRkown0zt37nmMjwbDvu1ubm5ur67v37/vmIgvnj6DEHJKsyyrqsoLfK1123dlWUZRJITI8/zVq1fOIiaKojiMfsEvVKEfAGNdAXW6UJeR5JjsFGFXsjnnZVk2TZOmCaO0LEtCSBSGTrhklXZX4pg/zpGxbduvrSv/dWq/22Q4C0l31Dme5XA4dIrHIAiceY5Syvd99wpt2zp3BM/ztttt13VRFH3NNXIq2bZt27adTCZu2RaG4Wg0ury+fPjwPqUUWJ2mw9FocHNzwxgrioxgCIy+ubqhlPrMT8LEatC32mhQFMUbd0yjtQKMEo9xpQzjXifE/sHBr3z3e8Ph0BhgDVivtwcHR4PBqCzL6XS+3ebD4Vgp8/Dho+FwpJSRUgZBkGUZYTQvC8zZcrls62a7XnPKlbR1rzUNbhvz2cX6+aq+rM1Hz8+usvZ0tX56fJy3zfnFpda2qBqhFCKMcKaMffDkbcRo3TTznT3M2WqzaZpmPBwVRXH//v1ROtiu1oBiS1BXN5zQ9XrdNA00Vis1m83u3ntw5+huU9YeZevbZVVVFsHxfLZ3uFeW+eO3Hj64e6+p65fPX33/+/+s7NvBYrZ7eLR/cDcIozBN//Sf/wvf/LVf3z+6LzFqDPxvfucn/8V/+zu//Uc/XwtI4lktQFkJTngaD9qml1Iiz2NxHEdJ2DQNRb7s7Hy2hzAumwpgCDHAAIK+111b96KHGFIqhCjL0mjpUeox3nVdJ0QneoiQz7jHONBmPB5zzi2Bne68ACceSTxe1bWBcDoaJ2EkhPb9KPD9OOTAakwAsAJASThTAELKlpttWZay60POqmzrEdzXlRCdlH0QBITRm2wbpLHzGMBCq7IBSnmE2FZSiDlnq9XSCkMN5owpLeI4rNsKIVSXVRJGSRxnWRYEwf7uHrTg/OJytVwvprMkjAilQsr1ZhNF0SAZL+aHnp8OxztRONxfHL339ns+De4/fPvo4aM/9ef/0mqdx8EAGlrlPcLUQFQ13euXx0qZ4XRksen6EiFgLJhMd6XBR0d3h2E4igKD4E8/+Wg4nVBO1usVYzQvsm2+9eIwz3PZdov5XFrAwyCJ4u317Xg8zsv8k88/S0fDXqhNUQoEP3vxfF2XYZpsNznBLB0Nb5a3LAq2dTmdT5WSVbY92tu9Wl0Lq6fTKaMekFbUPYSYecH+zp6o+wAhDyGhegkUpiSNk4gxjxI/DAAmcRhACywEeVMFvr9aLkng/fjjDz/8+KNtkb8+Of6jP/mj0XjACBqNhhACCiFDCGGotXJpk50Uy+2mkKJG4NOry2OruqPDD2+vf/f7v/3wYB7F/Mk7j/7Xf/Nvdet64ieTeJj6IbHQGA0QkEr5oedgGQ2A0NKt7jHGzqp3vV77nC1m07auKEZJkrhoOrf3Qwgpo5uulVoBBCnFXdd0XW+MGg6HbdcURW6MUsAQzu8f3ZskI4/5CNPOylb3F1fn2+UqJjxE1I/iT7/4st6WniXDeOgxP06Hg8m0rtq6bhnzkmQAIN7mRVf3gzBNWNAXdcT9qqqk1EEQKWWCIKqaWhntkBahlBcE2thtXnR976oqRZhRyihljBFCHNFzMBg0Ze3YAaHnO1BFKQUgxL846hzH0WEplFL3jduOuh2s+6c7HTHGm82GEMJ8TwNrIIAEK2ta0RdVbQAkjHPmOdp7EAQIIcendIwJR58nhLx8+fL169ez2SyOotVyuVmvJuPh81fPG9lqiADERtnYC5DSaRhQCiGFXhzsHRxAi9uyY9DziUchRoq0pfbIACqEDbDWdqIzFnaiZwFHHBdtHSYJpn46nH7w7e8gSgajeDBKwzAej6ej0YhR7+J8iZE/my3iOG6aJkkiysje3l6ZFaN0NE6GqRf1RZtEKffC2219ue16HJ9vmhb5D558M54sJodHyWJuCWV+IIVejOeEMK2tRz3f9zdZsS0rSLBQspa9QmBvb6/ve0L56+NTCOF4Nv32r/4y9rkxRvbi3Xff9X3/+dNnl2fnx2enV6vbT774EmACWrG9uiKE3GzXLAp44BujlleXPkL39g+/8/4Hw3SgGc5kf3yz2lQdtXjix33R/Knf+NO/9W/+W3/5b/4bljJNwOfH23/w3//07/3j73+1EnR2MJjvbW+WoKv2FmNhWiR7ZYzJsq1jJjDmCSHW63XkB8YYpQzAyOPc9/0oTbQ1Td9ZaI1RTgB9fX3dtm0SRaLrMUJuWKYeL4oiCOMoiSlnQguCIae4qesoitbZuhaNg/my9QpBgBDUWkNkPUaasqqKknjMjyPqe3lVtm1rlUYA+r5vjXFBxhZBACHAKIyC3Z2FVdr2cj6abLeZRdhikjV1p5Xv+7LrIYScMwuB06BijBGEjLE4jhmhfdP6vp9EsfNvKpvauXBACJfLtejfTLWU8SRJrq6ueiV39vfqri/qCiH05L33jl++VkL6foAQQQR7gf/w4VsB95QUgc8RBhBaP4xWWZ6MRr/z/d8H1mopbm9v33nv3adPn+7s7Lzhq4ah53k3NzfT+azruuvra6XU8+fPy7K8vr7u+z4ZDiBCy826NypIYg1B1bXbPPvy+bOL6ytECeXs+va2bpv1dqutee/9b3DOP/300+VyqZS4vr7Mt5l7Vv0wdIxpzmnXtG3bhkHQN23TNNvNxveYaGpnnlU1tQEWQiulFL1a7OwRnxpkhZJ5nl8ubyyCR3fvEkLu3723v9h59OiRY2c70yvnA9EqYa396IvPLorsom9+//XTPzh59e2/+dcLaG7PzlaffvEbD9/6pdG4uLgpt7mVkhLoCoozUneYQxyH4BfBp2/8+bR2togOYGWMOYAbIXR9fe3aTAdivEmko9QYk6aRMSbLMkJI4HkIoa4Ti73FYjGD1oxHA2t1MhgUVel5Xpqm2AJGuNZ6Mp5NJhNgbJXldV4YYMu6Ho/Hu7MFQ6Stm070GOPD/X3ZdqoXCIC27SmlUisHLtV17Uqta5Yd1M4YS6PY0WMc6p1lbzQiTdNyzn9hj0Ndo+0M3J0Y1c2gDpH/2tncoeRBELi0S/cDvu87owWH1TiY3m0LHGrvxgiH+LvDw0AgtXJCMDdPQAj39/edk5oTqFNKz87O3BGitfYCX4iuKbLY55zTuq6LoojCMAoCay1hNExiTmhXd1LKKEqU0MTC1IuhtAxyIMFiOhtErJei6Vo3eTdNAxEZT2dtJ97/4JsAQWOAVtbzAil1WZaY0q4V9+7de/jwoXPEDMNwMBgMBoPhYNTUbeD5fddFUSS6PvR5U5RKyPFkkXfmOm/vPXpnm5UvXh+vt1nTdsPRJEziOBk8fvT2ZDhO03Q8mYRheH19GwdxEiZSytOL09FkTAhVSm2328F4NJ5N677rhfjoo4/SwSBKk/V2c7NaThZzTMlbTx5zzl++fNlJdX5xBTHo+v7y6ooxhgG0xuztLrbb9enp8fHLV69fvJadLLP8408+q6VK0qHuhGdRwv2f/ujHXz19JiH83q//5t7dR4D5iqEfP736u//gv/u7/49//HJd0+FUIFZJUTQt8mkgO2GMSQZJLdqqr5q+jKOAWog0ENISHtDQr7qmrXMEdeh5nHNLUCN7Ba3neR5lXV6Nwtj0Ukulrcmq0ktjwlldNUVRaWWlNZ0UwyDuipKEGPkw9D0CAUQGEUgYbfqubrq+ESmPPMQuLi6Kpmq1jMdDZTQA4GZ5AxFwWIrsJNCAU8+xKouiSPyAIHR5eRWnQ+SRWncVgCxOhZUIWymlENJgIFQPgEEWrNbbtun3ZruDeMB8XwgJtRkPhn4UW/DGUW9/f380HQ2GCUHAalVXJeH06NEDRBCmOAg8IyU05vriYrYzZx6lGCMAIaZ10+V5TiBIOTd105Zt1XaWIeQRyPnZ1dVynRkD4jD45MMPJ6OB1SaJw88+/4QxRjGB1rx+/dogaCxUvQp40Im+16qu64uLiyzLx7NZK8Uq2x4cHGipAi8cjSaQklWRNV0XRGEcRkWWVU2T19VNnn3y1VfFtoTajJJ0Ok6DkPkBrZs8ij0NJMSoEl0vNQYMWxr5QZJE2ioAIVImDWMSeIaAtqonyUAB02pZFAWwVmtd1e3ZxcWHn3/633//d77/h//i5z/58fLq8vnz58zzMERaa2l1J3qI6SAeZOtMd6JuqxrqdhivFuP/8Hf+h//6w4+vnp3IZ8cHjP/v/71/l1kw8H2KLcKmLStggJTSmbKWTW2Uwr/INK/bFiB0fXtb1/VyuZyMxg/u3eeUOYYMpRwAFLiCQogrgo7MPh6PnTITYYgRIdivKzGZjCEEy+zmm9/9RuDh8TCBVlNKIcTbTb7OCwUtwyQKPIDQarNOgyj0/L5vGSe675oib4qyq2oIYZqmV2cXfd0STqfzue/7QryxXVzM5lYbgCCmpKnqrmkdY70sSweyO54+Z2w0Slw99Tz+CxajJATJrmeYOE/gTvQGWOf21dR9EASe54Vh6Aj7Dr1x+AkEgBLSNI0r684szBhDKA5CX3Qtp4RAZN/YG2tXpt3hAQAIotDFsTpVlDsU3TLWXdvx69eDNMUYR3HcdL3nBdzacRQEjIi+ni/GURJSzglhiHjbTelhT3X9wcHBNl8rIzyOqQHTKFWiJxggqZtSKGO8IERGi6qB2CsakVettnC12QwnQ+ZRL/ANIFEyCuM4juPBYLDaLAfj8OjOjhBdEESj0WS5XBPCOPfffvBWHMQAo9F8yjFOfT5N0tcvXxhC3nrnnelg9Ojw8OTl82EcEos3q01RlXEctmVflx33AkDxZGd67969t44e7k924zh8662HHmGjeFg3XZgOmq7VwFaiy+oyjeLTF6+yqkxnE4XAv/rRn8DQ++LFM9m3HGIt5Hxv93R9k6vWYIuM3lzdhAhfnJ3WXRNPRjyJXhyfjMfT+3t33j6611Q1IdQjFBqzvjwbRh5ndLneNNLu3X34G3/mLwzmez2CmQK/97PTv/MP/sn//ff/8FzCygYChEgBhRn1GBdCOIhGA0s5gxBCY40xQot1tkUEB3HUNI2ypmxqbYyQEiEUcM9hhW7t7owp2r7b5tntaqmt8b0AU9ZK00vBCBGyo28SCHsheoBRUVd93xOIMSYIYas0g3g0GmFKIUbr7cYYE8dxEsd5nnddV5UlpVRL5WbYruswRE5a7XuhUqqsG0SYQy3d8Bv5ASdUdH3oB0AbCKGWajQYXlxcSCmHSdq2LQ/8LM+NMXmeO7zYIbyU4qoqCCH7+7sIga4peeC1fZfG0d2jO+dnJ4xiQtFoNPIDniRRkW0mw5FRervNlVKEe0qZvumdKzfn/OzsbDad9H1fNc3O3p4xoCzLruuQBVabL7/8Ukr5ve997+r2xo/CwWBwdXW1f3Q4mk7ysrh79y4NPG3Mzs5O3/dKSN/3m65N07RTwkJwfXuzWq1Go9Hu7q77XC6urqqmSZIESI0B1FqvNxsAIYGo2GZWaWttOhwkgxQBwCllmFhrESEIA0chd43wYrHQWkOMq6rihLp5qOs6oWQQhXlTteKN0+Hu7q5SClNiIUAISa17KaqmoRATgOuqulzeHG+XaHfWHiyu0+BZmWlKs5Ozh4PJO+M5yGpqrRE9QRAB+3Vz6nmeI5C4QDhXLkejkR+GVpubmxslpM89o7XLo3AkcUcDd817FEW+77ug1OFwOBgMpJRFVYZhiCw6Ojh48u47l9cXWbY5ff1K9V3ke0VRxGkSxum9uw8o5WVZl2V5eHgotYqiCFLStK21lgAYej7nHGKMKWUeTwbpNs822db1H87i0REcvzbddWePFwaB77vrpB4PfP9rxvrXqdzuy8Hfrv66F3SrUQBAEHIXWuCQGfe/7nt3Fnqe56zQfN8vy9L3fcaYQ8zdfONKObLALSRcn66UcpnaX7PvndPkv76/dbyan/zkJ27YDYLAowRrNY2jvm4cCvTixQtjTC8EpbRta621lnq1XnuBzzntRYsR6NpqOAx9D87GCUHAGNO2bds02CCPek+/eo6AxQQ6Fcv+/u5oNJrP55TS6XTKPTqZjff29larpdTSC4Mg8NI0nk6ng8FgZ2dnvd0KpeqmwRiFSdjLvmkrzvnu7n5VNT/56Y+KcvtL33yvzNeM0oAHXVWuV6uqabO8qut2Np6MxuO9vb3taiu7fhgn2/Xq+uL6zp074/F4Op0Oh8Pnz597YRCEYdd1hDMAgIVgvdmUZXl1deVQRMY5pMQAO5vNptOpq2BAG4RQVhRC9VKJ9Xo9HU/OTk6z9SbfbOfTyXg8Go/H4/FEKeP7fl4W850FD5NWGki9dz/4znvf+h6gnMbx87P8d//k9d/5z//hz46XrT9G0nZNV6leEov6pq+qChOvU7ZsG+JhQhTQnemF7GVV1zwON3kmjUYIJXHcVrWUsu26RvaG4k701hjVC2iBUVpbAzCqmtpoyKgPAam6Nk6HojVdJS0AvRRlpwgJmqy0QgFEGiGNAVprgjAwxgWyIATXqyXFZBDGWkjf9zGGBuimaYA2Pvf6vgeU+GGYDmKPEQwJQzSEyHYNQdT3g76uTdcnmCNlEIRaa597bdsOxiOAUZkXbz14GMXpNi8C7qVJgjE2EPgBr8s8W2+6tkUYFGUGKMQe2eRrCG1Z5qvb67ooVS/CMCSEYEKU7EHXJoyMk3GcjBVlhZIWQQyBylvUSYpREPKPP/rpdDase/nVq5OmbkUnkQVxHMdxOJmMEEJnZ2dt01sDt9ucMe/18el6k6VpnBXbVbHdVnm22cq6ZQj7nDddBwi+vb1FAM5ms7bvbpa3YRzfXF3VZZ7lG8oghhoAoIE1iHrpsFNW9ib2076WBKKuabfZ0hIDiR0MBqPRaDQapcNhlCTGGI+yyXicVyX2+Wg0Cv0gZB6DGAEIAPC517VOwyU2VfHFs6dffPHZenmzztbDyZtUMOfWiS1QdTf2UyRBWbTPL86ahNyOyB/B9p+vr66ur83F8t/5jb/oaUVqEXMfQ42QIRRbCFzPiBGxBjhswS30jDGDweDttx8FgXd5df61TZjjllBKHfoHAJCyV0o4mwGphNLS87yd3QVjZDQa/Mav/+o3nrxHMeNeNFoskvFwZzTKbm5mO7PBdAiQraoKQQoBstZui60ESlPQWaWgDSPfGGWBVlZ1ul/V21q1CmpXXh2mgRDKsswpg5xKyJXLry0EXIv9tSelQ5C+jtSw1lJCrDF91wQ+pxQ7GwIHTLkDz3mKOeKjQ3Kcb9qbCuJqN0Jd2zJKHavd514UhMBYaAGjpBcd55xAhCyAxhppnKsHsJBgSjCVSlBGIAIQAQCtBabrW20UADbPs/V69c6Tx8DINPKNaKFSPuH5pvzi82eU8LptvMhrRdH11WAYj0ZDQlg6HLW9EFJDbOOBR6j2OGi6jccBgZpC6ILLm6K6d7CXBqzKVruL2WKxay1Mkoh7cDpLB6M0SdMg9LhHJ+Mdgv07dw7rulS6HwySvu3Kst62zfRg12W2tKaX1Hqp34qaWLi+XWpoWEjXm+uri9OiKCghh4td00tNye79O4PBYDoaT0fjYZKOJpPHj9/xKeWYdFJ8/vQZp94wGiRR/Pjtt2XbeZ731vvvYp8vFot3Hr39Sx9868lbb+9OZoMkvbi5vcrWwXiQjobffed9JozVJq+rdVN++uwrL+DAWlG3qm5tLyM/0BBghjerC2S7nYP9ou+j8ex6nUNMri5vrNUGWMz99aaYDGfvvv3ut9573/O4MuDVlfq//Jf/wz/58BmyRnFOFbS9kgiDr5fvlPOma9u6MUq7R2Wz2TjjC0IIQLCqa/sGqwOOlDqcjJXWURw7ZWBdl1pIh+pWVdV0Qlq72Wai0xSzVgjq+8PhuKoaPwjemEpjZDFy2UlKKU4ZssBoHYdRVVVd14V+ILu+bzsXLuNiNzDGmBDMmVvfB55fFSW2wMO067qybp27FoQw9Hwp3lCJHWqJEMqKfFvkeVnwwHc90c3NzXq9VkplWVZXFQCgbppNkW82m+VyGQTRdrulmPRtFwWh6HqgjRCCeX5R1tDaxA8xgE1V53mZ57nWtm37rq3TOAnD0HHLXL9srXWMnTCMb29v87IwwF5cXTZNgzFeLpdS65v1CmM8TAdPXzyPBynzuOuItZRJFHuMO1tBLwiqpnYSxDRNLy4uIISnp6cufsjRHqy1jk/yJguXsTQOHVY7GAwQsJTiIAh837+5ubn34GEYR5zzo6MjAIBzKK2KYjoeV2U5m06TMAp8jq2RTRdx3/M850rkeZ7DlDebTd22Djp/oxeVndB9LwXEpG575HkFw+Jo91+tLv/gqy+fvz7+m7/ya+/ige0VY4x7rFc9QMh9KF9z+IQQLkXAQcar1erp82dpmo4HQ/f0Ov6J+y38r305LMIxvqWUVVUJIXYP9t5+/FbgeafHr9M03Ww2t7e3i/lulhWLxa4x5vLiGhiIAJCyHw6HhCKE0OHhIYDQ554xpu47P43j4QAzaoxhhDobL8cgDMPQfdxfg9pN0zgeuttVxkHY9b17ZJAFhJC67lzVdr/oPiCXdOb4Zu5mdkzQxB3Anue6+7quHZrvTr4wDN1n7UKpvt5FuxdXSjkPYXfbuEhYR538BeyO3Cr7a02v+xTcseQeorIs3Yrl9PQ0y7K6ri/OTinQie8xTKqsaNu2Fp1F0BjDGLt//x5BeLleHR7ubzYbgnASxV7o9UoAbTjFGOrRgEApPEKFEMoaC02RrxkBaewvFjPnAek4xL7vPX361MlukySB0E4mozzPwyR6+Najruv6tkuSJExiDSz3vbqvEaOAQESQlHKxt/vd7353upgPh4O+75M0nUyn6WDw5/7sX3jr8ZNNnvEwmIzGcRD2TX97ezufTjer5f37DyCE6XAwHI8IIdv12qNsNhy/fe+BFvLm6vrBgwc7OzsQgDSMpsPRdDjCFnielw5HZVkKIZqyGieDxw8feYxvt9sgCr/zy99795vvn19ehGHoBb4Qou27g4ODx4/f2mxWPOAWQS+KEfOzvG56ked5HEazyeTevTvT+ezJkyd/+k/95i996ztxOhYW9xb/3h/+EOlGF1ktCcxVq4GOQ97UedfUSilHIzUaZnXVyG46mzCEQo87/1hh9WRnXouuk6ITQghxcXmJCF5vNxAjgCCHGAMQBryqc4StH3qYkmSQBtwzyiLG66YTVRUwajlBEScUaiVW2xUNWCd6RmjqhxHhoNd90/l+gBlzrSJQmiBsoWn6rpMSYNQbdbm82ZaVAUiLfpjESppeqCDyiUcaozUjlpOsqYwxSsqma7V9M7pO93a8MDo7OwMAvD47XWXbvu85odvt1mNc9UIrm5UVoRxAvLxdZ6sMW9y2/cXF1fX17e5irymawAuNtoRxTH2AySZfD5Jg5PljL8aAKg1Y6EsrtnnGvPDyZrXZltga0/dOHuJ5nhBqu821hTfLdRAlFBHfD+LBgHHPY1xLOR6PtTFpGgvR3bl3txNivVwyQgPGJ6NxnCYGwWSQOolTGkaffPJJXjfE83sNmk61dWOFCiiKGUliPxqGXuphn8ZpFEdRFMSDePTWg8dhGKbp8N13v6G15pTdu3cvCENGvWGc7M93Dg8P+7Y72Nt/Y84D0CCI9qbzpqyQBdxnANlWdWES+oSFzCMct6rTstdGZnXuDaOregkjoq0llulStZVeCUXu3bvamX0pmp9+///37/36LyMAhLQGkigdOBI3QogT6gABaIBS2vN8RIhQEmO83W5dMdpsNkqpoigcrPy14M61t03TGas9n7uySAhCCCwWi8FgQBBgjKzW13u7s9QPm6JsDDi+Xk4nu08ePkm4D63xIlqJAgDTVuXx65ddWWAhPYIhwWeb21w1EqjY92DXM4ugsZSgssr7vk2SyPM83/edKodiwggFCFoIRN93bTscDquq+np7PBkN3kRqYMw9DyLU9b1Dllz23hujdgJb0XZd5/52h9g4oNKpSYuidu+YO9qjKGqaxhqIIHGdPqWsbbuvRUyEEEowwYhgLIVA0HJGHBrjtrtutxmGoXMJdi/ueZ4yOojCTvQXV5eL2c7ZySnUkgIVU6L7rhNtp6SyEGI+HsxUq0ajURj6N7eXUeAPkkG+KQFAbdcHQSx70zWtVQooA7WyGG2qYl1seMibto7C4PzitG3rpqk451qhF89PPvjgAyG7wI+klHVT9KKCEGw227pqtLaTyWw0GIaEYAsa2bMglMpggLNNlsTpcDLBftRW9eX55a/8xp96+9332ra1CP7gJz/pjL1z9ygOgzzPv/zyS4vgW48e9XUdeP62LOLpJB2PhsN0sjO1yCJliAFEg5T5KfP/wp/5s+v1Whvz0UcfMcbyPNdCpkHSbAtm4XQwIpzdf/ToaGcvAPjP/eqv6V787NNPKyWTyVgTlE6H4Th56+Gj66urTz/6rCybn//sp2+//Wg4nUx29u49fOLxeDGZc2SNrDi3q+014vjV82fj4ejf+p/923/pr/yVAHFPIpLEA2F1JyWhpBeqyG9CL/IYb7s3th6+7ysttFuaC9karbV2ajT2CyiQcmaB5DyRSiIE+77vmnZvPqvrqhat7/tt3WAEN3mWhDGzxPe5NZJ5foCJ7Dtp9epmnfphyLg38dq+i+O4LqtGaYJx5Adl1zRVH0VR1dRWm8lkUreN/EWSpNaGYbIzmxdZSQgZxlFRFIQwpQ0xUIgeM0o465oWEholcVfnSTqgmPR9nw6GvdK36xXAqO97DYGUcjgcbtbrosjG43EYxlW+iYeDpq2ursrZdLrZrqIoaroaIcAZefXyOaXUNrZuKohx04u9w6Ntnl98/PEwTQPGes+rmq5p27pp7t17UG7Lq8ubncVuOhqOhikGEGI8WSzw8+dhHPd9TzkLw/CkPd1ut9Tjq9Xq0d27fdsSiBCEXd0wTETXE4QoJlEQnJydTeczisl8No/D8JOPPv6d3/4fIYRCSyEFZARjHEZR3zYEY4RAEEU88LfbbehzHPAyr9I4GcSD0A+GwyGmhDEWpYkQQgtNGLtZ37z/wTehsaenp7PJ2NH+t6s1gaQsy0b0ge8P4kRoFQXhtipCz2/bNvGi9WaDOQmiqCqKwA86C8qm9sMAQhhEUeQFYZiuVtfpaHjddoVSPsHfCDgsuxkhJ1XVYiswCoLAAICIddOME15CKYSSse9prdbr9XQ8PH79GiH06NGj1ycnVf1GAeRQGtdgai1dnn0URehNYLR+8ODBZDIeDVLZNnfuHKaT0SeffEIxwZguFpPxxFRV09WN6rphNOxrIURnrX306NF2szk9PT3Y3QPWrqsijmPme9ZaIFTM/byo0jQFGLlG2LHiOtG7BtwP/NVqlQxSVyidYJUHPkW4qWuCcNu22mjP910mn2uZlZQOq6mqajgaNU1jjPZ930jVdZ1DYBzZ3M0NGOMo8h3ETzB2lpZhGDo4i3Hm7HSMMcZoN1A6yMhhOI7s6IZ493Y5SZRTqLotrluGucPDnUwnJyejQdq39fRw5HtEiTc6WAtB1wsIMUKoKmohxN07h8fHx0f7e5999DnBUCkVMg9a4AfRMO43bRF6uG5bTUAc+cFgMN3Zb3pVVrVzpXcCY0b47u6u88Rfr9dVXcRxVFVl27bz+fz169fT+ez89Mxa21VNOk7jOKqqinM/CHTo+1mWffHsedmK1e06jPzf+b3fG00mFNEyy2/XG2NvsyJ//PixURpwmuV5XeQ742lR56WU09niThgqKW0v0Ujfu3N3vV5vVmvue8Lq//q/+n9ba9Mwun//vrV2sberrPH88PHjxxcXF1999um9w6Ozs7Pri0tnjDGfzp6dvt5mue7FzmLvan1z9+geo3QUD+FwtFmutBKvX7xWFgBI8qwMQz8Ng/Pz4yjmZ2cncZqcnJ/cme2E3NuUeTIe/uXf+mu///3vo/PtshM91xZVwkAGvRDzIMsyILWWykJQNXXMODIaAtQrVbd9GEfIAp/xtm5cEp6BAFOilILGAm1C5kV+0NVN27ZJMuh7ySnt23oxnWmghK3bPoOixUZI0wrbQwjjIKSYyL7vqxoaXVWF73NpdC9E2XaYe5ZijaEXBlrrosj7trVGYwQ5ZR7xZNWbRiJtoTZd2UIDDdAAGoZw6PnQ2G2RK6PDKKmqqmn77XoDAECMNF0Ljc3zvBeq6npKaZnlzlNwd3cXY3x9e8V9lm1W65vrKl8PBpG1Os+3dZ5ZKfJ8a4BGDG2r7PHjt7TsB6PRsxcvy6w42tsHwNSiIT7d5hsh1DAdNWVPEacEN2VJCJZaLvZ2icc//vyz8Wy2zbMwjjzmXd3chEl6cXkdseCtu/fPLi6Y552dnIqmPtzdaYp8c30Vefz58+eMMcbYT3/441dfPfvkpx/+N//w/9MUpcNekAEeocBYn1OCIHUCzqqRxpZlPUnHutejcPDg6G7kB/PF+NFb9/YPFvfv3kvjxMM84iGnrK3LwWgShmHftHuzxWKxODo6ckayziLYJUJQiBjCqhdpFANjCURZseUehdbIvuWcMo8CCjnnttF92XZdK5Apuure24/CYQoYI2mcEfKFFvBwf3dnEhiBLUjigTHWNeOMMWAtRsj5sUAIN3nGA18ZHYahS0OsimI6ngSe3/e9Y2E7B2An03ecChep+uDBAzd/NGXhcdor+fr89Pb29t6d+7PFvCrKYRh7CDGPCiui8fDy+mpvZz8Ok7cfvv3y5evJYucb73/ACB3FKdZ2dzQl0iChVStFK8q6qtrGHS0OQGeMBIGnlIDQaqmiIHT6I9n3FGMEAQKWMxZFkZs5nC+j7/saWAd9EEJcIbDWFkWBf1Gv3+CTxjhbYPf3OuDCWssoVb+IYXpjaYAR5cxYKKQWUiqtHf/9a9TL0WBcrLZbALjG340OCCHCmAGgE8IPQ4hxWdeEsbptpdZd15y9foWMQhhyz9sWpTaQEA8BjACYzSZhHPzqb/6qF/OqqmIv0J0QfZOmsWyElsYA24rWw9wKXbYd9gPnB1O15qvXVy9eX6bD+c312irLmLfZbPJim+WrPM8DP0IU9VLmZaMtWa7XFoLdg/3b22vCaBRF88W0bxoCYOwHxKKqqBDAVVUFQbS7u/cme8sgY1BfN11VUwT393f39vZOL8473cfj4cXV5Vcvnx9fnjdSMUY2y9uXXz59+dmXtyfnRqj/5z/8L5+dHaOAE593Xbe5WdqmF3llrT2/vvr0y6+yqp7P533b1dvtwXynLqt1li/rYri3uLi5fPX8RcQ83UoEsDHGj8LzyzOr9NHR0c3Nzbe/+73Dgwd9JaeDiRFdEpJHDw+MEYzT5e06jJKb5bqSclm1V1n2/NXrj7747Ony6uGvfAd5gU8Zo5h6nke5VzVd2/SBFzLf833fKK2lEkJAgMuyjtMhJjTLMmWAo8oAbdqmqfKiqiolpepFEkZK9gRDaRSlNNtkRmmEUNv3WZYN0/TNLzoWvTEAI9kLj/K+bfu+v3PnDsOYc24hwB4LRin2GECQEqK63lobRKEfBsPZyBgFLbAaaG095kOArTEQ2qpvLYIaWMZ503cQ4CSKPcqAgU3TBEHoujnm+S9evWa+l5XFar0u6qrr+08//XQ+n3PO4ziui3K5XA7H48vr66qqgLEe4z/98Y/bury4uKjruixLPwx29/aqtlkulx9++OFqtTLAPnzr4dHRUVbkhLO27xCmXhhZozhlbdNEUWQNvLm5qcuSYvLJp5+n40mSDqu+bbteKHnvwf2jo6PFYjGajOu6fv3yuC6quswJQmkQNXl5dX4xGo02WRbG0SbLHExslDo9PsYYu3Tnum0MsBhjaKzL9JFSIkiCIOKEu3bM8zwIwChJ5+ORFcoYLYRAwPRND7Ttm45TfHBw0DXN69cn+7t70+m06xsATeBHhDBCECEEM6qNKYoi32w5oVBqIyTB2NUm1YvQD8IwrLvWrTp8PySYXd9cfvXVFx9//PHJ61MCyf1HD48e3s+p+YPXL354cXrv3j3YWgwRxthNEg64d/wQd/GODtj3/XA4dMFDlNI0TZfL5Wq1chtX14e6n3fsCwjhgwcPKKWr1erRo0d379y5e3iUrTejybhpGoqJFr1Syg+909OT25vrs5MTzrkX+PPdHaUURfTVq1eHh4fL5XKdbeu2EUrGUXRzeZVtttbaOI4xo8PJGCDogG+3PhVKuuQNJ2S1EFRFSRBO09Rh1gjAPM/btt1utxC/WYFK88bEyQkvXBZHFEXOwt7xETnnVVU7Maqr0V+nbQghqqqy1jrExsVLuTLtDBedl6ojzrt9bBAEbvX6BoX3PLebcfuDr1/EWan0fe+cn11klUPnl5ulBXo4iJQUr06OeeDXbet5vu+F2+02TMN33n385//0n/nuL317MZ8Hnkc5y8uCcubOM4wxoihOY8o9pZQ10GgAAAqDmHPuIiJevz5BCI1Go+PjV24dfXp6ul5vfT9EiNzervZ2D9q2Xa/XFsFWCsTpcrWSUgbMZRNSTrgjDn344U8/++zT0WSotEaEMu7P5zu7i50o9C/OT9998s6j+w/uHh5ZJR88uHf//v10Mrr76MHdu3fn0+l8PLm+vBJCvH79+uDwUBl9dnmhIYAQDgYDqXUnxJsJUojhZPyHf/yDjz/9hDNitCzrKh0M/tJv/ZWb9VJDQD1OCX/84JFPGNCmqgvu0fXq9vXLV9/81gdfPX1aVXUcx0Cbg/3dhw/uck6yMue+nwzSTugwGQgFrzbZaL7btrUx6uDOkcIAxSzomq6HsNJKiX6cJkZriFjTyrYTHDEgdScNpDxJEtm3CEPGOWHUIsgINUoPwtjHdBylTsXeVDXESCjpeR5EliOchJGyKB6MoijeLLN62xuJICBSGNHbpugD7gvRO1jw9PRUAyv6Toi+aptNVUhrIIQJCUAtQ8wghJu2usk3fhx6gd9LgQhWGEhgDIKN7AFDHZDS6LbvIELaqjKvPMgSP+aIyl4gADFlXddNp1Op1eXVVToajsbjruuO7t45u7xw9/12u/V9/+ZmiSmr2/7V65N8kwNpz16dRjzU2hLPX623x2fnfdNbZdN4IKXemS+Go7RSnZfGedsCzCD1jAGy6whUFFlGsB8G622ppaaIYEYBgudXl2maFm1dVDVE6OWrV1cXl2cn58en5zfL24uT03qbt3XzL//lv4yiaL67c5OtNAU90ifXZ69Oj4/u3lns7s53dngYaGgVMJgzDUGvled5BLNkMEwGo4DRxPNs33OMmq6e7kyjJMzz7cHu3mI6s8rIricIU/xGAgMhrIryg/feP9jb//TzL45PT5wMXXQdxcRaq5HJq7K3mgX+cDLGynCIQ99TRhqjpOz3dhdtXa7X6zSKOWVJkoTDdOfOwdtvP5omA9DbLz76/I//4Af5Kru6PZee3Qz4y772/GgUeIPBoOwaz/OSJIEQa23LojYaaG0JYdbaJEkcrZv7oeeHCKGbmxshhO/7BOFhOnDb168Ze13XVVV1fX2dROGvfO+7jDFtzNXNuqz7vhWH+wfX52fHr19ui7VFdpmttDWP33qLI9q2HcKUM58x7+7du4yx4XBYlKU/GHhpWgsZJGmUDgj3sqbSGDZtbzSQQg/iAcPEMRqFEFEQGqXdSjyNE4Jw3TZREhulEYCj0XA8Hk2n0zc0fIQcJVED+/WOejQadV1nrXVAiuPhMEbdgjSOY7dZ7fs+z4swDOM4dk6/jtDlbmxtYa8k8z3MqIHADwNMiQG2bhtX6Cklfd+98UDWWvS94026ScitUh0nR3TKJXu4yB3MqLIgGQ05IkhKqFUQ+KPJuKgrZc3Nal1lFTao3WyPv3p6eXkurTKICIDTxdSP/Wyz5hQ3ttHMAiON0oTxpmt9DJ9+/LNvfuvxOrvuuiYIvJPjV9l2e+/eA6XMdrNx1P6qqtq21Vqu11utbN9LqUwj5KrIBvNJ0VSTyaQqSquN+zGIwMnp6060QovxYjKeTf0w8H0/SaPxeDRIoouT07u7h7uTGRBqZzQ63N25uLx+8fI1xejs9UtowVtvPQ4nw3Q+OX79ejYcA2A2m9UXX3y2zFYlkNHOpO/k5dml1Obi5lZCqDE8u7y4Xd0iRpPhYHW7FF2/zrZZVd6sN7frzTBOPAhDysaDgSMln19clV2lTFeU69OzF9vt+gc/+MHTp8+DKEQEE8+bLnYQCeY7h6PR4vZ29T/9m3/9b/2N3/KgvTffIVopAEDe1kmSKNEZC0IW1E1psI2iSFat7/sGw6qprdW+zy2CCCFhFCe073qMsep6CEy23Q6TWBPbVBX3fYxxURTu8G/aDlKmtc3zbRTE8TCGAHKGpagwJJPRVGqhtfEob5uOBrzpuzAM266jHu/6PuRBlRWAMI+yuq4hIwAAjCkAoKoqYxFQhhJSNTXn1PO8XnYUEyUksgAj1LUtBBghVJX5ZJC2fWcw3OY58zwAwNnZGWFe27Zt2yutg9BDCJ2fn49GIwhMdX073dspixpBHMcpQqTv5Xg4oRjfrFdV2wyHwxcvXr18+XJ/fz/Lsqpp/upv/eW27z7+9NMwjjbbzPdj3w/LonAzL6McQuyFQVmWx8fHo/H48uy8rmuK8OuyyNabkLJ+PEbWCtF+51vvfe+7v7yYz8vNdn9vp6oKbdXZxflf+K2/YglqhPy//f2/9+LVy/t372w2q29965fyvPjo44/bvg/j2BijjPa4BwDAEJVtE3IPGQshhAi5bu766nY6Gg/T9OL04u7do7LtHjx6+PTzL4MggoR2smeM7M4nx69ebNarnZ2d29trBKEL5EIIKGssArOd2XK5jKKwrxtjTNd3xOOMMYMZhLDrxJ3Du14UGgOAkhBgyMjtagmtSaK4M0QKvV6v/9E/+kcE693dhR+np5ub+O6jaDi5VW3kB7LvgVJCdF9jxC5Ewqk00zRJR+nZyYlPmYW4l1IZ7RBnF7yltRZdjxDClI5GIyH7IAgGgwEl5OzsbGexJ5gaDFLP49l6RQhJ03Sxuzi7uCRsRhB99eJlHCd1201m06dPn84mEzfICqsPDg48zzu7OE8GKaWUab+oC8ZYEIZtJyCEwlihlLWQEFJVlWPIRFFUt53v+23dWmt7KdI0deFijt/St30Yhq3ogRBJGFVVZYEt6ir0fGerQAiRSsm+C7gHIayqKk1TAEBelU6jhBHCCAWB74jnblnqPGdcaQZSWgsppVpL2fXQ9xFCRmkhhM89Pwwc7Ob7vsNz3Dd93w9HA601tBZaawGQUqbD2O1alVLAGIooj6KqaQaDkemXHjONMdba3cXONqs8zrPt+pOPPvaAmc8WX7x4fnp1bRAmHjPIVl1NKFbW+D4HpUBQU4KgBR7jRba+d+fuP/v+7//Kd7+nlLq6uoIA3FwvF4tF13UQYCHlcnmFKCqz3PO8TZ4VBeGUaWMxJVKpF1cXHsanp6dhGC43G+rxvtzWSt69+7AocyU7A2xeNOjgDvLl6vZ6OBsihICxq+ubeBA9fPCg2GxevHy9/+BRWeYnJydCKK01xOB2s54tpnt7Oy+++hJxulqtMMYQo8vbm7KqUh6JpsUIT8bj2vetUdIqpbRsG79uv/zq8535rMjLtu8IhW3X9V0fBl6rxWnTEYuX6+38YC8dJoe7O0+//PKb3/jGepuN53OtNaW07lpmIcY0W23H9ydhGgAt/uiP/igMOfeiNA2IMZp7jBPMGNE9AELhQPseyrUsZU8ZkG2fxMO6bYxFAGHTS4C1kdICVFc1hMAPw7aqJpOp0rqpGt8PLAQAAM69pmu16EfT2dnZmR8GaTpEAJZVFflBL+VoNCIGZEXeA0MYJdxHlBVtyTivqioMw6KpIQC6bQdRGHDPVfZG9MAYYGEre0Y5QsRaW23zQRR1WmZZRjHxPBwyX2vdtV0Y+LXsNTIQWoQg5mxbFmk6vF0thZD37983FqpeBEHA+ZsOqFXi9OpiMZtvsi2PQ6vB5dnFnTv3gihykP1XXz7bP1gAJa8vzo3u9/bm18sbCxHx6f/3v/unH3zwQeiFq+2GEd51YtMsGSNVg3sFW9GNB0OEAMBgtd5+/PHHf+Nv/LX5dPYv/vm/+Nb73xwOBh7DRwe7jJLZbDIYDF6/PkGmixN/m68oI+PREFB0u779wQ9+9OOf/OT0/AxBe//goKqqPM97bdPxZJNn2Oq8rFjoG6uA1FpbDWGxWd2b7zVNYyGImT+bLZqmUdr22gyGY6ltEERO1r/drtPxUAtrsPdHP/yTe/fuEQiyzVp1LQBAExpFwdnFKfe9pqs3q9UgijDEAHg09JvNJgxDqVXVy3feeXLvYD8Mw6rtEEIXpyd1UfZtHzM/iqK6roMgWq/XMY0Wi3ur1erp66s4zaHUHzJfcWp7RZRNwrCoqyD0s6IIeGiMJgRhAJ2b9Ga9DHx+dHDICOW+F2gVJNHydnV7sx4MorJqOGXWGEqI0rIsyyAOpdZ37tw7Oz7DGI8Gg8lo8PLl86LEnudNBgulVF3WHqNFVRZFNUkmnucJICeTkeoExXgQhYSQdV60UgihAi8UQiFEkji22nBAy6oCAPpR5KSeGOOuaaMgdpiMMSaJwqqqvMCrqop51ABtgCWEGKONMkkYdVI4rAMCMBoO86JAlCirMafWWoAA9iiGSAmBIfI4l0IYBLnvaamCIJBdr6RCBIdR5KSFvu9rqQijWmtgYBpHZV1pY5VSaRK5hQQCkFOuje2aNkmSoiiyPPc8T0hptfY4D8Ow690akxqrMUaIEGht27YYoLZqCMZN1fRSAagMIkXTFmUuadw2bTxkQHZd33Zp9LOf/ezO/sEm2zYG3Oa554VCd8ropiwCj9leIA1HUcxJRajFWndVPZgPhG7v3/vlMm9v8O2Dh/eWq03ddvkmd+BV2zeMsaoqlJQ9AJPJpKoqxryzly/9NGQej+N0dX0VMe55njQaIoQxDjg7Ojr4gx/8cRJEyNij/YO2rHTbUz94/vJ4f2f3L/7ZP/Mnf/InZaWns/Fqudld7A2CwGhJKD2693BzuyScKSOqqjw42P/ysy+mSbK72LlariwE37n3cLtanz59dXRweHj3jgTmixfPpFbDyaIXLWWeUmpnvtv37WKxSzZrpTVlOAnS5XI52V0c3b1/8uI4DEXk+Uc7e0W2TcJY9LIoaw3g9WZTFAXFKJPbpu1/89d/vSqKq6vLxWJydO9R01ZVVbW3K+JRVnctsKTrulGU6k5YYAywlNJWtAARg4Bj12ECmk5QCDAg1vRKqenuIsuyumsm81mZ5ZRSyIgyb3jxZVEPxpNNkVVNHXi+VcZLvNvb20GSVlU9GY6qskyDKAgCgkHTtVXV9H1LfNpL4UbLUZLWZYUQNMZ0UgCKhRBRGGplq6b2wtBREYosT6O4LEtLEKUUSo0scpIwAABA0KnyYu47PFEZbawNo5AwutlsoiihhKRp2orecZDTNK2qSmo1HI+0VNvV9nB///j41cWVN51OgdSU4YuLi/F0BAfJ8al2VsbruoiiaLPZ3Fxdb1ZrBCBFWCMMoMmKglDOQ58gXHVVOhw4te3F+dUojf7nf/vf/MbbT3zGjdKr9TUBIArYl599NBwOlYZGA+J5nexmO7PPvvryH/+3/+RP/viH1mKE0HxntpjNr64vAj8SUlmIEEKTyeTi+NXe7qLu+pvVMqA8SuLJzvz6+hpiMBilBkNrbZHlzrCzqKs0TEaT8TbPvvzyy7qsptPx1dVFr+RoNp9MJm/wX9X7vs8pE0pm+Waxt5uXRZIky+VNHMbj8fjs4pJ4vNemKKogiL77y9+eTaavnj6rqmo8nXzzm98AQpRhdHl5GSeBEGIQxRfXV0kSvfvOdz774nOl7c7+oum7TjafX15QjJM4DX3fWh3HIe6FWx5qrSG0nHOoNbKAYeIznmWZF/ijyfjk4rypqr7tQp+1bRuGYVPVaRhZazFlBoK273b395JBqoSMg2i5XEJoH7314Pj4NTD69OJca3v3YLeuG0LIeDwOebRareJBmG3Xvu9fnl3sjmdX1xcaAtX1q80mjCPRicALjl++ms/nSAOc4OXJFkthEQy4X20LAKCzWnQWCM7MshM99z3HXdHaUs6A0YwxTChUEiEUR5FRumtaxhhmtK4rxhiGECFUdW2nVOIFwFju+ze3Ky/0PM/r284Syn2PGiOkdCxbRzFyj4NLaJBCODK+I4k6OrxRupfCVT0hRBiGsuu7vnfTjxOREUaMMcBYNxBYCAdRvF5uJpMxIWS9XDnJmwIGMl8BBCzilHJqh0kaB/711WW2WW8MODw8Or642JSZ5wUIaI+yi7Or3TCVTYksiXmUNaWSgFrzNaS2t3dweXWzN19obTfr7ObmZjKZubBJSJCzwrdWu1Sp8/Pz8Xi83a6HwzQvM4yAVna22N1eXhVlTint+o0TuxljduYLJbTUFmgDAEAEX1/fpMOh1vqf/tN/orV++OTtH/3oR48evn11dQUIDoLg408/efLkiRcGlOIgCAeDQUi9xXx+fX1d1y0J/cvrq9FwslmufJ9TBG8uLzZFNt+ZV03dtu1gMNJSYQ8vJtPJZPLhxx/duXPHC3gUhKZqh8OhIuSjTz4OsHd0dLTdbp9+9RWl1Ej1059+mNdVOExVLzZZtru7u862GOMXL589eevtItuen50d3jk4O78M/VDKlhADKEAIEw1R1VSMUOMyk43gGFFCtbGqrhEAOAwgI7LvtOoIpRaA683SWbOWdUU4E1ohgiGETd96nieBWmWr0WR2c3U1ieI8z/PcepFfNaXvcSk6n9G8LLwwaJqaUqpEHwQBj7yqqTHGRqpqufF9v9YSh7zqOiPV0Atl3VVSssBrutZj3GjpB7zqKhYwIYRsu2E6KOuykQQzqgCCRlNMIQDZejNNB1VR+l4opdqst2zKjQ8wonXTLZ89J5Tu7e3FUbrZbLpWQNAAazGABOGXr1/vHx40nTg5O9ubjuq2WuztIo/31baoyiYvVa+ARcYY0MuzV8eE8SIvDURZthkMRpRSg0xRFBgizkhZ10KJ3d2dv/23/o3d2eRHf/yvRG8rzCCB2yoLo9nJ+QmAGAKcbbaU8YDSV8cn/8l/9vd/+KPP45hMp1MX5VwUWRj6B3uHSoJXXz799ne/03TV6ekLRJxjH51PpgyjyA/WJ2fT4bAVPUR2GA2CIMjzPM82lNKdnTmiSFi53Cybvtnm2cXVpTD68ePHs9F4vV6vbm7ruj44PLy+ulpttlXT+GGQZYVVGlG7O90J4ijPS2sgkHZ/Zxfvseli/vD+vf/sP/v7HvNHo5HR/ZdffHLn8PD6+pIQwilLvCDP8zvz3fnOompaYBGlVFjNuN+0vcKgFT2oK8Ip9HAn+rpqoiBabjdBEPRN37Z9MoyhtRh6Rtl0kVxvl5TyKIkZwtYPN1nueZ7zWXSC515JPw4Ap9PdRStFOh72dVuW5WJn9sXnXw3jqO8lgBhhW1bNaDi5vr4Woi8VQoRYCwmin33+xbtvP1FSlptsvJiNh0PGGEAYGLterx+/9SiJ03/1wz8GEI6Hg6ppECZ111pgptPJ9fV1HMcQWgiB1BZB6MiLfS/jOHYV1lV/KRUlpJeCYsIJbUWvjdHgjSZDSWWtBdYGnu/cNTbbbRAHnFAtZOT5QgjkecJoaXTEeRgEXdcVWTkYpaKTxlrOOeesrZu6KgAAURJba5uusdpgSjBGFCHRdYBRZTUiCGCEEMQWIgT7tsMYd6K3EFDOlVJt2yZRWJalMYZyhgmRsrccnV7dBMNx9uUpgYwwTinmzKcUK2WUUk9fPCectdeNH3CGyCrroWYMxgEjRmlgNVA1o0BboBEwFpRFd3uznr+3SwK2yraUkyhNsmyjlZBlf+fu3W3BB2F4cXrBKeultRYul8syL+IkFE1tjUKYQ02kNpgSxii01gv9Kq+ff/lsMN0BEEuthvPpgrLV9dLnTPfd4YP7p6+Pb1dL/8TblvnHTz9rmu7q5vrRvft3dndffvnleDqpGw0hJoi1VXt1eSO1Vk0bUTqbzJM4vH//7ihJgbFlXiigjBVNWxBA2jyHEEfDoVLq6dOnWsvpeBIlYZHlxycn4/EYU/LkyZP8dksIObs+P7xzdHl6ThBe7MwG49HVzWUSBDiIblcbzCjy2L23H5xdX3ieV2Tw9OQcEaoNUMYgqw0jREmttXGfFqa8bVvVCwpQW9dSSoIhAKbt6qIuIAatFJARBa0X+EopY23bdQaCKI7zqoQEB3FUdy3zWDoaZvlmMZtba8fjMWPMeRI5DkDXdfEgzfKcYiJ7YSAwEFRVZbVxS/nA3awIMcYQgJxzYy3C2J3nTrsh2s5ZnWitwyjiYSC14r4HCbYAOP2Fg19d/2KMIRhbbfAv4oCbrrXWOuJj0zTL5dIYM5/NyqLAGMdxfHBw0LTV5eVlMkiDKORhcHl7AyC+XS2rqqaUWoSFlBTRrut6JYMkzbNCWdNKgShpmkaIzig9SOLI4xRAYsF/8O//L//D/+P/6RtPHndVOUyTvq2qOivKTGu9rYq66zGjeVlJpQhnn3z2+X/8n/xfP//i6YOH+994/wNtgDambWu3dSjKeptnzvQxDiOXaFFWDcUk8HwLAaX0YHePY+L7/s7Ozu58QSDyGd/b2xuNRhDi8Xx2enZxdXP9+uT48uZaaOUYlm4dFwbB/v4+AODg8DBI4jCOvDBIkmQymUyn88FgRCCZTRcHB0eDNI3DaLazePDgwT/7Z9/3uRenSZ5v1+s1hvDVq1e+7999cL/p2tPzMxfa6W6Ghw8ejEYj1QstlbPG5Jz7vn99fe2iPgnCdV1bbdwu0VpbbjOKUZHlQRCsVquDwz1rLcXE87y6bR01xRjjipcBOo4jQogxJoni8XC0Xq4opYPBQCk1HI4xo9PptG0aSnkQJ86pcT6fO7h8MhwRiKbTadf3z54/H4xHouuLooiiyBidxvHuYqGEXK+W+7t7Rmuf8jSMGCbIAsdkd/HCzhLgX6e1jEYjV9MdhuO0SH4YOKYpAAAD6FjnruMhhHiexymVfe9+BjPKGLPGWG0cdq+BBQCEnu+UUMaYOI1cWXd+OwAAZEESRm84RRhCCKMkppS2bVuXlXs0IISYEmMMgQhi5KgsXuBjjD3GHXXHkQgxxnGSOINiqzRj7OOnL9d158dJFEXD4XB3d7fMi+FgQAjpegEojtM422yA1NvtljDctEBqUjWKBqnAXi4s8eNGCG2VNopTBiwSQmhggyDIssKFnR0c7fuh9+L1q/V6eXZ6QTlX2lzf3LorSQZp29Y88KQUBOG6rAgnYRiKroviwBgTRVEYhlXZCCGNMXXdtm2PKFrs7rz7+MnHH/6MR958dx5FETAWQYgQoB4P4/jeg/tSi9V2E4YhI/zLL59+8tkXyWD41uN3dnZ2HIu6b1ol5fn5+euTV9IIxsjp6YnVOuCe6PuiKFarled57zx+gow9efnq2Rdffvrxx3Ecn11eaGPqptlkW8rZb/zmn+r7fjKZMcZ8LxwOhz5hWqqu66CBRmsl5OdffLper4+Pj4UyFuGiqIwxaTJEECNlDEDQ4W6z4aTcZgH3kjBKgtAIBaSGCBkIECHOrIMxti0LbQyGyGccYBTEEcBguVkO0xgBs87WmGFpZJXnUJm6LhvZl11TFAW2wNEYlNHC6m2VRYMIA0gRdlt7awwEgFPWtG0tBQsCxlhf1jFmWJqbKiuh8n2/3ObQAqWUzz3RNIkfWmtr2UsCO6N6JS0A1lqOCQQgGaQIY2d0Qxivylp2kmPWdQJA3AmlLZRCWw26TlRVE0XJNis44WWWN3X59NnnSRiFfpDGSVmWX706fvTuN/KyrstGCY0gUciwkENg0jS9LbZfHb+EAedx6CWRxlAibSGwRhEAUu4zY957+61f/fa3u9Wmul0OkrTrG4StVA2yBmOcxIOibhAlD548Whbb/9c/+kd/9z/9exazKEw4970gWm9zabTnh5T4aTJWRmfFmlAYePTm6irf5FIZ7nta66ZpZvOdVmsJiUG0F+Z2uT25ulnnVSdM5CcBD4wBfhRLCF++PqurngW+NkZJuVouszw/ODz0g+jg4ChNh0Kbd9//BiRYC5n44f7+YVVV6/UaQqi0aJuqqtv5fGc+nWzXq9cvj5umub2+cXvCtm2HwyGi5Mc/+UklBEsizbDh5Pz6Kh6ko9Ho3t277zx+8ujOvXGcJpgPvFD1IvR9DrEPyZvCxygG0BhDEE59Hwg1mow32RYYe3tx1ayzIst6qxWBiBI3zjdNQwOmESiqknP+3qNHAcZdlg3DcHV7yynNN/lgMECMF01LER1E8dX5hTHg4uLq8ubaGPPgwYPPPvl0eXNLMd5s1wf3DgFBhHvMC6jPsmLrc286GO/s7O7u7o7jcBolO6MpA9iHJOY89D2lBAAmyzbueiCETiLk1psupML3fWCAxzwhemsNYQRi2PUt91gUhO4HHDRvtfEpH0SJYwHGYQCMtloboy20w/FQSoEhQBAga5umxhRbDIWWCEOCURJHQvTSKAOtx6jDJjClbd/3Uk7Ho8DnzKMaaKUFAtAqjSAEwHaq52EgtbbaREEYh6HHGCIYc4o5bbuGUkoR5owuV5tKmkIZ6gca6DAOJ9PxB++/RyDqup75XqekwTCJYoJoK2xv0Dovg3QAo6ig5Bay0dvfkN5AWAKQBbBnBGJrmryVjWrbjnlewL2iKD7/8svVZnN1deV4t6vV6uL6inDmmJrb7ZYwepttSOABCiEGFKPb6yufe3VR11XTif5muSQUtW3TVPXq4iZfby4uL7fFdv/o8K0n7xBOaUCVlF1R1bebkDAE1MXt+fPzE8sYQLZtqtur6+X1sqjqTunLy+vnT1988I33kyShlH7ve9+bzSYY41YKw9DR4WHg+5PxOEmS/aN9ZdXx8fHezs7+cJpd3DSbfHcyAwDMFvNPPv/s9OqilcJAcHNzc/foTsT92WiMLfj4w59nZTGdTsfjsdUGK7hIhrdnl8U2o5xJrdpOxMlAaIMQQcYCAKDrjofJoMoLZ1aXV2XTd46IBjASSiJggDYY4yAIkjCSUspOOGZC27ZKGQihEhJC6IQYCABGsYuu5p6ntHZUsK7rojh2rbSQsut7J7opikIo6Swy2qZJkyQZDuqu1UKKrnecZTcfuHSbqqp8nzt1HLAWWSClBAhCgiHBjDHnEda2rdAKIGitHaYDYKyz1Dg6OmKMbbfb4XDoUtCOjo7cQvX6+hpDtLu7m8aJ88yTUiZJ8s//4PcBgkrqpm4Ho3QwGKyWG6VM30vHvN6sVsPBmHq8LIqqqoSSTdcihCjFxmhgdV3mB/u7f+tv/PUvPv1snCbDdODe+bopDw8PR+Oho81pY7549vx/87/93/2d/+g/+pOf/Dgdj1wS58OHb602G6EUIex2vWl7cX27vLq9yauyaYuqzLVUBBFMPWMAZ2w4GLR1c3h4x/O8b3/nO28/emsyGudZsd1up9MZIeTo6G7TND/7+cdnlxfaAOpxrXUnhctdOrhz9Pr81HFUKKV1UZ6fn0sp93f3yrL88KOPACIHBwd937ZtvdlsKKUnJycnr4/Pzs48z+t7qZRACEmpp5MdRr3NZjOZzAAACtplvgUQemHgnDhHo9HR4eHOfH7/6M7Bzq6WkjFGCeOEI4vSOHn48CHGBABgESQMY8ogRi45XjZdtS0RtHt7e2ESI0qapnWGJ8YYIaVBMIyjMAwRgNDYfL0RXUcpzYqCc77d5r1Qk+mcU8Ypu//o4Wq7uXfv3vvvvscp/eKzz4IgSJJkMBjEcUwpvby+FlpNZtO+7+fz+f7+/na7kV13e3Xtc28xnz68f9+jNIkigrHjuadpmqap21587TjvTl9n0Ohcm3zff5NObK2FAFOKCNFSKSGsNh7jUisDrNSq73sEIMVEdL3VxiWUUc6klAgArbUSMggCp+9t+04Z7YByZ0aWDFILQS+lEEJLxSlFCDFCyrI01jozmel0qkQ3GY/atnWX5yzYOOdtXbd10/d9VZSuAjhqZtM4Czmd91JBNhiO9vb2giDYbrcIQGAsY8wCdHF1aYxRShBCju7e6a2tLDDJoPLCk6rbaPjF2UUtFWG8+wWTVSmVRHEURev1Sqr+/PzUZ5x5PsRkf39fKVPVhRAiSiJrDSHk4uwcQth0YjKb9VJe3d5gho/Pjnf2dtu2hZgEnmekPdzfz/Nc9K3uhRRiu95WVRUl8e/+7u9eXp7f3t7OZjMALMa4KsqubiDGddvUdf1rv/Zrh4eHOzs7+TbbXcyTYXJ1ezWaDL/xzffyPD99fVyW5e/93u9JYw/uHL397ju/9uu/kW3zMIjqpjm/uAAIvvPOO9/8xvuff/zp+++8uzOdRX7QNE1VlkWWc0LHw4lG6ODO3bpqkUWM4qIoyrp674NvDobjVipj7WKxmE+nbVVDA5fXN7e3t13XXVxcMOa98847fd/D73zn263qeqOiOEYaaqUgtJ2WnbUQQmIMJVhKJYTwGW/6runFaDTq+54Q5G5QLRX3mPO4wBjneY4xpJRabTjnSlkhBMLYmDeOjJXoOKEcYo/xsms83zfGOPjljX0ShKaXUsrJzjzLMg9TxljV1JBg7nlCCAxJ07UQI2gNAdAoRRCyEOZdzcNAa2OkSv2waRoW+ITRvCoHcdKut994+23f571oIabGmIAHvu8nyWAwGg3HI2V0HEYe4+vlsq2bg6NDY8zJ61e7u7sU0U4Ky+x4Mmmqtu+6ti2TQZrl9RdPvyo3GYTwqxevKGd5noV+4FEutKqEEEpiYGXXh0nkMWKa9n/17/8vCGM//9nPkDTfeP+91fra8/2rq2UYRBjjII6KqviTH//ow59/EkUR41xrGwTB47eeUMraTnz21ZdV22BoEQDDNPU81gjZtNUwDr7x7ntK6D/6wY8Gk1mZ5QG1BzsLSOhbbz+RvTg8PKQYM8/75LPPz6/OkUXf/MZ78/kcMvSHP/zhp59/whGjBHVlPRqNOqn+7J/7c8aYV69epWk6nU77uoMQDpLUSZGX6/VgMMiLLcNoPBnmZXl8fKq1JZSPx8PVavXDH/5ku93u7u4URbYz39Nav/P47YuLi6womq5NxqlSqq+ad99552B3z/Vc7z5+4vPgyy+/zPJinW2l0RDCUZpijPOyWC5XddM5T5UoiYyxQeBPBmldFfPpIorjn3/8UZDGeVnfXt1EhDNMyqYllHa6393fK7LsyZMnd/b3OCZGyDhNX52f7u7vTYfTIsuUVV3XxTQo6mo4H/d9X6w2GEIDEGHUj3wMoMd8DKDo+sE4Pb86P7pzry4bzvn55VkcxxSizWo9n+28PjkO4qDtu88++2y22C3rglJ6enrqZD5d12lr6rpOkoHLGAnDUEslpSSECiGGadpLkZdFFEWOu+2sjJUQmFJptYVQauVTpuuOYQIobkQfhxFAsO27qqoGgwEwFgHY970fh0VV+WHQtm3AuNVvbL8wRBAj54iAAAz9oKyrpmuZ5xljGCGUUmNU3/cYEt/3Rdc7wyilFNR2NBo1ouv7vs7L6XRaNXUn+mE6AAAU1ZZgDAy8c3A4SuJ4OJgdHQbcQ1L+7m//LvSDTVmVov7ud36pvL69vroiSdz3vQ/gb/3Vv5zn26+ePYWIdUKtVispJeMkSZIwSAaDwWQwPDo4GE4SnzOrbNd1GqIoipq6NlJ5ngcJ/viTT+4cHUXcf/bsWTIYBlFEObm6uZwu5lZL23er5XK1zqqyKYuiFSaa713ebhBCHvUMwI7cqbW0RvV9p5SeTqeE4L7vhZBC6m9/75cBNM6+bpNtMaJlURhj3nr77aptttvtZpM5kTYA4NWrV9z39o/23ZR2c3Pz3nvvNV27Wt36PBglKUeMAlRuN9Tjn798ttjb/eLnnwwGA0AIjYIeAN/3D+Y7N2dngcfKpt5WNcRod3e3aZqD/d0vP/+CIrzZbCaLnTAI8tUmz/Odg8NO9D73hBCkN4p5XPVWSYktppRq2QWcK6cPg7CXwhoAwZu4gygIpZRKSCUAAABCC6xtKsU5L/MiiiKKiex6nzIvDG+Wa4YpwMhZGuleaGB936/rejicyF4QiFxHIJT0GJdS+p5XVRUGIEmS6+trxpjnsdvVKhgkAACgdLHZIsaZxxGAFgANDOPMcZmn40lRlRxTCY1LyyybJuUDd0MbY1rR+z6v65owj3M+GI+ANs6vYzQZJ0ni9DvIhW1y7/b2mlDU9U0P8Hq7CZM48OqmqJUSgzQBAGqtMYCU0k2WS63qrIl9P+BeXpRBHA3DMMs21SbDGBmjNpvsz/zqr5VlGccx5/z0/HV6lt7eXu/v73vc/9lHHz94+PDzF89enxzXdT2dzYwBzr368PAQY9x3EkKMMRVCcQoBAEbpwWBEulZpEQRBVZTr2/Xjt97GzMv84NHR/tXFyWI+11pCZH/+0c826+z+/fvT4WCYJs4o6nd+97cfP3ny4M7R2fFra0zfdo46rQAMovDHP/zRdru9vL764osv+lY8evDg9vpmPB5nRd50HfP4V0+fAt0zRsfTGSFoNpuFYbjebDZZ7iK8i6L0vMAluhBM9w+PyPUVIhgytFmuBrNZVzdnZ2dd180n081mQ3E1Go0sxhrDoiiGw+FiPoMQTqSIk/Tly9dCCAB017STnYWLdxwNhpTSzSYbz6a9FISQxWy2urylASaEhGEIBCyynDEmVT+fz9erFUSoruv7Dx9cX1/rXoe+37XdMEk5Jq1ol8tlGIbAGIsQRLbrGqWElDL0A9F121XGfBoPk6oqbi+XBwcHg8EAIdBV7WQyQQgNBoN0Mvj4s493D/YBAErKyWSyv7/vosMdvuRuua8t3aWUnufVdTMej8uysNYyQr9OzEiSpGvaIIoopet8Szk3xnDKcExl1wulkijuRO9u8uFw6Fz++7YTSmIhOOdd0zJnSpMOHC++LMvA94qm5hBaY50dcRonVdd6gW+lKsvSY5RTppSRvfAYE0ohiAAhQeBlRY4INsYEnldkGfM8n3tVVXGf+b4PIZS9rZr+r/7lv/L7//Kf9Qh++9vfPnv6DCFQVhXGeG9nN9tsqct6Fb1SKtPmi7PzXkmcTELuozybDO1nn3+6u7tomsbzw6qq7h0eWaBl3xEMPOx1XXd472FVVX3TS9k3TeN5bDhIADDL5fLOwZ28qm6vbjQw0/nk6upmlCZtUUGAm65TVlklPcYJxnEcdnUXhqFB+OHDh6vV7eeff/rg4T0lhJGaYVJ1LWHsu7/0S5eX16enp1EYMms3m1XV9avVKvY9AMCr1y/uPXjQdK0felLKuqxC33/77bcVtNssM1pTSsfj6U9+8pPheCRE19YdQ7hWQHR911ST2UwYHabJZDYt82Kxu3dydYGjsKnriHDnzna7vp3OZ8vNGhg4HU+qqlptluPhZL67AyC9ubnZGU3qPLu9vkSEoQTGcUw0RYRgDjhCSCvbdl3ic2U0tZASpnRvrEUQEUaUkYx5QigPe4DBvm+DIKjrOgrCqsggZQH3m7L2PD6YTruuq8qGIswpU0YTztu2HaeD5XoFEJxOp8UmgxZESVK2jdZ6NBrVZcUZa8qKEdKrXgJDCKEId103Ho+zxi1F4WI2L0Tbin4YDdqmwgRXdRF6UVlXJPClULHvQQ1aLYRWGOOubhjGUgrCWBhFnZbM86zWBIHtaulH4e7ooO/7ssjGo8Fms2nbdrVe3793ryxzo0Tft4QNqq5lPgtZUC6LTZ7NFnMMA8/jL5avrNKbbKsATIZxXZaiFwghxGkt+9CjbdN4lEklpJTz+fzBgwdffPwxI3Q+n+8s5ghiDGi2LdPRcDQbvzw7e316UpWN53lat6LvQ48vFjv7i32p9WBvoDT84ulLjGnT5GkS3N7edl0/ns8IpARRhNDdu/fnO3uQYgQg7Pv1cikR2Hb18bNXbdtqYG9/9pN37z189ODhsmuFksU2O3724uDend3xNC8KnzDGvOvl7Xgyvrm5cSEDnZAE4e12++rVK0bo519+YSGCGL149UqIvtjeQIzL+qfvvPOO7GSWFbdZdnZxGXhc9p0lrKzaNKi84fD6+rpVoteyKxoppRVKFDWZQnckGGOqopxO50EU7j+4+8lnn1ljrNJlXsx3FrfZJp2MHjJ+/OIlsVoIgXqFlU7TdHd3d7lcd0p2vey6ngKU1TXGSGg1no5W6/WbQLu6XkxnzOOUE2XkZDYtmjr0A59RjOFsumjayu1j58PFcrnc2d9RSj0/eR0FMcck9cOzmyvf50/ef4dSen55VRRVPIh5SJercu/gAFqCMKzK4sH9u588e9YLMxwkV1cX1uo823RtlyZJ3TSc8zhN8jw3BjgTLoe/N00ThkFZl23fTSaTrmmBsbrtA8KEEIiRWgukJSMU9DKhvK8aYTRhFABY1oXnBQghn3vOElIZ3SnBPA4h7JvWmekzz+ulIIi0bYc5V8YyTEPuV1VFGJNSKqXjMKyaBiEEMQYAyV5hjBEAzqgHY0wwVVJFUVTWFYbI8xgAgHn8erWcz+eIwG2xrsom8Uab5cYLwv2Do6ytlssbxoiyJo5jaUFf1qkfdF2XpikEWG82tVRN00Tj8c5ib4DC9c3tzz78UeCHWksCEWLAC5iWymf+dDpdrm471XHGNqt11dRpmtZlASFcbZaI4IvTs1EyPDk54UGwu7MDgO2liHhUFZ2oewzsME03mzVlpKjrar324qHHI6Gk0J2CkhB0cHDQSxuHCbbmxcvnR/fvJ6Px02evhBDffP+97XZLLA6iBGKw2WwCRqMoOr28yrLCERw+ePL+9eWllgoAsNlsjLVt03ieN0oHPuNFlkvZz8YzpbXFeN1VAcX79+8sm+LLp18ZJZHPLq+vxsMRYbxpGqNUttlUmw1CFho7H45ffPrFfDHN+mowGlKI67yIhhOPMtMKrK0lYDqf3NzcRHGAXGiAaxMsBEESFE1DGCMIy7514S9930vZF3XVSdFL+QvnJty2rdND+2HcK9lJEQRB3wvnq+6W+44RUdc153ybZ5xzgnCZ5ZBgTEnTd1rrwPez9QZjXNX1G68lJTHGkR8YpXkUCCWlk9kJ4RKI4iB0OjoXD6StmS7myuggCPquCzw/iWJOaMR92fXA2iSOq6ZulUgHg/V2I6UcjUaQ4MFg0DSNC7756quvvtZ5K6VWq1XdtWmaIkriOB4Oh0aqe/fuQ2MRgKPh8NWLV7uLHY8SRBiixF25i5LggT8ej6/+/zz915JlWZ7eiS29tt5H+3HtIVNWZelCKzQaDTFGMxjGxmZ4Rd7QjA/AeQGSj4EHoM0VyBkCw2EPBzMAGtVdXSIrs7JShXLtfvTZeu+lebGzcRdmERYR7n722uv//b/v990/+L4vlSjLXEt5cXZeFYVSKo2TrmkhRpzzVmjmRVleXt/erXfbwXAcpykAUEodBMHh4eHBZNrrIb0yW9c1AGA0GNZFeXFx8ckn3+/7cbJ94XnB6empaOuvv/zD/f2tUurl++/zIMyKcpvtH1fLVmmAYFU2fhhEyeBvfvV3VdPtsny72s5ms7OzCx6ErRT7/f7ly5dKSOdcn5/Mskxb87BYPK5XDsKu6/p1/367o9w3xqXJ8PWrt19++eXj42PTib7LVGuNEPjkk+8dnxydn58xxo6PjwaDwfvvvz8bjj3Oe5JJvwKRnfA8T2oljH7z5s1+vyeE9H1JZVkapfuW1x988r3ewWKEeHp2YZT97W9/O5lMuq6z2gDr+jppRLAyerXbQoKbprFGPzm/ODk6/g///n/ts6C73Y4SMh6Pe2kx8Lw4CPMy4x7N8xwBuN1uOyGePn1qra6qKssyYxQPfETw/ePDxcUFpdw4++bt2zCON6tVv/MMQ3+73wSed3ZyOkzSw9nB6dFxr333jQV9u2lfoFFVVQ+N6fdM/enph2Felr103lMetdaYUmOMHwb99ZwiHMdxGEfM4wCA8Xicl0W/uOoR9kop7vuUcyklI3QyGALrrNYAIYCgtdZIpbWOo6jHy/Q7FYyxEpIgxDn3ObfWBkEQh2FPGugLHuq6llKKtqOY9PVt1tqsKEajkTMWIYQpGU8nwLmqai6vbyaTSegHB5Np33m72WzKLD+YTqCxcZrkRdHWdRqGo3CwX+/zrFytNm+vrh4Wy82ugIRyxtI4yrKdMUZrdXt9k22z+fSgr8zNiz0laLNZrVar9XptlG2rVilTV+3BwUGSJP2KSClFMQv9aDwYK2UQgP2EaiHY7/fOuaartdMYw+vry3fXV0qZPM/rrt3tdmEQPXnyRAnJKH329Om//3f/XmsLKTt/9ryu28D3ldB13c5mM0QwgsTjwd3dnZTy7uG+b1WjlJ6cnnZdc3d3k2WZ53mz2UxKuVwue/Id4PTf/n//pyiKeiyr0no4nSilAsLKXZbneTIa/vDHP4r8uC2qYZJ+8OF7nZRZlpV51ReubdbLvj4XYaqtXS6XTdMAANDA8yPmlVnOGAPQaGeAx3dNxX1GCZatoJQxSjFC09G4XwcJLYSRiGGl1Ha7tcA1XYsJI5R/96AagxjrC4+MVARhjLFxVlnTSgGcCz2fMW4RshBgjLEFxEFrLfZYq6TQKo1iqC0yzhiz3u+qro39IOCesLozapoOddUK0WJGhRDD4VhZs93v+i/JQNBKUWU5hzjgXq8/ZlkGCUQEL5fLOI6p71Vd2y9Uu6o+PztzAAzGo29fvdJae4RapaUWmLNayIf75XaxcdJihj/9/LfRIFCiLvfrZ+fH2si8KpMkaduOeyEi3FrLGIm4RyEcRFHT1IBgA9zxbB4xb7VaDUZjEvheEkttAEYWAG3dPq+jcODxwCjtMdY0DaO+FG6z2VkIpGrqpgDWCSGatgLAOgc8LxKiLYqMIKiFOjo6Ycz75S//5quv/lAUlVF2s99I1bVV+83Xr8q2Uw5sNtvNZhekiRfFr9+9u31cdNY6yiyh589fHD95cvzkXCNw9uTi9uamKcrRcEgwJgDKTlhrWyXytrlfr3bZvq0bo3RZVsvFriw6ITTGrG5V3QrZdqJpWyEww9qo65urpqmarvZjL47D+WRsOjkdjkLmLbZrCaznsaYum6Z58+7tV6++zoo9p+z08KjT6n69pJw9f/58mg6fn5zHlFitAaU0CBACFIKP3vvwZH62Wa4mg2Ho+wDArus6KZXWfhA4gBxA4d8fT87as5NT0XbPnz3hjNRFqaTc7TayE6vFUkvlcb7b779+9a0FjmES+H6R7Z1zWVU6jH74wx9uNrvH9caPk6u767qrtQXPX35IOcMM+5w+3t9nVesFEYVgEPA0TlarTdu2RwfzvnHp4OBAKUUp75kEjLF+oYoIDuOob3ExVlktGcWYYQOtcjpNY921ozDExjho43FaqG6bZ23d6E4GQZAVeTyI/SQwTlNOnNWM4B4V1x/QPQizf/5ZX93nAFRGCKG0btoWk75/CUilGWFdUdd5hRBWRmdZFvpe03UWOESI11evYKyEzPNcKVW3TRCFZVnu9pvNam0tIIRFvscpuby8rtuureoiz7uui6KAMco5bYpqs1khRMbTSStrBByWzpbq5fkzI81mn63zPB4PGymaphGtAAYwxk6Pj4/nhw93j2++vUyj2FqLgRNNHYV+FEXj8Xg4HCNE5vOjviBwt1sThsq62mW5lFK2XVYUURRRQJCDTddx3zs4OGi7GlFECIqC8PjwZDyetlJI2R0fH58/eTGbzXfLLQeIA9Tuqw/f/yjwo1xWlkNrbeiF0+nBZrN7c3l1/7CAEE8nk6uba23teDodDoeqawOfV3XhoA1C7/jkcL3bbrM8jiLf8yCEd8vHu8eHyWy6364JAmWRIei00w4CIUQYhlmVT47mN4vV8ZNnF2dnRb7/8Mffb0wXh8n50dnXr147jIDRy+WjItCfDII0fvHe89FoAIwmVqpWyXQ4LsrSWD0OgkZqB3FeZIHnQw81VR0EQSvauq77y6ZQEkLYSREnMYZIaqU7AxBUnWIUJ8OB6XenznZSMMYsBG1TAwgppXEcq7Zrq1o76wcB93hbN6oTaRQXqnPaTiaTqiyVUsQnTduGcXT7+HB4eCirJk3TRgoHwHc19s70poLvZluteRAoIYXVk3TIGdNS5XnuhwHihFjdFDnFCPn+ap2Pg3FeFsN05DMvy7J3794NR6Plcsk555w/3N61bZsMk8vLy/F4fHBwsF1tF6vlj3/4o/v7+5OTE4hcUeWr1coAOJ1Ov35z6Xl8PJneXF+HjIdhSCm/u7vLdntMSTBIizI7mkxGSbpeL7GPLXCt6NLhQEjN/cBBeHl5dXZxfuBzhNDlu6sNWAuhILCYMKG6g4NTxjwI8Ke/+TWlcDAalFmeDobr7W44Hvl+6ICFED48Pj4sVwghrSxnxIiuKMuy7SCEeVWGYcgdfPn02fhgerdavHr9LYQQYmSB68MUhNE4id97773F/QNnPI7jo6Ojuq6TMMnLcpftISXa6CAI6n3eL0jCMBRCWefKosYYa861kUEYIgy9wJdKJkkyGafz43kURUopJbvpdPbw8LC8f7AQ+Gl8eXfz9Mk55/z15esoScaHB01VDybx43q7Xa0dAte3V5yR6XhcFAWEMIjC7//wB1988YVo67eX74TSx6dnd3e3FjiKqc84nUzuhQAWNHVtEUYIdUo2jaKUrtdrQkjo+X3XqNBKbrez2ayp2uXD4+RgkqTRfD5X1p2dnS0eHi8vLx20QZRgRtPx5NWrN+89f2EgapqGEjKfzqqm+/TTTz/+8H1rFITO933IvV1RxnHMGHn15t33vve9/X5/d3dnlfZ9XytFKW3adjAYLJdLxthgMOgrL6SUWqrBYCBEKx1omiYIQ6NVlMRN00RhWOwzHvgOwc4oxCh2jmLi+37b1b3hvW4bChD8+74Oj/FWiNl0Ch2oyzKK47ptMCXaOSFlwD0jlRd4nRC9I7Of4DFCRmmGSZqmnRSIYEJQVuRhnLZSAII7KayUPTo4DEOOiXE232eIYMpYKzrfZwRhISpnVF3XH3348ae/+fWLp+dWq+Vy6QwA2ErVpWm62e0cMA4CL/BFXXddVxX10dHRIBzc3NzUXUkIIZjJTllGu1p8+vmnZ4fHx8enbdtS7t8/3AZBwDxaliWlfLPZ+WEAHJRC7fKsqIvxeNxJkZelFwRSKYxAGEfbzcoZQwgbDIa7spkM0qwWiNO7h4fJZPbNN988e/YijuM48HfrDSM8SQZFkR0cHNw9XGptLy6efPHZ72vXPd4/wM6kz5LlZh1HaUhHfhxSTCgmf/EXB13XjQbDqqruHm7v7u4gQ0EQjIfDrCg6JZEkZVkul0sYBrP5ATbGwzTbZ4SQNIpb0fW+4WqbQ+SGg1RZo4xRRnPPm0ynn3/+O8ZYzEKjLAYwiiLViePj48V6U1XV/Pjo6vp6MhwNB0OEKfYC3ziLMR6EcVvVTkkELLYAakshAs4pra1zkGCtNeWk6xpr1GiYCi3ypui6hmIku5ZSjAnRxmAERV1LKQFGLPSbphuEKXHYOVfWFfM9jDGllFHa1o3RWjtbiZZSyinbrzdaSAtB2TatUYSz44O5brqAe32xbxhHwhnssYBxAlEUx2EcccIC7stOAYRHyaiuW+oHgDHMqDS6kUIoGXE/8YKu6yxAbSMuzp44AzCm50+eEczubx/G6TgK4v0uPzw9qbq26yQnlAC3364dheOD2Weff/r04mK7zb76+jWL42Q69fwwy4p0EAOof/vpr6IobNqqKIo++nwwnVpra1ErpSLPZw7FQai1Ng5+9vs/jCazqm0sADc3N2HkMYqgA3VZt3WHECEE+aEPkMuL4u7+cZCOsizLiz1lWBuZJEmWFcwLm1owRqbTYSua5XrTdGq3LzCBnOAX7z0P4sA6vdmuEIFCtKMkfvb0YrVb/U9/9f8RTa3KcpjEGNm63M9Gw4CQ4+n0eH7Ytm1Pbbu7uiYAlmU5mU39MBBCyE5URQkpkUZLKau8aLvOONvTw/O64WFUVyXFCFIaD0cnx4fO2NVmvcv2Pceqbpv50eHh+XE8Tg0Ffhou16vT09PhMIXI5dvd8fQgJV6EWeh5wzQF1q1WK6EUpMR6NBONcdYPAxYF8XgYDwedkiwKOq3KspRt93BzF/vhMIyhdgzi3iOLMEjTNIpjAGHdiU6oZDTGjKaj4cP9PQAgGQ+1dYvFAgKwX23+7u9+7UUxD6P3XrxvpBqPpnVdB3FUNnVbNx6hEfGbvGYIBh6r82K/2T8u10dnp1J1yshVlmeNcBQXdRWGYRynx8fHR0dH/eDbUx/iNPWCoG7bVghKsdaSebTrmp7gGEYR5Uxolec5wVh1gnCmtPYCv21bnxLOCOVMKKmNiaKI+57v+yz0LXCMMc5Y4PlpGGVZtlgseBgURUEAJAABZz2fK6MgQQQhjKAfeBY6bRSCgGLMKYEIdF0LgKuqymFEPO6gpRj11lLMGY9DaU0rOillU9WHB9Mw9I2ziBLnnOqE57HBIF08rv7ub381m07bsupFVMpw4PlhGPbqoueHtWjud8uGmEy1BoI4jJXs5tOJURIAoJSxDvos6LouHQ0Rp8vlsizL3/3udz3HePGwLMpaGW2cvb25k1JmWTYYDSFGxlnP8zhhnDKIgUVunW+FkTyMtLZt1SJtNw+LmHu66z58/31rQduKzXJ1c3klO3V3e4uAJgS0bXt9f7cTtea4Fa2S4h//0T88mx5Np9Pru+vVbnt5c00Q0p2UUq426+Oz07Zt/+qv/mq1WWtnk+Eg5B7QZrlcdVIORkNl9ON6c3h6Nh6PtVSDIMqXm0kyODqYL5dLZQzl7P7xwQIQJYmWavOwYIQQQoQzlWhP5oc//vB7k9HICDkdjrqqnhzM7hePEfeHSXp/f2+AS4cDhyDClDZNDZzBCCVJAq2jmEAHBoNBFEW9Q9FBYK1z2gBji6LgHjPG9KjoPk8IAOCMOWu11tqouii/y2UA0NerAwBGoxGEMPB85EBv5pVKSSEiPwiikBBitDZaJ0nSZ1m11skgzYtCtF3o+ZBgC5yztq0brXXf/KuFdNYWRdHDuy1wDsJWCm3MYrOW1mhr/NC3VnPOrVFtVfchLM/zXr9+3RPA4zjueyn7VUEYRzc3d1LK09PTo6OTxWLR/8932Z4yr6zrn/3856PZ9M3lFQ/8u8cHhFArGqlUGidCtJ7nPa6W8SAN4qgoCgRg4PnD4bBrWtl1mFGlFIHov/in/+zx8TEvKkqpsYoQRCl2Vk8m4xcvXjBC+wVDX0o5mUxev33zi1/8QggRRSHGyFo7Go/jJDHAYYiKPG+FwpQRz++0Koqs7eogiuI02e33nBKCMEHgpz/96Waz+sO3X0sju66zVvdU9KwsfvnLX2qtt9ttVeZ//mf/0Pf91Wr1u88///zzz79r8yGEMUYgAgDURdMbUfwoxJQoazyfdbIjDP/nZs6yLIUQAKDvf/Sxs1B0qg9ANlU9HA7nx0cQIwSgUgpA+Ord29MnF+Px+GA8cUo3XQswmoxGshNt07RtCzHyAr/P6CdRPBoMyrLUzg7HIwd6kbENgzjPy8PZ3Eq13++HwwHlrL8XIwicsUroNE2lUmES996VKIqSJLFaG20hhKPR6Jtvvvnke98LPP/NmzfpYPDm7Vvf887PzxEhSprBYMAp9bnHKfcYF3/vLzyaHwIAvvnmm7atozAMonAwGs5m874Bg2IySFLVCa31YDDwudf73P8zc7Fpmt4F39PVy7LslxDz6SyO41637XFsxpjA8/veDAihg8BBUHcthqjHu1OP101DCLHaKCGhA8PhsH+ppGkKrMUYU0ystRa4oih6aiawVnaCEMJ7md7zIISyEwfTWZTETdf2ARlIsDRaGd2DcbzAJ4RQSou6AggKrZjHZauaqo1Dn3EipQyCKPHDOAgJIcfHx+cnp23blmWJCBVKEULCOGEeF6pDBH7x5ZfL9apumu1+izHGAHMWcOp3TTObzSCmyro4TSCEH33v+1XdWgM484+Pj7f7XGo9Pz4cDNNOtIgQLwzWu60xxvN50zTbzcYYxRnpVaym63zu9d9/55yz4OryZp9l73/wUU9RNUq9fPa8rus3b96ESQghLPbZwcE0r/LBaPj7zz7PdntrtbZmNptMJiOlBMLgt7/99S7b/ut//a+rqnr+8sVut9tsdtvt1mM+p15VNU3TpWn69OnT45OTwXCIMT6ezSPqPTk9y7Ls8vIyiqJnT560dUMQ9uJwk+/Lsu7q5vHx8fr6WmvNGFs8rl69fZMkyXA8qOu6Khsp5UcffdTWNSM0TVPn3G9+85uHxSOq2yYMQwbRIIwWi6Xn+VZKK2VeVdf3jxYCiJASkmJCIPIo8whRoovjCBijhZBNo5V0wDpnIQQEAOKg7/t+wK2WyDigzCgdCKPX2a7HQxOEKaXaWW0Nw6SpatF2fabJWgsJbkRnlZ6NxlrIOIoIo62WCljMaF2UDGGOSb9ZiuOYOMQIM8ApZ4XRWZErZyAjURJraxyCXdNSiGTTCSG8MHAIWwC1A3EysA7WVXt9fUMp8zx/uVwppbfbbZqmx8enr75987BcTCazumqVlNPx5PjsFDLyH375Hz/+4Uc//Qc/v7+/H02GQRw4BJumQwhpKSnFhOFvr9/drhdFVUVBoDsBrEsGaaeV1pYRjo1TdcuY5/v+frsGRp8czQGw6TBBBG73G4QBJRwaQBEeD4aXV28/++zTzT7z46Rp2p6agAksq8xYOR6Pnz19r+vU/cPCIst8AoB9+vRJUZWXt3eEe1Jq26qD0Qwg3Fm7y4rtPjMWJKOxI7SSslZKAlB03XK9IoREof/zn/1sNBo5gvKu2WWZMWY6nnBK+w05paiqqqKuqrYxwDhniqbBjDigndOMMUp5HEcAgDwrV5tsPj8ajSaj4SzyE+yAFlJ1YhSnB/EgABghdHhyrKFL0tRau95uciDIKHTwu/aoMIqKKg9DvyvqELHZYBhQcjo/OZ2fPNzdAqOPJgeyFqvN2kJQFAUmlAVhJaXDKEnTuqqIg9iByPP7Pqblctm27enRsWw7iEg6HMZ+EPGg7TrGeZFX4/FkOBwFQTQaTRCmVVUXeTUejwM/kkJleXGzvPeTKPCjfg/8sHiMAm80TNcPKyMVo3izWj/c3A6TYde0URT94Q9/GI1G7738gBBGKR2NRn1E3g84woD5nrKGMWKthgQnSQIBoITstzsIIfc8oZWQMooiBOB312dCtDWIEEAx86ho2joriIOMMciI0Kpqysl0BI3FDhil+0Vu3wqitU7TGFPkxz5h2OOUIpjEISWobitAoHFaqi6MfNE1VVUdHB12SlZKKGvS0ZBz3taN1cZam5dFEIXc8wBGzjnksEc5ArAsc0JQVpRS6tFg8O03X+92u7qut9t1GPmsd1prjRCGDomqGcYhNLIqymgwPHx6Pj077inEXSmcduPhYL1ZrrcbRGhP3Ftvt0GUSmm6Tt3eL5jHx9NJVRer1eLZsyfaKmn0cDDuOtk1dRpH0/FEtUpWXb7aeJwGIQPARlG0zvNFnlM/KuomGaRNVz8sF3XbfPHVF3/4w+/vHh/qrv3DH/5Q5Pkf/+CHtmz9INgVuQWOMNy2LSHIATM5GOZVfnX17vTsmCA8PzxcbtatUNaCD9/7cDyY5kXTNspZnCbjOqsCL5xMJrvtlgG0fVzOxrPJbCqBlcASzraLVYJ5RHnbdcODKWWsbOqmKeLY/+b3f7i7uVcQprPZcrvaZFvnYezzUToBCj578hRD6HPv7PhsOpvd3d8j7ay1liAgpRxOxkJ/F2xDmGJOZf+W5hw4RwEyUmKEfN8viswh5/u+VIoy1t/NZdslUUwQZhTLTiAAOaWEMESJECJJklE8NJ2si7I34SilKCZO6T5M3H9k267rOXZCCI+ypqw6JTGjVV0bY9I4EW2nteaEWgga0SGEVCf66lHOyGg06pNQzrnBYNC2bb/RJQhFUSSEwMB5jHddlyRJEodVURCCMUbD8fDZs6cUo/VyNR6Pi6rM69LzQ8y95XIZBeF6tdhsNuv1uqz2/91/9//47Le/6SO6m/1GCBFFkVHKY9wqy/l36G3ue57nIeNmg4HHOEEYIZQkCbDuyy++2mw2r1694oF/cXGx3edZUdZNY4zxPK+sKmOVg04pVVVVtsujKHHIIQIRggH3ojgI4iiIo+nBbLFYPX/+YnZwmI6GvatnMBhUTfvu8rosa2MBwpQgTBC9url5eFw6bXwvxJQYiCBGWVVz5odhfH15dX11a5R+eHjwGU+jOAzDvCi0kbvdzvf9KAgRQhRhj3OCsTEuiiJCCELQZ4xipISSqtPWCCW11pPR6IeffHJ8eHg4nY2Ho+16nee5EhoDeHH2xGO+aRRz5OHusa9YMsBhSscHM0fQ3fKxqMrtfl9XbW8Iebx/8BiHxmS7zctnz2ej8cPd/fHRkZTSaXN8dFTXdRzH2tlGdBAhREndtUKIOIwIxt/Ni0q9ffvW9/3j4+NXX3/DMOm6rizL3W6HMWqExIxu9jspZdM0WqrNfpckSZ7ncRTVdZPv9/tsF4Q+IuTq5rrrOspZWVeLxWK5XF9eXj45P796d7nb7SByF2enwJooCAAAySBmvlfmeV1W3GeYYc45sDZknqia2At8xhEivcuo9x0qpQijUkoHAWXMWivaDgPotOljUISQrqniMNBSjYbpwWSKIVKdCILAQkAYLauKMNZJqa2J06Ssa2WM1MpamxVlD3pUSvVsKGOMVMo4gAg2xgEIKWNK6ziMyizv+YvAuX4f1hNyfN+HGBvndlnelNV4MMz3+7ZphBB1XQutPI/tdpsgCJ49e6aF9ijj1IuiCFOinUYIbVbro4OjYTqq65JzVhV517Tzo8Prm5vZ/GA2mxNOlBKdFBC6OE4eH5cOwnQ4WC6X+3yPCeOeFwS+lPLu7oZznqbpvsgtcGEYVk0dRJEFjhCEHBBNF8fxfD7v6ooiDDEquyZOh5R517f31sGzs7Nvv/0WIXR+fup5jHAWhqEX+NODWVGWfdPLarUKojArs35ZQj0exsH19XUnW6G6tm3Pz04O5/PDw0OIXJiEb968EUIoIYuqjKKo6zpnbJUXfSz58e6eEAIJbqRSwPIwaFoBIfY8L/D8w8PD9XoNEPY8z2j95vXrKInrplHWEZ9LY5U1xydnfdby7vrGAld19WQ6krLr2jZNEoQI1E5rjBslqrbJ2xr7nqOkaGppDaKkU7Jvue0X7oTSThtAKKTMEEQDr2gaQllf6iaEIAhhgDHAyCFtXdUJAPEoHdlOIm0Dxi0EnZHOOZ97WophmjprYt8fj8d1XfdtDNRjbdu0beN5nGHcVlXg+xThsmoo84wxRV31UkzXtRY648xms9ZdB6TkBFNMmryE2nLEPOYBAxBwPmdJFHiUYeAIw7t8B40OObu+uczr/OjkECKHnX16dr4rcgnBZD4nHrcQWYiwRci6NAycEaoVkecRDJM4vLu76yPdnDIMiGiEc9Bqo5uuzUsLgXY6229fnp9zCD2CiXNWyaIqk9Hgv/kv/+vQj6jHaRI3QoZR1H/to9HI90OhhIN2enwAEYn9gawlAWA+GQ3jxGO8t9C9ePne43Jzf/v4cL8AFAKCrNKceOcXL1vp4nhIKKceJ5x98MEH3//4e6vH5eJh4ZQ2Qk7GU+PsNtvWZW5awSxEGjihbi6v7u7udrtNXZcEAs5IvtsCZ7bbNSHI44wSDACADvRgLGxcQHhAWFtK0gdbAs/zPAeBlPKTDz+eD8bXb980RW6UPD6cB9yry6Yp6tFg7IfRweHRfr1bPiwn41lZNcLqrCyAcicHx8KYWgnlTNuKKIijII7DYD6bcsraunl6cZ7G4Tffvvb8UDrzsLjvcSIWOMqZgzb0OIXAGlU19ezoMBkP93m23ezHw0lTdzeXN4EXXr277LouGaR+5L26fLvPsun8MEyTfb6fjkdXV1dRFJVlOYzjyOOcEq0lQ3A6HgEADw6Pyq7qtNru93E6FMq8fO8Dh9z7H7734UcfBWH49TdfWN1Rj46no8nxTDt9MJ396PufEII4p0rp6XgaUHJ+cBCxADvijNXa+sRjhFuIWBgxL6DcD+M0LyuK2WgwJhZ0TRsmMWTEGR16nBoTYqS7VjQ1Q5hABAzwvRBiWjQt9bkwyiJYNDUPAsyYw1gYzTkHFlptPM+rOhEMBgZh5UAcJWVZWwcsgOXfd+w4YxMvSLiPlIXaxl4YeB6l9GH5gAgWQgSez6mnOhH7IYbEOQAIb1oBMXDQtkIAB6fTg5OTs6pqnIXbbCusJAR7lAWev1wufd8nFAHTLW+vVd3+7Ic/g4DUbaOBdMwqq5SxjPDpaEoYk1oHkU85qVVrECjqfToI5wcHRV5JbYuy1FL115Gqqfst9H63G09GmLPFdj1MRx7zd3kBMJFSZvvthx+9P52Ot9vtn/7pn/75n/yxU7LXwbbbvTKu7EQ8Hr+9v19lO8Jo1zU//sEPy7yYTCac+/ePy02WT6YH773/4XK5vLu7+/LLL66vL+M4HI+HP/n5T6IoIATFYaCtWiwfoijyff+rr77ZbLcff/whRO7XX3z27u5mMBggALwwKoQwBFai9X3/o/c/0lKM0sHp/OTk+Oxycb+v6zSKdS03270fJPm+ODs6C4JgPJ9Yjk6eXhBKGcVJ4HMIEcekV+K47/UyX17WVdMRQuIwskp7mBKM66KklOR5vi9y51wUBAgABABCyOP8O97ecGCB22VZ1TSEMcaYg4An/r4ujXEc+8YYhHErWgAAgagtK597SikIrHOmL0jsZfoekgecMVr24eyetswYE1oRzkajEcMEWmeAo5xZq9MkCpkHtM53+8Bjvh8WRaW11tZ0RlngyrKUXVfXtRRiMho7baq8UEo8f/mM+eyv/+avr2+v2rZN01Rr0zvK375919srhRBWaWttti8ODg6iIOqt9IN0NJ0epMlwv9+PxoO+igwDHPpBFEUY47KuiINJFDNOIUZN0/zyl7/6N//jv/3Vb37zf/2//d/3+6zr5GeffdY0zdXVlXOuLEsAwGAwAADEcWitdQTtihIQyhgp8r3PmdUy8LyDybTY7UUtLp482WX7+9s7q5XV5mA8Mc6maYoQGg9HYRieHZ/EcVw1DcbYo4wgMkyHDtizs7N+GMIQyrYr9lnbipv7h1Z0m3y/K/Ntti+KQjrTdLWDoBWdH0fD8UhIiQiWShdF1fe6AYzCmKvO9HgTACxjbLfb/dVf/ZUF5uDg4NnzJ2fnJ0WVJ8MEU0IYQwQjQqU24+Focfsom/bk+HgymYzHY+jA4/3DYDT0goAQcnp6ulmth2k6mYw8z2vblvtenCbP33vZNM3d3d2XX3757t1lkeVKSG1MJ0TbtnmeE4gowlLaVnRlXQ/H4/7+0YNlTk5OJpNJFIRaCYhR3dWT0dRqV9bV+ZMni9Xq+PgYOYAQenx8vLq5CoKg67oX779YLpdGquV6VXTNV6+/1tqWZXV2fvH67eVytbq6vXr16lVZlj/5yU8uLi6GSep5XhRFCCGA4Ha7zbPMSHVxeuac44QiAL8jqlvQd5b2ZjDnXFGVhNH7+/u+nnS/3bZtO5vN9vt9kiQIQmidM6Zv6QPO5HnueR4hJM/z/mGpmiaIIohR/xS0UkgpwzDqf7cv/4MQ1nUNEBJKCa048yCE6XCkrWmaJs/z/l5V7DMEQOD5Un2Xhh0NJ/0X1cfEnHYYQqdNHMfGOO6HCIHb22vi82/fvdvudxDi+XwuhIiSFEHieZ40mlPmcc45N0YBaJeLO9mJo8PDH//oR14YdLoVWmhoLAB9LbgDoGpq55yWqqkrY/Tx4clus62qSkppgQMAPTl/mgRJPwRjQtq2HQ4Hou1Wm3WcpovFQms9Ozo0zmFK0sHg+vra87z9dveHzz8riuz169dlWUZJDCDc7/cYkr5tqmkan3taqqcvnldN3dOwt9ut53l5nn/z+tXLly/3eVaWZV9i/Prd26+//cr3+eLhzue0rsvRaDCdTm9vb/f7feD7vufleZ5lmUNwt9thStuue/7ey022R4wuFwuO6T/9i3/sMT6bH9Rt89Of/vS//m/+q88//3w4HA6HIylVmZfffvutMWaz3/lB4BB89+6dliqg/uF0hgZeiJRlDtpWhpRXu8znfJAkHqGzeDAKYh9gZiFHpPfMjtJB7AflLpNVA4QBwlCAuqZFGFdtYzBkSWgpqmXX6s4RuK0yx0gnjdAGR/6+LeNBHHEfW8Qg5dxTRitrLARCiMFg1K9ourrxuedR1pW1MyaNE2utdtYAxzmv63r58Ng1rbW2b5xnzMOQ9H3zfhQqZwFGhDPIaSlajUCtRL+k1c5izvLtLvECY0w6Gi4fVxhghpmxIG+7y/vbYZKOk0EaRNPhcDIcPH9ycfdwm9fV9f3D0enZ6zc3jEaLxx1GPoGkqzstpVHqYfHIA96vEDAlDkIIoVVuNpqenpxcP94DjiFmx6dPPvroo9nB5N3tddV2N++u89VeinY6mSCAi6JCCBGCpuPRIE210GGUjJ4clUAEQWSUTocJY+R0fjDwvbdffn04nTgCLDTD0Ddl+Uc/+VEYeDdXl7PJ+Ozk+E9+/jMC3OnxyYcffXx7d7cvci8MprN5Ohhx36/KXNaCY08ZYxCAjBRNHQ7Tq4eH3/z+999eXgaDBPu8NQZ4Xtk1NAoMgZloJocHBkOeeDTkrVTSaOqz+elhOI6aWoSD5PTi/Ozk9PDw8GG/+p//+t8HSborytVuSz2+K/OyrSxBwuiyqXsdSTTt4uEx9gJgbLbdxWHocw8ZZ4Uq82K9XEVBIKXsuq6sK+571OMGQ+Z7x8fHZV5QxhhnBGNKCHCub+Luf9yEkOl0uNls+uaKvsNaCIExvrq6whAyhIt91in54x//WLWdKOu6rLIij5JQG+mM7Zo2HMaTo5mDsBXq8vZusdlKJSglg+kwGqRKaZ/6q+1Oal3UVdM0CMKqLHfr3bdfv8q3eb0v89V+cb/otMAe+ej5BwyQZ0+fnJ+fjWYHPI69KGxFO55O6rolDHsek11rtYxCv2vrYRrHYTCbjIbD1FHcKDEajbarNaVUKQUQ6jN3ECONAGCkqKokSXrlE2EgZAuNPphNgsDvZOcxarXqvvOtY4TQIEmcsVYbRqhSCmDkeZ5sO+gAp9T3uZCyrNsoHjgIpZRFkcWDmDBmnOO+Z4wqyxwhQCn1vQBjaJSEziJgD2ezbL+r6vro7JQxtlgs/DDopKSIaW2Xm42DYLPbnp6eFkVhtWMY39zc9C3nv/vd7zjBcRACAOq6ppR2XbdarRywT56cq1qC1sxGYwrR480jdoxAxH2WFXvO+Wa1AtoiA1Tb1VVllc6yTGsdUO4x5vu+cXZf7hslNHDbbG+cHqTxIAo9Qq218+Mj4+zd3d1gMFBKQWORth6hdV5wTNqm+R/+7b/Z12VRVxDC44P5ycGhlDJJkrZt+9cApuTN1eV4Nn377uru7uHk5MwZEAXxbDZfrldpmn788cd//PN/sLh/GMRJU9VdXU1ms8flIhnECCHm8TRNu7q5evXtm6++IQhlXff0/Zcvzs+L1dpaTRh9/vRZ4ocXF+dN02DOxuPx6u7h7vW7H33yg+Fw/Hi/uHxzhfI8t8aItmOE+L5vjHHKtHVDHFRKQQgNcK0UfhT2Sc4iz53SHmVJFBMAOWMQgD6Al1flvsgNcLXoeOBDShCB48EQOoMQ9INgtVkng1QI0YmGEOKHcdl2wmhpJOXMadNndq1zQRx1UtRdGyVxlCRd11kIlFKYkqzIOedRFMVx3Nsn+k2sMYZTJoQoiqK3avTV49rZHkkWhiGEsNcZKSZt00CCt7vdcDj0GAcASKmFUVVTv33zxgj5+eefTyaT169f39/fSynT0bhp2/Vm9/HHHx8eHqbpYPn4eDg/rssm4uFsNuO+74dBX+JOKZ3P59PRuGuap8+fZ3kZxtFmtxNK/eBHPxxMxiwK/i//7X8bxNEXX/5htVotHpb5vujaNvB4JVrIcM/L/flPfjpMB8poyrlz7vzk/NtvvwUY7ffb1eMiiqIPP/qA+97d3V1dVv/oT/+sLMu/+Zv/BDG6vb8JfP/rr7760fc+icOwd5EeHBz0ZM14kGKMIcBJGJdFkQxS6nuUe34UrTe7oqpWm812uyWMB1GICPF9n3leX9E5m82EVrPZrJ/MgjiwEBgIWOC/9957zOda6yAIXj5//ud//ufvf/gR4d4fvvnaOjcej/vdzPHxcZ8yffH8uXOuRwFb4L748g/OWJ97/dzme56zthffsqIwzmVlIbRq23a32xljhsOhlPL12zfr9bq/8PZqVf+vKKX+HtC/n81mnPP9fs8Y61NsPdBRWxvHcU+I/eqrr3reepIkZVn2uU1CCPN47xx7XC4Gg0HXysDz3n/vA4/xq6ubk5OzMPyum7uqqsFwPD86Xi6XGGPCvNPzi8lkQgh5evHsxz/+8WA09KMQGBDx8M03XxOC8ropmnY8Hc3nszAMp9NpX6ARRxGByKo+/Q/Lonh4eDDA8cA3zvb7JGDsYDDgnNdtAxB0EEdBaKSKw9AZQzGWXROGYX/rL7O8q+phnIR+QDEJw7CvTDDG7HY7nzIrVW+FghACADrRUoiANX0jguf5fQlwGEc9OrSPtfe36cEo9UNPW+sQwIz0acEsy3xOkyTKm+r1u7dhEvu+P5vNwjDo6jb2QgppX0SVF0UPUEvTVAjx1VdfXV9fD9O0D+72C+SmaYCx4/Gw67rPPvv9aDhpGyFEOxoN+u7ApumstYPxiIcBZ17AvbLI+7R5mMQnZ6cY4zAIRNV4jPeXekoxoajnWb57964sikGcrFarZJB+73vfY4zleT4djUM/yHb7zWaTJMn97V1b1ZPRGCGEMJZGl2WZJEng+0cH8zRNLy4uPvjgg6IojFTA2MDz+5dlOhz4vn9zc3NycnJ+fr5db+7v7x+WC8bIP/nH//ji/BxidP7k4unTpwBYIUTb1u+9eHl+ft6KrtNKQ/P26rLcZ/fXN//sn/2z09PT+/u745OjJEl+/NOfTKfjDz/6CEI4nU5/9Ztfbza7i4uLKEqQsI6FEaZEKLUrc+px2dTEWodgVpeFbIFHKy06o/ww4JRFPIi9EAOkhcqrsmf2t22LEQq4F3APAOD7vFOdgaBpmhgzrgGiTjrl86DMSz8MGikaLTunBQDAY0JJjGHftt5vkDolDYbQZ7e7VWOkP4jrphkMh73HwDonjN43RasFRcRDtI+nV1XFCA2Z5xHqtLFK71ebYRibVphWIITCOPI4t0ozxuIkUQ6k4wnBTErtHKTcs8Yxwk5mB4+3t0+fPXlcr4BDdS0Go1k6GGRZ5vucMwKgtdaEcfzw8NBLwLLVeV7tshIxVlSlqJpsuQ65l2VFMhuj0Ds6OOKQzCaj//iL//i//fIXr66uMKM/+9nPkMcqKbpKPN48dFWJMIAMb8u81XI8Hufb1Xw8OD84AdohhPdFTrhHKU9HY+Hc+z/4fi3E9eXN8mFxND+CmP7dr35jAMyq+tPPf58X1T/8s3/k8wABSDDyfC6lPju7OH9yFiWhMVYpvd1tCCO7fRaEURDG+/0+2+0D6ouiAcrUWRFQP6Fevcuc0ggAD5JmX1AARV1j52LfZwQfHh5Sz9vu8qYqfvzJx1Dbu+u7bLfvmtajwdHhcTQMF6vHNEoPRjNZ1sg6BMDp8TGxTrXtviwUBm8fbifzA2MMQahpqu+4K4zlddVoGUTharVq23a33gyHw6ODuQ/J/c3tdr+LhwOP84PZzFFYirp3YvQcvj6YCiFACBlnKWdZlvX4AQDAzd0t9711vp/MD4ZJ+kd/9Ec8Dhe7zXA8Ojk6ZpRKKVnI67pcPy48RKADTVXFvjcZjWWnnAYHgxlQzvP90cE4TWPOWVU1v/3099PZoVTmfvWw2Czvl4+dVoDQq9s77DHMqEPIOeeM3a2WGBNKWZ0X2Woj6rbMCwuBlF3o8YAzjKHPKKM0jeJkOCirinLWyyzfwXuF6LoOY6KVBcaKsqYGeIQhACF0EEKorREdIcTnno+xrKq2rJSQdV1CArWzSimPUWe1z7mzhhMKACirIgpCn3tG667rMGUWWOPMaDKuqgoCvHhcaa2SJJ5MJgCgqmo49wG0rWh4yA1wkJIoTVrZFlV+v1ykk8lyu2Oe11TlJBkcDMc+5gHjz8+fhGE4Ho+DICCEUEw8Rneb9cFknCQJYyzLsun0AELsjOk5KM6CwXB093A/Pz0UVn7z9usgDBHBxjnuB6t9jj3+6vLtercZj4f7/dYhty/y9X4XpxEwmiLUVEXo8dgPjZDYgtj38jyXWg2GQ4BRnufb9ebbr18lUfpf/PN/fjifSym5752dnflRKJ2ZHx998t4H03hgrT06PVHQbYusv+RNxzNg4e3V7bOzp8fTQwbIR++9//z581pIyL1oMBxPZteXV1fvLpumefPmzfzkuGjrr775ZrPZ3NzcdE27uH+4v7k9ODio6/r2/u7bq3cm8K43i6oqvv/hB3f3j69ev1WdqMviT/7sjx1Gg8Hw/Pw8juOyLj788Q/O3nv+4oP3D08OhZWYIsSjwBFkIOjLVpxzFjhMCaZEKKmsMcD1Vi2AEWEUANDXC1gIoijyfB9A2GedIQDWGOwAsLYqCkqpx3iR5RgBY5QFxjnDGHMO+kFU1hWmiHDSdR3EVErdtm2/he9fGH1b/GQykUrlRdGvtnrPNUKIc97X5TipsQW9CztOE+dcz7voZ/PA93frDUG479yp6yYIPM5YlRcGuHQwoJxhTpXRjPLdepPEcegH2XbX75qCIDg7O2eUt6K7vb1//uzZi+fPt6s1QgATuFg9YoqkVgBA1YmQh4Efaa0p5weHx85Bo/TPfvJDY9RitdznxW6/D0OfUsj94OTs4l/9q3/1P/wP/6/xbIoIbpXe7bN9lvUmoq7rkjDCAHZdhwF0UgPlIITGgjiOMSU3D4/nz55Joa8ub+7u7kajibHgb375y32WV01nrSWUN03z6s1r4+zp6fHBbDIdTz587/0gCLqmKfNcCLHb7YIkFkoBALKsh+kThmlTVVoqZ0BV1Pk+61tcnDbQAY+xKAh8zmeT6bMnT7VUopMAQYYJQ3g2mWqh/8lf/OOj2fzt27efffb5cJRWVYUcmkwm6+Vqt9nEacoY2263VVUtVssemzUZDX3G7+/vOedBEHiMi7bWUva7nK+//jqMo2SQ7vd7COG3b17f3t4uVsuvvvqqB8MSSqVSEMLeZOKca5qmbb/rXMUYAACOjo4ghG3bNk0zGo16OuNyuZRG5lWZ5/nr16/LutqXRVVVd/c3ZZaHnu+cm8/nt5dXom5O5of9Lc9a+7hcFHUFrKOYFHX1+R9+X5aFEOLu5vZgetA0TZZlDDPGmDA6r5ub+/vzJ0+AdQeTaZwkypqLs7Neav/www/TMJqMRs4oY9R4kPYCve/7/eziM96fa/P5vKqqpmkwxsz30jQlhDgLgyAwwAmtxuNxf+j35UFJFNdNGYahx3jTND2M3vM83/d7qk/fE9tL/LLrrDbfMaaca9tWaWGthRBnWcY5Lct8vV73DVlRFPXnQNd1zrkwjOM47YSAyDnkpJFZkTsAWtUGcYAwCMOQMjYYDKzWDBMlZVvVWsi729t8n3VdBwlWRkMIndLX794eHc6vb6/KptTOSilD5nmMb7fboih6xlRRZEKI7S6TyhR1Wddlr94gCMss55QeHR11XReGYds0dV3GcdibNT3P64+Ctq4ZJkkQNmX18uXLIAyLoliuVkEUTg9mSuueXo4Q+uijj6IgDMOwR/rsdru//sUvRqNRHMfn5+fHx8d9UqFr235YZIyt1+teGVNKjdIB5Xy/3wdB0Lbtdru9urykGJ8/fXJ8ehLGMYRwu9v53Gua5ovfffb07JxT1nWdsiZOkrKtIMYXFxd3dzeTyegf/cM/n42Gd1eXv/v006Ojo7qufvXLv2tFRz0OADg9PR1PJ4vF4vDoyEGAAADGmKyqEKUMYmAs5LQFBgCLgcPGOaECyjHGuzxfZftWil2eeVFoIWilsMBJreI4RgAlfoQtGEYJhXg+mQFt2rbFHnMEGWMQBkJ1hBEltVEuSZK2rrizAWFNq8tGccpEXTHG0jRVUmohu6qBxhGIQs+nlPbmLSklRkjUzWw4Np3UWtdt05t5qraxGEajgcNICMEpcwAcn51iRvO6KtuaMFaWZdc1cRzXRemcW2/3q91um+ddJ8+Oz0RZO6W9OPz+j37oc89jrGrq1X7705/+FGPsM357+e7JxVmZ772Ah3HQdK0X+EIIRjiw0ClttfJ9v1FSY7TPdw/3t9MkIRYAzhXBPGTPzs//6Cf/IPaTZJC2ov3Rh9/7sz/5U8CIIEA41wkVcz7w/Wy7a5pGA4i5NxoPgDVdI3zuF2XdSOXHsUNw8fhYZWUYJScX5z2KiHDGueekIQ7ldb3abfOmAAB8+9XXk8FQKTGbTULu79ZbIToWept8L4BR1iwWD52Sddc2otvs9sq5pmsppUoZiAnEBENCEc3LcpdlgcefPXvCOR2ORxrAOI6PZpM/+6Ofc+YPh+OI+H/0o5/807/8J4igTz/97fvvvWyqbv24K8vSC70gjYuuOTg+elyvNEGtUR7BWBnPQmS0MhIxHMdxkeWMEI8zBACndLvd1m1zcXHxzetXX/zhD59//eX/89/+29fvLuu69hhrewXPD3Qneq2GMRaGQa/SEEKeP3/eU22Hw2GPPIyShFIsjSaM1m1DPO4AIIScnZ0BYDnneZ5n2U623eW7d//yX/7LIsvXi6XVJk4HUisDzGgy3O63r968zvN8MBiut9vFYvHBi5fVbt9UFcUEWVft66KotLUOwddv3+7Wu3xfjGeT2fzg6fPnJydngedtFovjw6PZZMoIPTo8lEL4nmcQsBA4qdMgUkA3qk3CyFk7TAdJFHVSCiH6s8P3/f59NhgNceiRKNTASq0IIT2cRGvdiQYRuK9LwIhxrkeVcUJl2/UClNbafDflwIB74+GoR+n1+cThMEUYcE6j0AcA7PcZhMjzGcLAWg0hzPN8u90mSWKMycuC+95kMhlPJ9KaVkvf97MsC8P4zbu3w+Ew8Pl+l4/HY48S3XVWS+dcPBo0XdejE0XVlPtdn2ZPBqlsO59xDOBwMmyapm1qZPVPfvj9bLMhgAd8MBqNOiWMMUBbZky9Xn788sXd1WXXya6TppNHB/PtdlvV9Wg6sQhIYC1BnRK9KXmQpkW2Pz6ax1Hge2yxWm12O+Z7CON3l5d5nm9Wq8lgaIQkAIaEHUymLbJf3122bfu3/+kXN2+vkYGr1er+4eHdu3fLh0dGaJIki832cb25vr2/u3vgmAzjJAqCs5PTjz/6aJikbds+Pj4uFovNbtsbpqMoioJwOhp//YcvL9+8DcPw8fFRSvmDj793fHjoM384HBMMPQqv37zzENsuNu9efWuU+OjjD9pW1EUtyjpgfDIaA4C6RgzTEVosFsbZ8XQCERJKCv3dx6X/3HiUUYSrspRSep7nhUE8SA/m87wseplbaz0ejvo+zP5Ds9vtPMagc3VZMeZJozGjFhgh2iCN8roWSkdJrLXkHrPWYkY9PzQWQIziNAYA1GXlc49A5BFqhDRSWaWN1j0InnNulfa551NGMekrPdu2I4QQQgCCVVPHaYIZXW832tm26wDFnRRHhyd9PDWO47wqhVZJlA7TlHM+nk2Hk3FZVbPZbDabBVF0eX2V77OHu/ssyw4OZn/9i/90enocBeF4MHy8u8eIVlWVjoYOQWVMHMd1U1IMtZZNUwEAAMQIIc9jP/3xD3a7Xf/STtNYKaGNxBA/3N+Ltn3vxcujg3kaJNPDaafV5eX1N199TQGC2gaRn05Gvh9WVdN1zWgyJJQuVsvBYACsGU9G+8222ueT0dAYZa3dbDZKSNUJrRUGQCv5+vWr8XQSRcHvv/jsaH44HKbj0XC/3VRV1Y/zxigNnJDq/v6OUFxVBcLYOBvGEcQIIMw83klhAYAYd1K0bWucPTo6ssAZY2az2dOnT4Mk2Bd5WdZKyH/+z//5wXTWU0rW282f//mfe573u99+enR0tF6vEcEOgN6jVu4zDNFkNmUeF0KUeUEpdgCko+HV7Y0SYjQYlHUNMUqSpKcnhmG4Wq8P5vNWdNd3txo4A0Hv94iiyPO8pq77HWMPelRK9fqM53lv377tb1gIoePjYynlmzdvKKWTySTw/L5gKEkSn3sM4TRNX7733sHRwXQ6rcoyjuPr29u8rqI0mUwmw8HIGDObjO/vrruumc/nh7ODOAghQtPptOua48MDB8xi+cCol2XZZr31PO/y8jL0g4PpTEu1XC6ttVc3132uRwjxuFwsVsuLp0+UUhQgBKBzTlsTx3FRV4yxMAw55wQiBnHfuxTGkQGOMq83mI0HQyX0druvqsrzPARc27Z9CLY3xigpvTBoRNcXOXWN6OPlfe2BkDIZpBa4pq77BGkURcYYSimlWAhRFjVCxBjT1k0axV3Xdq38ewKl6I03wLkgCBI/bHtu3WYTBAFjrCmbNE52u108SKXsoiQO0nBf5AhDQlDfjDZIR03XMkLGo0GdZ3/9H/695zNldF3X/YK6X6d98MF7xwdzDNHNzVVe5R5lHufL1WMUB8aY+WwWEJaGUbbbCiGGSRp4fuwHTVH2A/F6u6nbVlnDPB4EQVlkRipg3TBNt+tN27b7/T4KgvV6zQldLpfa2fVmk2WZBW5fFgYCba3v+/1hOBwOf/TJD44Pj0TbBb4f+D6E8Pz8PIjCk/OzntcmpXz79q0Qom3br7/++tU33xipKKX9XieJ4ifnFwdHh0+ePFmv10oIoZVD8OTkpCiKn//85/ODg7YqT+fzsiwHg8FkNt1ke23NersZjFIvDL785uvNdrvdbvfb3Ww8efXl14+3d7PpuC7ztq5QFCd+EBqli6oEAdcIOG2Q0NBYrbWyxiE4GA4xRP2BayEo2rp/TvpasrIoMEIIobprLQSEM2tA18o0jJIo7rRZF5mFBjPkAHaIAY8t9lvf99quW5V5ozTCIAp9CWxnTb/X7p9SCDBwiBHaKy1SiD6f3e9Y8l5byDNMCIRQNJ3pJLbAapPvMz8MiM8RIwaBPsAtuo4R0ge1CWcHh/Nit9dVywDqytpAIIGtuna336+Xy0GSblaL0TANQ48yyDm+vr5WRlPuh2EspV48brQC1sG2bSEEADilG8oQDRjADlhnlXZGUoqVAxbBQeR//PLlZrnCiHZdywl+8exZGka6E7PROPZjjgnH1Ai5XS4Iww1wWdeVu6zY7KIoGkyHHbSQUweMlp1H8Hr1OJ0Mi3w/P5wZLXbbVRx6gc+iwFOipRC8//xZW2aXb19vNqtONBhDhACETmudZfuqLr4bcpWMkniz25V1BSmURndKOuf8MNgVhQZAtgJjooETwCLoALTU44DiJEkG48Hh8SHjHCC62xe/+/WvkjhEFO2rQjn7+vLy6cXTuqi+efXVcD74zaef7fflOBk0u5wpN+TB7dvLpqo1dB201qPbplxu1qfnZ9paoVQ8TPdlwQKfMVZk+dH80CJ8eXtnLfCZH3i+McaLImXsMEnLLFdaOwDyPO+nbwBArzxADI0zRVE8Pj4KJTspCEUvXj7LqzLyA2QdtqAnDHOM3nzzdbbfv3371jjbSjGIk/F4XLR1Oh0bCChndVUSA0zdiKwcDgaEoOEg3e+249EQU7LJ9wYChEAURVf3twpY0XXZensym5f77Pr6mlJqmi7xQ2n0B9/72At4PEyi8ZDF4bvrK2PMeDwGxiohGWONltP5AQQYQWL/nnU6CGPGWFFVfYVZ13UUs6aoUy+Y+PF8MMZKm1YwSj2feb4PEWKMEYRNK2IvkFoxj1NKOfVE2w0Gg7wsoiTupOhNGgjC0A8AAH4QqH47bR0C2GM+J9wPOESWEUQxJZA5AxBCxigpO4JZGo2IRYMgcdIQQKx209EYKOOk7YX15XZT6y4apQa7RneQYC/w67rWWkdRtNvtdrtdEHi7/Wa1WnRdQzDj3C+Luus6guD93d1+ux+lI8To9GgGCXh8vFOqbarsYDpdLVaEMAfx43oznk/u765CjpQSxhgpJaW0jxM753rhVyn17NmzIAh2my2wdjAYvP/iJUH0+OAoYHyQpg4CEngGgnf3t3yUZEZ0Vn/00Ud/8qOf6ryuyhJgND89ToeDo+mBT9h4MBRahWmy2+2MkB6hwzRlnld1batkGifHh0dt2370/e89efbsvecvsu2urevrd5c393dPnjyJ05QEnnAGYPTxxx9bpeM4Xj6ubq6vX3/z5e9//9kX33z7zeXlu8c7RQGO+KbKDWefffWVc2Y8GlxfXT3e3iFrpmkqq8p2LbIYlk1dVGWYxF3Xcc67rhNt1zQN57xHhvWCkVYKQ5TnOaVUCEExYZR2bYsc6GUTyhgiWChZd23vDd+u1pHn+5R51NNC922NnLE0jfdF4YUBodwChwA0Wv5nhgkEAADYl2g7CChnURTVf4969xjHjPYYGWsdJTzwwn5LNh4Oq6IMGMcIrVaLfgTBGFttukYgADFEnRQWuPF4XBQF53Q+n2OIpqPxeJiOxoP50WGe5x9++GEax5zz4+Nj0TVSytlsNhgmeVWut5u6Fc7Bv/zLv+zzZm3bVk1JOUEUCdURzijnSouuaYfJIEkGnufFcbxaPN5dXZV5NZ/NDuczj7Pjw/lmtZZt17YtAIhRD0NX5PtkODAAAgB6e5Jz5uHhoeukMtohWBU5QdgoKaWknEktP3rvvXdv3lRV1YhOa00wFFosFg/Zfvv7zz+/v71r62af724f7u8fH16/e3tzdyOUtEoThIBWzuoyL5RSURStViuEEAaQINw1bZ+Uk1JC6/pi+yiKlg+PZZZrIdM0jYPw2ZOnT55evHjxwvc8CECaxrf3d0Ip7nsAgMfH5bNnzx4eHpADB4ezu4cHz/MwhI+Pj31BIwIwCIIgDKXRcRwTQna7XW/KLusaYCSEWKxWxtnFYiG7LgiCo6OjH/3oR57nTUbj7XZrnH337l3//u5P8/8sHPc557que09bb9HpDR6EEITQ/f19URRnp6fzg4PVcumcOz09RQiFQTBIUgwgoqR/KPI8Bwh+++23dV0rLdbr9fPnzweDQZoOV5tlXmbW6qoqPB4sN+vdejeIE2mkMubJ06eHh4c+46aTk9F4kCSDwaCua+5517c3bdtOJ5M4jqMoigfpJz/4gVKm67qjgzmEMB0NMaNVU/cCN8Ok/0L6cpvH5YJwdnJ8TBHmnDtjCcKiaREAGGPf533qW2vddh1hlDJmLUCEFlVtges9ZlLKwXhU1BXEuOm6TogeKrnf74WShFFtJKWUMV5VdVVVnFBGaH977Z+vMIwxRD3T2BnDqQeUC4MAIbxZbqqiVkq1bcsQXtw/XDx7aoBrurZoK8IZxCCIEt8PiyxjCGOItJSEIiHa6XjSdxgYYwCCXuAjhDyPB4FvgHv+wXtN15b5fjoeDpMh84LVduUFnlJqv98fnMy1McfHh7vdzkKAMGaMPdzfc0L7nBcjVLYdQTQri6Zrf/zjH//lX/4lQqgPJCOEsixr21ZICQAgnL187z1trRf4f/oP/6yV4osvvvjoo4+6rvvtb39blqVD8Orqql//AACctZTSF8+e/+/+6T/nlHmM1W1DKb25u42iIMuyzWaT57nQ6ujw8OLsPB6kQoj9fv/mzZuqrptWZEWOEAniiBASheFus8UQQgeUUlGaTA/nWV12Rn39+hVjLIqio4O5bNqby5uiKIbjsXFmuVl/+vnvUCeEHwSEM6u0ZyG3kFCaHkyIx4uy7Eds3/MoIZxQrRRHhGGCIWqbpp/sGGO96kcwVlIzyrU1BjjMPYQQFHKIPe7oKByGjAYEYWmAMEEYI0wYghwABjHH3EhV5kUcRf20CClloZ+3dS1E1XVJOrQOAuuMMdZagFGjZBCFwzjZrdZxEIZhmGXZ4cFB17RayDiMCIJxGBXbPXYg8QIM4Ga39YMII7pbb4iDURJv8m1W5UJ22WaT+kG+3//xn/3J9e2tMoZ6/PHx8fz8yX6bLR9XTSsIpYBgPwmUVdfXl3WZQ2eGaQwAaERnAeRhJKQWSoqmicOA+yFGTDbtZvHIff/g+NAZoIXWonv+7LxtmoPpFDOy3m37HGzbthA6RFknlI88LJ10qhKtVfrbL75khE4mYyn10cG8bjuHSWvU8xcv832+W+1Y4CHKpHXWwTgZOEw+++z3+b7orSkQ43iQEuZVdduIRmiBEKmzigEQc845b7rucbVI02QYJdgCaF2VF7rpql3W1HVT1z6mHiKqE6EfTEfjw/HUp+T89Mxn3jgZPjmZf/Tei4PZ4W5bXDx7ijEOGPcwDSLv8ubqdHZy9eYmSZJdvn799k0pu/hoJiiEGCVRVOUFx4RCVO4zUdbO2Mv727ytGSEh83qjy2Kzvls8irp1Qk3G02yfA20X94+B5yshwyT2woB63ADXixi9QuicS5JEKcsYYx4/OJyPJkMLTN00293u6bNn2llCeVN3xT5Lo9gLIqltFEUeZZvlCkJIGRsMh/luP0qGgRckg4FxmsehJehxvbm6vH379u2bq8vz5xdllYu22+12slEvnj2vshwZNx0PLYV5W1+9u/7++x860X35xe+F1sjnWZ0HcfDk/GmdVSIrh0H07OWLr9+9JpxNJjOlDGd+UVS7LI8HKaX04vh0NplOD+eIU44JQ/jk/Exp3c+v0ppwmBaiMQi0zrAocABhRPtnlnBPAtAaI5zppIiiiFLaS6m9GVEZ7SDozQgY4944ixDa7/eUcKUUQtA5yznfbHZFUclOpVHclNVwOMzzvHc69GeFUop6fLff120zn8+FUFpbhFAcRseHR4vVMq/K4SiNAk9ryTgvqlIoOUjSgPKiKMI40lbv97u6zI0QWqqmk5jRoq6ANW1bFyLLu/y3n/4eQTYbjX1Mq0Zi5gNG7tf3hLsw8b59/QpgtCtLGgSNVI2WregGg4FHGTLOKo2MG0YpAODVm9cOgOubu1//6reMeYQwa23TtjwMyrbxPY8zppQ6OjycRunxYLJaLN+8fWs5eXN/EwTBfDrTWndC/Olf/HkYR77vPzu7eH7+5Onxqc/p7z77bZrG4/Hw+GBGKArioGjrDz744PbqcrffVFVRlvnVzfWr16+rujVK+b7vHPzkk0+ePn16d3f3208/DaPIaZNtti8uXj6/ePbJhx8cH8zKLNfadq38sz/6Yw7A26+++uv/9T82Recx3kpx/fj47c3VqipffPI9xDHRnYiDkCAcBaGUcrvd9qISxhgjRDBumqZpmrZtGSYEYy2kx3lPgFFKSaOl0U3T1HUdhWGapowxL/DrrjUISC0BRn4QrLYbSikmqKcOtXVTl00Y+Fqr9W4rtGKMBb6HHe6qjjFGCQn88HA+t9ZGUdQLr33QFDjUK+zW2t0245wb7QAAzPOKqjLO+r4PjK2qqsyLQZoywqRWiJL/3IMDrUviWBrdCXF0NB+Px04Z2Qnn3DfffMMIWSwWEKHtbgcB/t7Hn6TpUEqdFbmDQGk9m0+zLOujH/k+M8YQyluhECIQkf1+nyYRsMZoWFXN0cFskMRhGC5WmzCMwjBUsjPGPH36VFtrnD27OD06OmIe35ct9n0LQVs3uhNREBnjbm9vrTEUYdHW+83OSKWsswA5AKfzubLuF//pb6nHIKWGYEyptmCz2wuhEKYAYmsAQshBFCaxMtr3fUyJ7/t5Xoa+F7KgziolBOcMQmiMeXx87NGACEItVRiGPYDwO1Jd33PLuOyEs5YAGPs+dq43/zZVVde1UmoyHW0WK9E0XdMO09T3PNF0QRD0BMG26xbb9Xq3XW83i9Xq6Ohotd1IKft4Wh8J7uMInFItVV/renBw0McvIYS73W61XvfGqiRJ+lQzxrg/X3qhGULIOe9/TQip6xpjXBTFer1WSk0mk7u7O+55rRS7bO/7PiS4k6IV3+0Yp5OJNcZa+/DwYJRFAL7+5tuu60aT2Xq9dhCUddU0zcXFkyfPnvpxJIz2At9n3mw6/fWvf73NtoSQh8VqsVpnZZEOB4vV0jn3Z3/2Z1meL5fLi9Ozs5OjHhHjMV7mRbbNzo7PKeG90aJXzznn/aDc57DW202WZVVVNWVVF2VfbzSaTuI43ux304ODIIp6Dut/zgH0QkSvc/ZWmT4B3v/l/aATRVHXtP2sAyEMgsDzPK11GIZ9LW1d173lPB0OoyhumhZCyDgty1JrPZ8fGeMcBJjSvsUhGibMY3XdikZQzuu6bqvaaq2E9qi3XW84ZUmYWAukVkEQrB4XSRS3dSOECILA9/0iy07mh0apOI7LtvHDYLVanZ0eV03ddDXz/GE6NEoeHsx9yvN9bpWeTqd9UdR4PE7T1Mj+XaOMcU3VBtzrQfweZWVZ9n+MBX5nVP+C9DxvPp9jQvrrsO/7w+EwSZI0iv/w+e/Pzs4AAARjY0wrOmX0bDa7vr6+vr5eLZdffvll77lYrVaLxcI4l+e5tXa323VN6/t+EkZpFBf7bL/bBL7fy1Cv3r199eZNOhwIJfva3u9//L3T0/MoTEbj6Wx+eH17hxn9+OOPX715k2XZ3/7t32436/V6Xe3z7cPi13/zyyAI3n//faXUF198sdxuGimqpr148uJf/It/qQ1EqDWu1boSHvPztlbOHk5n2Lh8u+OYMEw8ynzGA+455xBCFCGGsRAdhABAaKy1zvU8LJ9x1YmyKIy2xjo/irDnBZPRVreP9S6YpgrAshEk8JjHR0mq20ZLTTj3h1Gl27pr+uCS7/uqkQzibl90u9JDZLNYeoz3nfFBFDZNozvtYdpUdRhHQspOSWWNAKZ1Gga8sxoSjByA1imtFXQaAeZx3/ejMKzK8vT0VEiZDBMHrTVqcX9HKKqKjHGSxiF29sn56fTwKJ1OHxaL7TZDAIV+GIaxUiovM2PU/HA2GqYAWi/wjYOE805IZ4FRxmP+brcZT4Y+8zCkhLO6q5GD1jppNEAQOeBRZizYVeXN4mGxXB7PD/7pX/6Tw2dnWSe3q/UgiqMoklrd3T4+PX/aNmU6iDBEVhuI2Xg02+32jPA6b4u8WmcZCqNWO2ksxLTrRBLE0MFaCOp72tl9XjatePfu6uHxLq9y34s4C4bDuO3qpmqh67MiEAPoHPB9v5et+yvzLtv3b/F+NC7LMonj24f7T7/4vCzLKPRPZrNql+13xWe//2K9XFDsttvtw8NDHMeh73uEQQc3u42B+vHh4cWzl23VAguxtqrpJpOJtub19XVRV8vtxotCLw47KY7mhxQgREjZNoSQyPORdk1Z369XjpLlclkUhXJAIUSZ1zSdEApaKKqOONzvD3sobiuFAc4BkBVFEEVFVQkhhsNhUZV127y7unTOIYw7JSn3j0/OAACffPLJeDzuE+onJydFnidx/NH7H0R+cHF+PozTrmr222w8GkCgP3z/vf12m+3yb799rRHqnJNKfPvtN7Vuj56cSwCtQxTh7WrNB2GHHOD0y6+/YpRcnBxfv3pze3mLGGKRd/Vwp4wmFgzDOEmSx+UaY8oI74TqWukzP/SjphMAYSGE7/tnZ2dRmmCIKCFSqaIoAsaJcU1VNVU1GYytNE7ppq4BxNo4qIGqBHGoLWqgDAbQ59533rNOQAtFI0Mv0EJ5nieVQgRXVdWjmzGmhPEwDiAGUuumaTGiCNOyqQBCGOOjo6P9rpBCxXHcyZYFvnCGebyvQzo7O7u/v6+qwmg5Ho5W90to4IuLp1Bbn3sUs54n4fOAEX5yeoQREFpoI9u63m22jFAhBGOsrOvpbPyLX/zi5dOXoZdMR+OyLDyPffnV507qiHmzwQRIW+xrKWzghZdv3lqt0iDADhCIpuOJloZxv2xbzL1oOKxE24hOOhPGqe95zy+eLO8e9pttkeez6RRo4xF6dX1NKEUAZrv9r37328V+QxEWZU0MqLaZlN2zl888z0MOEON0J5xzH37yvXQ6bpRAFG02q/VmKVWXZbv7+1sC3AfPniHjjg7m5yenX379VdO1R2cnQqu6a8u69XgAHXr36s2rV6+2+x1i9P2PP4ri+Mc//1kyTi2C1PfqrgsjP43C0/GEaLfNi/vNZjIbe6H38uMP48Gw2BS//9UX/+5/+t886CNn7MFkigEss9xBgAg2wAEAxrOpNNpC0CkptGpElwwHjega0WFGGWN9kuLvu33D3p3dZ1z7Fseu6xhjFqK66ZJRChBwEBDCgiBq66bOs4vTM2Mt8z2H4HAyBgB4jFtr+9sWxvQ7GDdC1lqpFaZku9sZC5Ik6d+0cRxrrb0w6kMrVVUhhLQxBjiEUBiGfTIbIdRL9hjj3W6Xpun9/X1/RiRJUtd1GPrH84ODgwNrbV3XjDFg3Xq93mbZaDyVUnaddMZuVuthOgjDuCjry8tLoST3AodgLTshJcG467owjJXSJ4cnGJLZ5KC/c3Hmc86LovCiWEPXRyU7raYHs5cvX75+/frf/f/+F+fc6flZWVdKaNF2RVmXVdP7shHG2+3aOGuc7a8VxMHxYOhz7+buFn+HcwFaKWttP98ggn3fX6/XEJHxdDKfz6+vbm9vb7W2fUy3L2yywBFGEYDQOQgRtM4ChxnFGEOEEMZ+EPQb7J5PkqapsXY6mz158gRAeH9/L5v2xbPnk8mEMOYI2OT7/o6stV4sFv3GLyuz84uL65ubrpXJYLDd76SUVuusKBxG+zwnnHlBWLcNINj3/Vfffut7Xl8Y0N8ue1j0wcGBULJpmq7r+p+4EMI5xzDhlBmtKSGM0r4Kpq6bvseVe7RPtHLOe3oPpfTu7u78/Hy323VtO5/Pl8vl3c0tdODh7j7b7owx6/X68fEx9HyGyWa/a0RXVZUxZjAcn56f9Rfh9XZ7eX2tjIaISGW4722ybDgejSeT1XajtDbGlFkeR1Fe5rPjOYKka1qf43y/PT46hRY6Z5SRw9nEi+KuaV99802eF8enp0Ipznng+Rdn513T7na79z/8UFsbBMF8Pl9t1v1KbJQOLATzw0PjXD/W7Pd7ThmGSGvdf5c4ZUbpwPOjMBykqVaKM9a2HcY4DGOMMXQgjWPRdQAAhJBSSltDOfO514/m/d4CY6y1iaKoFb23PSyKQghRVTVh1PO8qqosgHldIYq6rpGygxBuNhsEAGNssVoSzEbpEGg7TEd1XRPCemfzd/Fyzp02jLEkjPr4Ur9a6KeQ3vo5Go2+/PJrpcxms4miyAI4GI4Rgm1VIwO6RnhegDGtqopS2pttRqPRbr2pikJrvcv2jPPtfocxZoRCiD0eUEqPj4/fvHkznk5msxnC+OrqSrSd00YK8fvPP9+vNxjCtqq1VNeXV0kUhUGQJMnDw8Pj4+Pbt2/7Krff/va3V1dXv/rVr/77//6/XyyXp6enEMKL0zOM8Xa7nc/nou0u376DDhBC3r17Bx3o602yLAs9f7vfVW1zfX2dZbnUZrPPlpvN1dU1AKit2g8++GB6MBkMx6Ph5MWz52cnRydHxz/50Y9OD49evnjhEHz63ousyMuyPj46qrL96eERMg5BTl+9e6uBgwQzQgPPL5t6XeXbqrAUZ23dWg091hpVipYEnkaglUIZjRCKwtDj3GjdY+8JZ5v9zgHg+z4jFFvQZJXc1/NkKPJKNjWETgvZ7asQcZ/S1frROFNWVV2028VmEEeya6B1xjiIkDAa+gz4rNHaT1MHgTKm37ApZ1jo10q0WkqjjVS9dBgyL6ScOQi1bZqmahsDnBDCSHV6cAiUKfMCc6YBdM69fPHCSFXkeRT4aRxud+uiLjBF0kiIsTI69P00irTTcZJMhiMjDTJgt8kQIIyGgZ+s1vtO6rxugijsRAshCANfyk5KSQANWRhGfpLEgZ9gQIRWg8GAMbZcrBHj49lB03Vd180m45cvnhPC/sP/+h9006a+//W333RKce5rbZnHAYFa2+F4WnR1rTqh20ESIm0izm7uL8umdFp5BEKnfIKRlm1daeQgxUDro8nMQfAv/8v/ar3cbNdr4LAxtswLp01VNH4Y9X0mwDojNEdEKeX7njFaaKmN0lrXde0AMNb2+aBWdJQzxti/+Bf/gjFGKBVKnZ6eHp/M4zTai+7Vzc3jYvHrX//6N5/+VgP3sFp4UWAQfnd7nQzS++U98Zhy9mG1rJQIoujk7IwghAGUdds1bZnlhJD5dAatO5rPe8iwg1BpnReFVbrY7oVQeV72pHKCca/2lnVlrKWMCSl7qZD53DlnlO2t7kEQZFnmR2EQR3VdR1G02aySJGEQW2HSMFJCDNMUWDudTgeDQV8xOJ5Nl5t1UZXG2U22z+vmq9dfA4KzvDw9u9hmxXsffaycbWVLKMrzPEjSIB7sNnm+K5um2Ww2VmloDEPws9/8Gjvw/PyZUfLk/ORhs0GMWyUZxQ5hBYAXhWEcp8NBWVUQYwfh2dHJ8v4RY/y9T76/XC6lVs7BtpMQ48Fo+MH777d1M5/OhpPxcrexFGNKDw4PHUYauChNoiiihEghrLV12zjnrAF9YtkY4yCQWnleQAgpstxqMxmPEUKEM8JYfwPwuUcpVcY4gAnzk0FaNXXTVJzTsq6Z5xHKqqoKI496jPLvruEMkzTyD4ZDJWQYhkpIo3U6HF9eX8lOWWm//PLLJ8+er3Y7YYzv+57n9YXmnud1deOsRghMJqMoCR8Wj77vc84pxo+Pj54XpKPx67dv4iSomlpZKjSOhmk8SKq28f0gjeLA84GxiJLpyYmmZLvLTk9Pfd/nvvfk+TNtjed5ommxRQyggPHbq+v1Yg0Aendzu28a6nHueVKrTz75RLfq+fmz+XhqhDqdzgc8QADc3t5iQiBGUiuIURCGWVn0aOWyLH/xH//6vZfPx2nym7/71Yfvf5DE4erx4WA+nUxHm91aWll1zWK9Wi9X7794OR1PHm7vxslACDE/OfbTWFv7uFzkVfn02bPNcuWMPT08evXNt7LtHu8XnODpaPh4vxDKrKvSIvj+2TkoawBx3bWHB/N//Of/8P767f/5//R/fPnsxJgaGS0xgVVdGKuANlqpOIzSNLUQSGcMAjzwjTHWuT4yCiEMw5Bx3nO0gbGnR8cYYyFlPEgxo5SzVnR+GHiel8YJtoACRCGyUjmlgTMIQMIYxBj3/T6UDqMkDRNgQBzH2sjZbGIh6GSLCamaWhnJGCGMWWshdABarTUAVkpJGeMEG6P6zCohpF8TUUI459oabQ2lHAG4Xe0opZRyxry6ajGmRrskGcxnB0mYNHXXtm3fekMo577nhUEaJ8M4GSQxRHa9WwNgZ7NZ76FeLZcU0+l4KqXyfX+332utQ59XRdaJdjwei6Y9PTp6eLg/OTmaTSZ9d1pd15QQBHAcRev1er/dTkYjZy3F5C/+4s+PjuaPt3eqExBjLwzqtrVW1229y/ZlXQEAIABStJySxcOjM2a73fpRWBR7jxNkLdTG557HKKfED/hwMvjTf/BHSRT9k3/yz16/vfz9H74QSnetRBZ5nieVGA6HUigAgDQSQsQYk0J4mBZZEUVRT7PpTeKQYGVNX3cVD1IDnAXu//1v/k3diaurGyHUdrtdbTez+cE/+NnPKMZKGYxo29aLxYNzThk9GAyMBWESI4Q8z2OE1HXNCd1ttrvNdjQaaWk8xq3S6/V6sVggSjCjbV1B5waDQVmWFjrCqei66WTStfV3JgpjewXGOdePktoahFAfN8MAhp7PGcEAya4zSnVN1fPTA8/v2vpwPg98roTA0O33e2t1mRdRFOx2u/1+f3V9++z5y7xuDESdlEcnJ5PZNIiD84uLthMO4rysPcZ7/Dp0LuS+lqqq6+VuY63lmCIIp7NxL1jf3dwCY0XTvXv7Vkr51VdfTSaTIAq7TiohWyGS4WA8nWgjq6LgjJ4cHX3vgw9WDw9GCN/3P//8c0ppX+EUhiFBiBG6fHgsy9IY89VXX83mUweAc873/SAKHQR9OWpblZHvEYr6gFJTf9c41oP4McadFD2AhXm8bVshhHO2v6slUVqXDUJoPpshhLq6+Y5AFXjWGQyR53kGuCiJsjKzTgMAOOej8aCuSy100zRKqfv7+yAImrrrpJjO5xdPz5NRMhgMeqG/ruv+M2aUXi2W4/E4iiJrbRyETdNst/vxeOiAMUo558bjMfe8oiieP3/+eP8ghACEQs6zIqecAAjrpiGc7ffboiiMBXlREcoxo8Y44GDXdVdX73r4oLW2NxdRSn/84x/f3Fw1TSWNfnh83Oy2jehOL85bKQ5mk0EaX7x49kd/8sdZlvUjVCtEkiR1XV5cXNzd3cVJmKTpYrvW0AVB8MEHH0Q83G42ou1ub28bqX7085/usmy12cyOD70waqVwFn7w0YeEMwhhXVZ5lh3NDyGEz569iJMkjCKE8W63++EnPxhE8a9+9SuM8ezw6ODggBGCITyeH56fn3PP22b72+ubfJ+9fP4iTYfAue12Szj71e9+/eb6rXQKmbaLuTefzULP75qWWYiNs52MAl9LSRCSXcsxpgAgB6C2plNd01ltMSIQQi1VUVdCK0DxviykNQCjVum6Fa2QVVU1XS2tFFIBh2LuJ55XiMJyaCjW1vmYYWmkaLWWxti6aKMo2mxXCAPKiDaKYMQosUZ3TRX6PAo8nzPZNtZoSjBwVquOUGiAcghs6sJy4hCwzqVpCqyDrjfje4xxL4w6bVSrgXF+nArjLq9ulLZXVzfGuE5qZTRBFCMktLq9va3zQtU1kJIzlA7jcBB2Rs5m09lkMh2PndFpHMdhIJo2CSOrVdM0FoDxdLTbrcPI2+9Wo0H09vXXYciNUfP5nFLqQQKVioNwMhzU2T7y+Gq9vr692mwf//RPfvbxJx+/9+EHZVNYYBG1FprBeHB/f79aLWTXTKM4wiwOwuF4pCEmfnB0fDo5mIY+t1p5jFGMu7Y9mM1Gg+H50cl+v39yfpHGyc3NbTgaSuuGgwGjdJftIcaE4MD3Qs4YxF3XWWs9zJFDHBOrjOf5xjnGPIqZkYpAlEQxdKDMciHEN2/e/u7LL//df/zrTz/74t/9z//Lf/oPf319dft3f/uro3T6f/iX//vpeGqtU1LKps73+6t3V3fXt1VVjSezfV4OokRWzdOj066ooNKirLbrTRAEnPtaGmOcVGaz3VLGtDVh6Ge7DfdoEHpVXVRN1TQ1pcRaTT2qrBKiDQKvV96COOqUlFI7A4w0SAOPMmidR1lT1Nv1iiJILDibzT2MOIR1nr355lstRV2Xlah55G33Gwgd59QLg/c//Pjdze2+qtPJNIqi3/zqV0YpqURvkiOEvHr1CmhVbrcXRyce5vl2N5uMEHZZtk2SyFgl6jLPtkJLac2TJ8848zlng/HIjwdHx+fLx4UzlvkBD+IkCFcPD87qi9OzJ6cnpm1TzsvtNmD02ZOLrq3PTo99TpVofZ/7HmOEaiGZ7/mht92tfU77CGG/T86yrH8HlHUBCYTIWat9j0BgCMGUYs65lAJjqKwijClnAUSUMQcA9RhlmCDACNXaaG08xtfLhceZ0Sr1AwYQp8wYgyEwSgEEGtWGYRhw5ow2RhEIGPMQC+4WG4MsD3hZVlKqOEoury+lk1mxgxYgC4HWzmilRFOVgzRWog08HzqgpQr9yBlgtfE4FU2tLYijVHRqOpkYpYss9zElAF7dX1pswtAvy7zsagWM1AJTFKUJ57wsit12K4RYrtd5VWIMjdGEYGMBxBQgt9gu3//4o8OT4w8/fJ9gqEWnpOwdB1Vd395eh6GnrfzFZ7/69v66lk0tm+F0Npsf3l9fhYzIpsLOYAvmh7NatYAhyv2uEY/39wEPzp9cKOhW+V4T4iDY7nf/6Zd/l4zGmLCrm+svvvryzbu3EMLDw8OTkxNKyGw8u353ne33P/rRjz75/vdV20GlH25uHpYPGoH1brve7GSnbt9dYQfy3f5gNsEYVqZ7//sfVkXGAAo9f7PZxIPxpupqgF/86CfIizwF9OPjY499aJoGOcAJbfOSQuRTBt13xgMEoBSil2IAAEZr6ECfvey5yYQQ4FzTNH2dozMWWjeeDCmlGGOMyX6/b7o2jANpZCuFVFq0XeD537kaHOpdMUEcQgiM0n3vEsYYAOBx1tSVs4Z7bDgc9Pe1yA+gM0p2jDEvDJJB2u8MEMHaGuNsEHjGGIIAtE4r6ywktBdk67zM33///TKvBuNREEeMeVGYHMwOi7yqyuZgdqi19v2wa2tRN0kcZvmuqoqbu+vNdsUpTtP47du3fael7Jo4DI0xxpi6bQ10yTDBFIW+B4HbrFeM081mMxmOQt/fb7fQASNkvt14BPf5rzj0265OkuTDDz80xkjVAWgJx03T9GomsK6p6mJfOa2augQYVa3knM9n8/Uqj0KfUlyW+eHJ6cHR4fc/+jgOo/F4KJ36N//jv43TJE2GVVVZpbu60VpjSh1GlJP9voQQ88DXzpZN3Zef9dNPGIbW/v95+q8Y27I0zw9bfm2/jz/hb8S1eTMrsyorq7q6qqu6p2doNMNuugFFIxJDh4FIQhRIkRyBpAwEAQJkQD3oQdIIMhQpSCNRoBt2twYccrp6pruqs0zetDevCR9x/Dnb7+X1sIsTDxfxcl8iYu+z1vf9/7+fLcuyq+x3GYw8z4WSv2odWxeG4cOHDyklRZn98De+X2w32+32wdlpMkgnk5GyppGNBRZRkucFsM5jrCiKk9PTr1+/6nIRAABEcF5XCCHnYDro1+JXTBjPYxC6bvK+d7CPCL5f3Dtoy7JEuBu5Q0IItK6zXDVN03GHgiDAAGOIgLEeo1pohFDTVpvNxhmzmM1vLi8wRpvVmhAUD3pCKynbzWYzmA4bKcqydNp0kfY07fmcE4Qmk8lg1F8u58Zq3/MwxkcHB/tHB3EcFUWhpfIoK/NiNB70+glErq1Ka5RVuuviv339ZrFYrPNdrYQXhEoZjPF2vTHKGmWNlnVVNE2jtd5sV4NeEgV+6HsIAYTB/mQa+YHneQ+fPI6iKM9zJ7VqRZ7nPPCPj4+HwyFQJolDAC1lOAz9KAq6+mi/37cA9HpJXdceJUHoGSsxxc4ZKWXkB4wTn3NlNSZIaWmUsEI5Y7VUUoo0Teqq8H1fKcE53WabVjaI4C5ARTHmjBCEOcFSSi/glOLJZNI9xWEU+V6olOKep4zOq/Lo5GSdb/zID8Mw9HxOWeh7jHlp2ldKBUGwzfKk1/f9cLfJGONxHBdN6zoGeNv2h4PPv/iCEMII9QL/7cUFp0SJ9n5xX7WNtgYg2Kr29NHDPM+N1j7jo36vkzxHUeSHse8HxliMsRf4iOBu33ZzcxNF0a9///vvvfvubr1KohhCeHV7AyB+8OABISTuDwxwpw/PAIJxPzk5O1GyFVX96uXXztj1ajG7u//1X//1k8OjfLeTTVvmRRiGu7wcTsbGmDLLkygFFv7OP/APWufath30+r1e7wff/+GTp+/s7R10SN3Hp49Pj04Cz8t226s351abti6dM0+fPTPGfPzTnxV5/vz58zRNN5vNZr1mmHieJ1T701/86XA4Pjk5UUq9+957j548zYty//h47+gQrXWzbsooiRkhjLFer5c3FYSQGDDgIdXOKa2UElJWVdXB28qyNEJiACGEZV0FYQghpBBRByPu+4QRhIG1HqbYgSYv8+2ubVuAUa0l5NRqA5TRQlJMWilao6qm5D7TWmdZRj1e17XTRjZtWZaTvSkmBGHMCKWYNFUtpazbBiDoUaaEhIBQ4rV109alMaosc95ZQ4GL06RuK+rhqswIBhRjDF1VFUWzYx7yPbK4uWEEIQLfXr4FEL99fbHarIMgsMoghwnjk729tmrjIGaYPXn0NE6jRtScU+rR28XdwcnBZrOaze+chVLoOqvSKCWMpv1+VuSIEqFkZ5p2xl5eXi7Wq7KuIcbOQSn1O4+ejHr9qqo8L5iMprvl1gpzMD0IvaiuWoKZqKXTripqBPBqteHc5xwj65zUR8fHcRyvZ5tsvsIYaCuVFfFo0JuOke+/evXq4vItDUh/1D88PWI+h84Q4PJ8l+e7KAh3RT7fbnIhjAMagNYoxwnwcK1biyBmREu1Xq4IIZiS1XIJnGMeb0TbKjmfz+M0YYyF3JuMhlVbs4BEAf3Tj//Oy4vX0EfByJ+eTVdlZgkqZdlYUTV1npVVVniM11r6/WR8cqgIXGZbgyH0mMUQEZj04vV205Wzu0rhdrsNk9g6uFxvzy+vOx0g5x5GpONRI4dEI6UQUghRtx1Lrju6ct+rRauM8aPQWrvd5WEcrdfrk5OTNO3XdcO9IB2OXl1czjdbDPBoMNxlWSNahugw7Q0Gg6ePnwQALy9uMUS+z9fbzZN3ng16/UGvV2+z/ene+e31l+evu7505wrv0m9AudCPgEMAgNVi+dUXX0ZRNBqN7peL2/lsm2XL5bIj8TKEYz8oi/rxo6er5aYWbRTH2zxbbdbnlxfpcKSNwxAt54uirlbr9WqxJA7ujcaT4ajMcoyptkAoc7C/r1uxf7j34OGDqi7attVKxWGSF03T6qJo4iDEBFort8UaIIMQ8oinGoW0LYstdEq0pcdxwL00THzqBaEHkGM+FkJ054+8LFgcoNDT1nbKASVaXdSuFqJtu1IYQijPyygItVRaqaIojHZdstbzvMvLyzhKq6pxwALorLWc+9Y4IRUkFHke9ry8riEgnudba0rZtgbWyimltJHn1zek2xCURS7l8/ffw8YRaxGmQZxURQ6dLcr6+ubucG+fE1qWZdM02/WcMliWZdsKSsK6qGVdibY0wArRVNlufnvXCHN+ebvb7Z48fmi19BjtDwdlK/Kind2vZdEySNu2jZLw089fXFy8JYTEUVJVbRgmo9FokMTnX7waRD1GcBCy4XB4eXGNCK2L2jZNAOA4GRzvHUVR9PbNq9/6/vefnJ32o4RS+vbyZpNX693WOcch6IeBs816dcchPJxOol6fh4GSkgD4/PGzRycP3776+vbuZrvdrlfbP/iDv/HFF198/fXL3W73+cuvWq1Xu+16vf7lxx9/78Nvr+9n/+Xv/R7qjKl1XWtrGymEVsaYzm2/2+3yPA/DkHocM9pR0btTPA98wlnR1jzwmcfzquxCCJ1ds9jugHXOOYcg9Xiaphgij7I4jJSQwDkEYRJGGKHhZFzX9bA/0FIBACAiEEKMMcZ4PB53p8i2bbebTZ7nnWHSGdu2rWwFhFBKBTBSRhNCujBy6EfIAehsWzce4520ZTgcNm1lrTRGBSELI2+7WyEEPJ8tl/Omacbjse/7vV5qrT19eGaMWa1WXuD/4he/ODs7m81mi8WirmujbBr3Pv/8y+VyDQG+ubr2PC/wo64bmfb7Hf2YYtzhtoF1nLJ+2hsOh8+fP+ect1oSj/lpfHVzs9ls2raVjQw8H1hnrX358iXn/MMPP6zLymrTIZl+VdrU2jkXcK8tqmK7k3V1fX0pWwEhJhRCjJ1z3KMWuI8//vj68mo8HJRl8eydx998990vPvn0vWfP9ifT7sO7LMs4jvvDgbKW+wwA0OX9tdZe4PuhDxDsJykntKirwWDQ0V201vuHhxCjTvSDHDjZP1zOF/f3t71eEkUBxnA0GX/25Re+H/7ad399/+gQYmQcyPNdXdcEuLos9/cmi8UCEhwEgbX28PCwMzgyxpqmMUp3/M66rsuiAM4Rwuaz5f7+vnPu6OioS18458Ig6ODj7r9JiQAAkiQOuFeWFSJYWrMtckQwIrgRNeVMtkI0MkhS4Yy0ZrFahmG43m4shl4ctkpqIQnGshVN07x69cpa+9UXXzR5+ezRk+1qrbVmhJZZ/rOf/UxL+cEHH3RSzadPn3abmHy7c8ZuVuuzkwedSwBi5IWBBS5KYgjh1c31NtsRRt+8eUM5Pz8/X61Ww+Fwt9n20zTLsv39/V46yLJCKsVC/3s//IF2Noij7Xr98PR0OBr5YXB4eNjv9ZbzxdXFJcXk7OSBtQBCvFpujk9OCCEvPv0lpZRRyplPMJtM9uI4pQh39C4h29FkYoCBGDWipZT6XsipxxljjEGHnIWtlBYAhDFArmlbTEjnNol7qYFAW9MNfzp+H7DO5x6GSLRttwj5uxf9blbWzYuMMb+yumO8XC67O27njRoNh2EQKGMwZ/PVcjQeAwP6cdLrJdZa4yz3fdnWGKKum0Y6p6iz9/N5msZN01A/aISKwjANozhOh8NxVVUd+2WbF+PpPkCww/0zRoIgSKIQOVDXDQ9CpdTZw4de4L85f3s/m33zw289PD1VQmRZxjz+d/7kpwBRYJwRcjwee54HoPVDbzweb7eZR71B2sMA3tzcYIg+/eQzpcXF9dW2yIVWX339tTTaI/Tm8uru6mYxm3388ceUs7vZfZZlmJIf//hvR35wdHQU9/pRFH3x2eefvfjk+MGD6f5eUWZnD061lre3t60Q89WSYPT7v//7f/STP66EeH3+VhnZNM18PicAZdvd+eXFm/M3XUmYEjS/vQkY3RsNkS8dapUScpvteoN+3TZJv1e1za6ueBqHo37WVEJIaAEEoCOOIkp2TSUJ9NNYaCW0IoT0RsNKtmVVFUVxeLDnnLEYIkYVsG3bBozrsvEB9hxCxsVeoBuBLdjtdgghoE3qh9aYMAg4oR7lRV2XbRsEYV3WoRf0034YxUoba50xNg1iAnHbCmF10EsgJcBhZxG22CecWOchzICj0HmMSynLsqSMhWlgkcLIadH2+/2syGvdJP1ECQ0thMDu8m1elOvN1kvD0f7o9ddfM0JefP4iTEI/8hfLZb/XM8oSSHpxb5QMnbBGuaZpmqaJ4kA6tcrWgyTdrTZiVzw9fYgdKLM8CALn3OcvPg3jSDl7u143wGzqwotDCCEjVDRNEARZln3/B78RhbGR+p0nz5qyZpgyysuqjuKEe75sJSO8yduQ+8SZgKG946nf64nWtY2lGBNjZlcXPZ//o7/7O8O0h6D5+svPDtP+P/uP/qPVdvsP/u7vCCG2eYEQgQYShKF1CEGEoO97jFLTKqht96WUiuOYc77abrpC0GA0+sYH74+n04Ojo+4WdXV58dEH3/p3/sr/8B/+3X9w2B87Db769Mt201x8dsEk4zgoi5YiD2oUMc9IF/rUoyTibHs3x9KovNJ12wuilHHYCp9xa8wwTjezxXIxA8BeXVxabcb9wXax9jDbLTcUEdnKLn9NCPO8ADNqnBVSWue0s0IrggGlGDCEfNLKlnmcUOqcCwizQuVtHY1GxPPTpP/y5SspNOP+crt58t7zbZ61Wf3s9LEf+c/ee3ezWvXT/uNHDzfbJcCoacQo6amq+dYH3xyOx1+fvxFK/uj7P7BS3d7e6laM44HOmmFvmOclAMg40Fq7qarBZNyIthMfhn5QZHn3ovne977nh8F8uZweHiyXy7aut+v1zdWVtfbg+KTXH/7Jxz/b29/vUIsIobu7u6Io7u/vz1+/8TkPucc5z3cZw8wq7SdRUVfb9aYX9Rjh1lrue2Vbz+eLpqoJIV1eIE5S1apWKsgI5FRBp6yFiFqFkGOt0JAQCVxr9Xy9aZWGhDoIMKNFW7dSOGOtVIQiQlASxQCAMI4scAEPGOHQOtm0UehTQpw2BGFjTJJGXZK1aRrf9ynDx8eHnHmUsCAIEHTFelWVmbYGUUIZkm01TJPlbGaN8jiuq1y1VRiGbVsT4HTbXl9ft3UTUs9p6zyKkwARQgjNt7s3r15HzC/Wu81ms80zgFGS9iHz949OGceDXqhsK52sm8pqQ6h/v9g6jxay+uSLF3/27/2zcS/+Wz/+8d/68Y+VUgd7+/muePz0qRf4HZFidje/u7lnhN7c3MzXm0rI09NT0cjru/swjLlHtZZF03pxUsr2fj33e6EyMs9KhhnEYLFZA4qVAxf39y0ECrrheNBW+avPP23KYrXdbMudY/DLN28u7u7iYf8nf/onvSiUQnz19cuzx48ur99iCjREu6o+e/hwtV5Yq3txsltuT48eEOzu7m72RmOgzdNHj0ejweNHD2i3dvcZp5xpZ5ebtYUgy7KmbcNekteVcRZj3PEG4jju7pIAgKTfq5paaS2N3u121tqyKBgmEACGSZZlXd3RAgcRElppqULPr+u6i59nWRaGoQUOIcQYq4pSa52k6XK76RzZYRS1bdtlYLt8cYexRgipVmCI4jBq2zZN09VmjSnxwyD0A9mqcpc5pTF0HR9jOOhrqaRW3bGi2+BjTJ1xR0dHWVEABNMoXdwvNptNGIbD0SAvsrIpr24uzx6dDof9NE3rptntdkmSfP7FF8vlcjwe73a7xWJBGQMA+L5flmXdtlVVPXv2rHsfHR8e3Vxdt2379OnTjoJdFAXBTBpbNXXSS3vDASFkMBjl+a47kvi+v91uv/76ayHEs2fPCCEd2Eto1aFFpFZCSCEUBtgPaJz4q/WirCuAEIIYOuRT3ub5R9/4xt/863+dG2Oa+m//V3/z//P/+n/c3V7/md/6UVPVf/a3f5th0rES26bhhFitlRBtVXOEoyCyxljrCGcWgrKpu4FDFw/vJtrc95xz+XrLMfkX/7l//ru/9tH/+z/6a3/1r/5VUTdPzx6dnjxQTXP75uLtV1+/8/hxHAXQOdmKgDGPAOCckrLM8rZuusaAbNvZ3V1dVVEYQutCz5/P5wAALaSomzRNi6JYLpcWgu531/U5KaVSyo5CAa3rYviUUi0kY6S73gGMAEbM8zqxIsK4u4ZCTKRWiOCb+zvme11cIfRCoeR8te5o3cTjlzfXw/5g1O//0R/9YTfMpZQapZB1CNi7u7vlcimU+uqLL7VUSsi9vT3ZtEYq1Qij9G6365SkQRDUbdNKuc12VVV1OfqOJ/57f/D7SZIggs8vLw6Ojh4/fkwxGfYHQRzd399/+sXn7773npTy9vb2yZMnq9XK47zX6+3v7x8cHJwcHZ+envaSxPM8DEF3dgEQI0RCLwQAKG2llIxQijEhRGgFMDEOWu2ctYHnIwQIBU1b1U3FGEMEOwfDMFws557Hg9BzwAzSnmqF1rqoK20NhDAKAoyxUBJgtNtuMYBd97Wu62KXibrpUJS73a67xwshyqpxzvlB0B8OmMdfvHjBOZ/NZgRjZwzDZDIdEQimk5FRgiJYFBlENooDDKBuReRRRrC2FhFa5sWgnx4fH4/HY855r9d7fXFeSxH30qqqhsPhsydPtqv1dDSOw6jKC6mMBm693Zyfn08mE+dcWZZKyNAP/m6NeT6fF0VujCmK4gc//JEQ6snjZ8PBuMoLCGwURUqp1XopRHN7dTsd7ymhDw4OqrqVRldV5XmME4ocAACMJsP+YIQpAwAEoRcEXuenLarKAlfX5XQ07mRb1lrqceZ7wOrQ5xDYfi+RSinj9sZ7slXz+fzm9qquClGVw/7gr/21/2dWZk+fP6WebwAkPo+SBBMInKnr9uL8qq5L4NSP/87fzvP89vpGigZa9+TsDG1EVRopnekQMUopYwHz/LquMUJAGWIBtM5Zm+WlkJpSigBUTTtMeghCAIAXeFEUYguAsT5hPvc6RjwCEDrQSgER0tZsikxTBEK+zrfYo7u6RIxyQlUroiTO23orimTcMxAQzjBCjFLnXNrrYUJaIcIg6Ig8cS/dZjtEsOd5q/miY4FK2VZ1Tp3phxEixCJMA5/5nmzaKPB8nyOKqrwcpAOIqNYGIbLbZBgw4MhqtRpNJhYibZ1RMo2jYS/94MMPotSv2koKuzc5zLdlURSHpydexPYOJ9K0WbtTSDtoCCF+FGZZmUa9cluKRo5GI4OAhi5I4k+++KyLMTx6+qSD+h/s7ddldXt1jRFdLBbKaKGFM8YI6VF6sL9XVdXzb7z3oz/zW00j4zhmmJRFgTFWWitnKaXbPPPjSGqFHOCIAGuFENiiDg34i7/zJ2eT4b/yl/6Zh+ODD9794OPPf/nw/af/p3///3x6dnJ28GBvMOGEE8KAsQRhzHAY+kkQ6lYAbZSQ0AFonbZGauWg9TzWtG0YRVEQlEVW1/VsNpsM+u8/e/aHf/iH//J/71//h/+xf+T5N979L/6z/+xP/+RPfv6Lnxw+mApRZLuFM/VusxSiAgAURaE0uLu+o4TvHR0fPXiQt7UG1kFIGFPWFHXlnAv9oD/seQGHEEohss2WEMLDwBG0KTIaeJARDVz3Qeh5zGlljYoD3yPYSqFaRSACEAqtgLHAWE5p94ABAGgYxsO+kSqNU4LocDTBhK2Xm5BypO3LL75uhNrWWa7qvG3nq3Uch8vF7Hvf+36a9P3Ixwx/8otfdqQzYPXh4X6vn6S9+N3nz6SUjRAaQ0PQ8eGhUbo/HlGfp3GcbXdZlndQaK21aJqPPvxQa53n+XQ6rdtms9tqa6Ioqooy8gPGGHDIOjiZ7ldVgyB88vDR3WzmIFStWM7mbdvywM+zMtvmlPu7PMuK/OjBSZokddmIStataoWO0wQ6kMQhxRBhEMQRJJhgr8qr0A+Q1tRZIMVkkFICCMG7fAcxMEYnUajaUrfVOI21qEOPW20O9/b7/f4uz7szmVCq+5EiALvDDXRgPBzJDtuNiEfZ/t4YABeGkbUuSZLDo4Orq8vHjx8n8WCx3FojIQRRFLVSK60hhKqqbNsiZ4VoDLEOA+hQ4gUegtCpWraQMuCMlaItC6XULss2RfbgwYOjg+OvPv8i8gPn4Hy59KJwtlwkQegRus12ADsl64Dj5XJ5dzPnJOLIXy5XGCHO0KgfQSkv3ryNgvBP/uSnf+eP/lgp0yo9ny/ns9mo3/v66y8htG/evITQOeco4Zz7s7tl3TaYEOdMWWTT/tApwznNygJTQgjp93pH032VV0CZ5XqTt22Qhsenx7fnlwM/ubu8jXjYi8Lri/OHj58eHh8f708Zgk0jnLTru7vj4ehouv+7v/MPVVk2SCLbyj/zo98Mo+B+dkcMQgb+5GcfI4YDxtbzhQYw7g+EaMMwOH30EHM2mUxef/Xy/PWb11+/QkJJB4EDwDhrrB2ORr1er5skQAhF2xKEgbH9JCUIdyt42+U5mqZboXDO67p21nqYduLHzmAppeyytFVTR72U+h4iuCh+xVZmjGGMCWeYUQMBpqQVAmH8K8MsgB3ST0r5KxWq1t2/Sinme0VdIQBHoxHGmFNmnQlDPw4DIRoIodDaAdC0QlnTHeWaquqlg+0m18pqC4IgUMr00yHFnHkeYfTs9GHnet9lm/V63alT9w/3VqtVWVbd2NcY0x+Nf/niE4Dgw0ePtnlGODPOdoMLCGEURaKskQOr9bo3HGzL/OT0QRRFu90uz8v1bnt3feNx3u2coyhqpexQHlK23VJhtVoNBoOrq6tnz55989sfLBcbpVScJBhjSDAiqJRtXpcOIKvdpD9sipz5nHPeSFEX+cF0+r/73/57f+Ev/M7LL1+aqi4Xi3/+n/ynPIgPRpMXP/tFSOn777wThuH+/r6WqpvAtG0bxzEAQDsdRZHVKgxDPwq9KJRGQ4yapkEYVkX+6quXTx49tlp//6PvZusNpfS3/8yvD9P4X/rn/pn3Hhz+t//C3/fus0d/6w//5re++Z5Rsikrz+OOkOF0ggiFAPR7Q6F024pGCsqZMsZBQBjtOCeMsbptOvIJIcQB0AVXOidRN8roDuyD4TAvCmUN873uCwAQRdF43HfOQYIoJZwRzrlQivt+HITO2jzP19tdlmXA2F2epWn68PQsCILFYsEpY5ienJyMJ3vbIq+a+t1339lut1KrX376YlvmZVkmSfLtb397MBhgCJMk8cLA87xHp2f3N7cffPBBr9ezCEwP979+/bpDNdRtu16unNJG6clkstpsnHPM82aLxWw2G03GcRwnSXJ0dORzb71eB0HQ7/c7oStjLIliBODJ4dFqteoPB0kv9bn38MEp5axp29Fk3LTtZDKhlA8n4yAIFveLfpq++/4H6/V2f3+fEdqtIoLA6/dTz/MAQFVVDwaj7oGNwzgKImestVZb0xsMAEEQoyiKmqYJPL/b8WitkzgushxD1E/TwWAghPA53263nudhSoQQ3ahWakUo7Xan3aoMIQQhZNQry7I7OL969QoAsLe353me57HhcKiUapVGlDhtnj97J4hi5gcWAemUlNLjgTG2+0YphTCtG1G3opUijuMyz7er7dX5xfHhked5FgDm+QZYyul2l2sLMES75ToJwvl8XhZ11OsppShnDmHqcYKgkcpozTCNoiiK49v7m/2Dg26V0u/3s92mqcs4Cgb9NPD5B9/61nq7HaQDYOxoMD48OLi5uZJadnP8xWIlhDBKJ1HsnFssFl2f6/TxI8Kwg7itWk7Z7cWVlaou89nd/cOzs6urq/V63VVYR8M+xng4HHKfYYylaqum3uy2oc+1kEKIpm2366XHeL/fb9tWK9MfjI4fPAjjIPCCi4uL2XKhgH17cb5/cPT8vXcfnJ0i3crA89u2BQT7SfT24tw5A6yxQhmpwiiWxiBGd0WOAeSIIAcIRB7j3aHbWtvULUI4CIKu1N41Bj3Pi5K4amqrTeD5UishhG0lVtZJHVBOHcg3G4CRcKasK2csscA2otvXKaW6DFP3+5BSOud2u11X++acd4zfoijqujbOIOCUaJSRHRQBAwgsdM4ZZ6umARZ6zBdCGQAp5ZTyzTbzfF8LGUXR3sE+xKgsimF/GMYJRMTzgv3JNIzjxWo5ng4AMOtsM55OlDTOwjAZGkDbxsR+ssnLTZEhggEw23zTtGXIGcNkOp3WSoRxtM2yKIowxgBCKeV4PI78YLVcUkK47212WynbJw/PpJF7R/ssYIRho9XR3p6oqoenD771nQ8McqWoLQZaK2W05m6er9tSYg17cdyLI6ARQgwSjDgyshRtvf/w+R/813/yw+98989+88NUgM//6z9+kIyq2cJn2ON0f3/atnUUJb4fEECTuO+cQ4xCigy0FBMjZJLEQgnueRYCQrFRejqZ7E/3Ls/ffPdb36IQ/Xv/y//1kydPHp2d/s/+yr/2P/83/tXf/uCdB71omiR/+Z/7FxI/HPX6b17fEBoj7ltCfS+FgBSN8oLY8/26KEVVy7btAGHA2Laq00FfO7tebox2GkDlAMIUYoQACCiTVW1aQQCgGNVVGcUhAE5K0WpZthXxWaXbSrXUYz7ngedx36MeP332OIyjUX/AIQbWtXUzGAwsBIPRqJWqLqqj/cP+cOgQ8hkjFqzXWyHU07OzqszX623cHxgPNUhT5m1X26zIN7s1QGi5WUspyyy/u7ldLZYeY51zo2oa7nuUs8525PtBGqdJEOxWqzAMtTXr7WZX5ga6m7vbKIoIQBev35Z5oYS4uLjIm2qTZ71eLw4jYG2dF69fvyWE3c1nmNG9ycQovV6u9g8PAMX7J0eIYELI/O6eQAQxHo7Hn7x4sX9wYLVRQjoINtutl0SNaKEyxIKklwprGykcwtpAhzhhMaJBo5RxTmq9LYpKKM9LmtY5wCChjRSMc2tBnVUe5sv5AjqQBhEntG6aXVkgSpxzGjrlLMa4bZr53T3GOEpSYyylhFA4nU6Pj07u7u4RQr7PZ/e3x8cP2lZ2bOFdVozG04dPHodxRL0or4VQJkn7LIpWeV60xg9T5AAyjgURT3t+vwcIwQSN+4OY+4kXrGYzo5UyxouizWZTt631qUYgDaOEhW0t+r0xwLiWDY3w/W7mD+JSCWVs20gIGOMJQsQPvaOHJ7WpL6+uvNDrDVLK2XvvPK/LvJfETdNc3FxFSVQ2JaN4t1is57PBeFhLsW0r6FGPeh88e48BkK0XQqvKmrIVQRo7ZDyfYYs49QCC3/317x4cTPJ8d3b2KInS2e3dbpdf3M6LVhAIHJCj0wPJcdEWn3/1xboqDx89Hk0njFIpFITIT4K6rSaDYRLEs9Uac9aKenZ/q7WtawEQjNOERcG2abZlJZxF/TTdbbaEkM6+OBqNrLVGa4QQQbjr/gEAWiEgwZjRX32kWyONVkoF3APOpWkqlQIUZ2XhxWEYRcZZiFFXKWaM5VWJCB4MBr7v+2GgrZFSHh0ddXtz7SxAcDQaaa0hAGEYCq222y0mhHNutQEAUMbiJOncqnVRhp4vlIQEx4O0FrXHmM+DWkhlTafJdsB4nheFMSHcKsuop53lnmetFqIZTSdSK+2MlHK93szuZsv5AmO822baAkr5zc1dllUEe0oZC1zSSy+vb51z6/Wm2zF89eWXAID9/X0/CNq2RQRDiqXWlBCfsDAIOkiOtiYZ9KTS3Y3k4u05AvDg4GCz3Xph4IUBIUQ7q61pRd1h/DDGHRlxNBp98MEHf/Ev/kVrbdM0QRBoaFoArudz3/cPp/u7zabf70shMMYAo7Jpy7b5X/xv/leNUd//rR/99d/7fYzpcrGuywZj/MXLLy4vzxnHJ8cHk+mIMuYAAYgph1pnDQKIY4yh0ZJiAjFClGjoLLAIISmarrV4vHfw7OzR3eX13/6jP/rBD37w9u3bP/fbf9/jR8+m+0f/0f/3P3369Plv/MaPrLVJkixWK2X0ydnpcDwqigIhlJVZUeVG6yAIMETj4agbeVtr+4PBer32gwBgVLeNsJr4vFStIyhNU2ttl3Tugu0dvKgLO4ZhCACQWnUYWwhh11k9Pjz69rc/BMB2f72DwaBbHhityyw/OjrqUBBZljHu53l+e309v79P4yTP89ubG4Lws2fPpJTjvWkYR845iLHDyDHy9uYKItSduSjGTVV1Eh/P89q6jtOk1eri6rIbRnfwpTiO892Ocy6tIYx1Z+q3b992NijRNFVVRUnc0WAoZ845VbfHB4fb7TYvyyRJVCs6esyHH34IALiZ3XtR2F2Onz177hw8Pjz8+uuvoziQqhVCTKdTxtjpw7OmEdmugBA757bbLWUkTVMI4Xy56PcGTdO0bUsQdsYAa6fjiZTSWgAR6lAHnudpa9q29bnHMR2kPY+y9WLpc09bAyje7naYMx74QRR2icm9vb2/G4+pqioOo8FgAJw7OTomCMdxLKXcbrcd6GkymXieH4axcfazz7+sqno4HDHmMcYa3abjoYUQOazqNg6jSraffvlVPOjFcfzq1avQD4pNVmyyk+PD3W4HEcry3Pf9RtStVBDh0A+SMAq9sFvdAQAQgmHkG2Ocg13OKkhiiBFCwBqVZZnvh1ESIgTu7+9933/nnXe0NKvVqqyruBc75Pb3p865swfH0+m0rKsOSoEJEcpcXt9Shgkh94s59z1ltJTyZnYHOd1VBerce/O7+XL2zjtPf/7zn/+tP/zDIEnDKJFGP3v2fJdtKcb/9X/1N+9uruuqMlJpbT3Pu7i8pIx1DZ7jk5Pn7z57/uydLCsUsNIZDJxs2kHaO9jbr4qybVtIKMSkNxyFcYpu7u6CIIiD0LYSaQut2+121rluMkMgxABgjClnpRYKOsxoJ8tGGEMIfd/vmlEGARr6kNOsKpWxyri8roIoKqrSOOt5HvH4pikroJfZVjoDPDrbrLLdNvQ8wkhjVWOUhaCLkflBgBnt8ISUkLjDFUGACRF1o6Vyxh4dHQEEdvnWC/y6lc5CADEmLI5ja41pBbJus9lhxJtahmFMPV6KSmgRhv5ms4p7aZj6s8WdliqNe9C6tm68MHIAt1KNpwdWOuBwJdRqlzuLyrI0zh4cTjmG/cib7g0Yx7vNEjlgIMjqFiDqAPZ4YIScXd64Vo2Ho6KuEGOlaBhj6/mCIHR8fPzq7fnx2UMe+JdXVzwKvIDf394Aa/pJXBd5msaE4azYzWZ3BKF+nPz6r33v8cNHu12uLegP+uv5rqzyw+PD1SZLe4PRpJ9XOyGEg2TT6MHB0f/l3/8/9Efhv/dX/+o3fuM3/9X/8f/0f/l//N///PWX83J7tb4zRA8HybjfD+LEItqb7BlMFcCEM4SAAyoIAs7parEMwxAAZ60NPH86GjvnTo4PEy8wQvoB/x/9T/7dd54//Tf/yr9dCl7Y5KJ0v/mP/BPHT9//T3/vb5Ag+vztK2lbHtFBL/QZGk9SC+RmO6cUWWsYJZ16V7T1eDJM+j0W+t37V0JnGI56/bIVrbGlkBI64vPFdq2AZYGPCAEIdp31vwucohAjAKMggBDGvXg4Hpdlqeo2pPzk+DCIwrvVom4rz2Oe50kh+nFCEI57aToa8IBrrY8Oj/f392ezu8ePH54cn9Z1W1XFZDSc9MfFKt+s1tqqWbH94uLc78XSmeOTk8ePH3/15ddxkCyXSwwhsQBYdzebOQgeHJ+MRqNWiG2eFU0NEDJKO+cQwbe3t5dvztfzpXJ2fLivRKOUmOxPHHKEIEbI/H7mUX568qDMC84p44Qi3PG3LXDn5+da6yAK7+ezu9lMKPXm/OL88ury/KKXJIN+UtdFUWVZkctWvHn5psjKNO1nVZnXVRSFjOBst6EMP3x4WjeFkPVk0EsY7XOPWQuUiDyGidVGBIHnXId8dZT/arCp65ZYEHAPQch8j3l8eDAFFFsIpJRVVXmUQYKl0VmZ9QaJ70cIUWNMWRaHhwez2ayT0sRxCKFDEPf7A47x9dXF64tLxCllCBqVrTZWOy/gtag9SpWQEfO2y9VwOPze93/ts09e3N7e9gaDV+dv+v2+0Xq9WDqrIcRlWRtofN+PPJ8TWtbV/WqRV7kUTRJGqmravGyz2oMBBUzVUsjmfnV/9GD63/nH/zFXCx96TtrHTx5iirWUt9c3X37+VZFXkPBGKAzcs8ePlGg4J1VbVVXR6/VHk2kv7mFG904Obtez2XpTS/X8+fM6KyjCWZZNT452oi61wD4djUa3N/dR2ru8vUv7yXvf+oZhuLEqYPT+6pxRXzTibDDd8xPYal2LZycPbl+/UVoADMI4itLk7u6OMfby5UvG2OHhYa/Xe//9b4ZhDI2lEKm6VXU7SHvPn73TT9Lbq1uUJAnGWDYtcgBjbIAL4yhOky603u2RuwORbIXHGISQUMoYy3Y7isl2te5GNE3TdJImCGEHjG2qum3b3nAACVatKIsCQCiVCuLIImgh8MIgSRKtlMd4L0mKojDA9QcDa203s4vjuJPFNG1LMVFKNXWNGe3yy9vtFiOSRKloJKZEGS2EZIxpawwCUb/HA58jpkUbhuFmt+0OxXGaGGc555eX57PZrFtyQugIZ0EUQuuqohZCVVXTNEq2hmDWabpOT0878CTjpCgKPwiEFt3IrAvwGu2apkEEOwiSKEqC0DnX6/Wu7+9u7+97vV4/SU+Oju9ubrpwd1s3w8GAUhrHMcZ4PBxqIfM83263w/6gu9Mc7h+s1+tekvylv/SXfuu3fxNjvF3vcEg///xTzql1Lkx7B8dHUsooTNpWV626vL9NR8m/+e/+lX/53/g3V618fXP39//u794t5/snR2+vzwfjQRQFbdv2Bn2AoFJ62B9EUdS54TnnEDpr7WAw8Bnv9XpPnjxxziil+v10vVhyiEXVOOfG0+lv/vBHddX+i//Sv/Jn/vzveMPp28X64uZ2vdtez+e385l1ajLsTYb9NPaVEV7EqqZO0qjXS2aLeVmWvu8Ph8OOFx9wzxgDnKMdL7usnHMe55zz0WiUpmlVlXGadGsYQgjlXCjZOUK11n4YdEN5pdRqs86rDBjrjPE8z6MsCEPrHOWslUI0rc+9Mi+cM9P9Kfe87pzbYYUYJ0kQ3t7cEIju5zPKWZ0XHqb7+/sQI23teDrppsyr1erF558dHB/1J6PNbiu1tgCMJmPCqNAKQtjWdQdZ1FrP5/OOUJjvsjgMOwzO/v7+arWClAwn4/li0dWdMATj8UQI8eLFi24I2TSNbEW23a026/lq6ft+U9V3d3d+GPSHA0rpYDB69OhR2u/tdpv1epkmUZqmnFOtdRCEoR/OZvPp/t6Ds9Mw9LMsC6OEeUFWFnVd7u9NqjIvsrxpGoKw08b3mDEKY1jURZjErdIQYqM0AMAC0D2M2llMCEW4zIuqKJ1zFGGKyd7eXiNFVbcOAD/0Opir1nqz2ey26yjwZNtCBwkh8/m8S2dCCHtJLOom9MKmqj1KCHYd/QZDFPuebpt8s0EEEoK0ELvV6nD/oC6LMs8JIaVo0uGgqepenNRllSQRQbTI8zROhBAOwt5oyD0PYVzX9d5kKsoGW9iUVZUX3PeUNQ8fPzo8Of6P/+P/pKqqH37v+0kYLWYzhME777zz/Omz29tbCxwkFGLEKP365ZdCtNqayd40LwspdFu1YRhapRfL2XQ6hRiFcfT21esP3n9Pi5YRurhf9NIB4aSR4vnz5x984xv93rDDNNVFKWTjeezkYB8BePb40eN3nltts/UOE4YY/+SXv2ybJvB8IcRqsUQOdKNpznlZltvt1lp7cX3lnMOM7u3vr5ZLBKDPvaaq37x5E/oBqst8u15SghAlEtja6Uw2qyJrZWuM5pwPh0MAAAbwMBmgRhlhIMRQg5PpkUcogaiuKorJ/mhCHcQW9dNB98hN+0Mf0yovyrLEAA7j1EckpNwq3b0uVSuMtc45bJytBQUg8n2tdRhFYRB0rEehpBcGRVU6a502QRBEUcQCf5tnURRBY5tdnQax0loB3e/3jDGKoFLpQuq8qsPAs6pVVkZpFAWhFrqoGwNcW5ePTx8AgMqyxARoK5HHFpstgu706DD0ws16l/b7ALmA0dj3EQTGmCSKlTStNsjzHMLGgaPTs6KuAu4ZITAwBKPGil1TJUmitb64uKjqOi+LRgnG2MsvvxoPRydHxxzTvdGYW/z68y/3huMvP/8CQjjd38+rcjweR56/WSxFJSaDyWKx2O12ddXc38/OHpz91g9/89vfP1hhMgABAABJREFU+w3TqNntnbOyvzc8v7t+75vfIpSXWcEYi+PQOHs7n33wne9eLpf/+r/97/xb/9a/BbQ93T/Z3M+TMJrPl0ESaewQgQgBD+GYUAbA0d6+QxARwn2PciLbGjnbi8LNajkejw8ODh6ePDiYTI8O9+um9CL/0ZOHv/MX/vw/9U/+Ex99/zv/5F/6x//Dv/Z/H47iT178qWiKX3726XaXPzp9ME17i8tbVUtHQKUkIoAyFqeJUCpO026qsN1u+0ka+cEw6QWUH48OQsBC6vmAcINGUa8fpZevz9O0r6VO0x6EKE16SmnMGA8CixEgZFeWyOMKQYthp1moqqqVMk6T9W4r2pYzZoBzzjFGIYSrzXo0GZdl6XEex/GuyIu6vL29Pt47Ov/qTb5cFvkO+75wrq0bBMFPP/7TOO0Th0Pi60Y9ODgRjSyLWkH3p5/8QgO33K55Gr29vSaEYAA7gB3D2GndVcOiXiqlfP/Js6PJXi7K/mSwXS6+/uqL2Xb1+uYSOBd7QT9NGWNVUxvgpvt7iOCqbSCEo9HoYG9vPB76Pvd9fz6fj4eD9XIBHajrerNcFUWx2KzDNEmjeH8yHfTTxWIRJJHUsiiKvcm0y2tpreO0pxxeZxUhLEl6q9UqipLp4RENw9Zai7BUxjloLdDWrbNKA6Sl8hhvpaA+Jx7PqlIjYIGTTdvjAdQWGccwYZTmRZX0B61sLHDAobKuoihKkuS9d99tauF5Qb/ff/Xq6/F4zBgbjkeEs81mrY0cDfvFbostaKqccdzKBhFoREudizmfTAfbMjcEMgQXNzdOqYPJ3vF4jyFSiHrbFJiyMq8gsMNe30OEArJaraTRrZatVjTwWBDWQl7c3CRJj3NunQTUlaaVEGZ5dXe3Jn7UG41fv/laVPnR0ZHRzhhblvWDBw/G47F1bv/gqMjy48OjIIotgLPFgnkeY+z05Oz2+s4ag40zTQM0KLLy6ZNH0Fln9ajXRwpMk2ES+IM0efnl13e3s5ubuwcPzpy0GEDYCpkXV2/OjXGGoHmRbxo1z+uNkSDg6XjYH/brul7c3S9v7+pd7lEGHWAIj4f9wXC4LfKmbZ+88ywZDd5cXx49OLmfz/7oj/7w8y8+HY1GVVOSIqsDz0cEi7qymAKEIISUUqh0d/Ot65pShgHsQiaW4k2+45QppaADGCKLoLV2s153YY9ARJxzhFBTVhjjLmGZhFFVVl4YMMaQRbZtEcKEEEiwA8BpQxxUACopAUSiaQjGlLHOv96ZmKB13bgfImSMwYTkeR7HscejVrXWOghxq2RRVjyOqR8IqWLmWat7SSqsNEZlRe77fqmEtSDpDfK8pMyLktAPuFYWITQc9t9+/bUQYjSeQueyfDseDDer5Wg0kq1QRmNKu/7kbrejhNdV22hpjGmbKvA5dAAA0Ov1OMCEsSQKFHQsCd5eXu7v7796+8YR5Pv+3c3tZrF8cHy8XCwfP3wUhWFvkH71xReMMavs7Pbuux/9WtM00Ln1ev3d7/3a3ez+nWfPoyB8++p1Lx185/s/DDn7+Y//1mefv3z2zW9mRbXZbAhhyJokDrLdjhCIsA3CNBoM/t6//+mHz95NAv8/+ev/6d/z23+PMlops9vmZVljQkOPEgPDyCcB3GRbBAmmLIkTn/HIWKEkdiANozRNGYJxEIqqgdZRSnvDwe3t7ag3/B/8a/996FFCaRKHy+Xi9vZms8vqtqqqyl/CyA8I8ebrjTHO87hzbr3LhqOJg2Cb7YbDYbHdcM678qRpZa/XW2+3g16/aVpCiMd8n/n3N7dKKS8O4zimhD548AAYuyvyIIz9IOgYkOPxeJtnlLEgDOtW+Jx6YbBarVpRM8Z226wVAjIoIFquV3EcT/f3Nrt1EqWMUYrwZDh6++Z1kkSz2SxJkocnx7eL2a4RTdNcXV16jP+5P/fnLi6v8mzHOe8l6Wa15pwv16uyrdM0rZomHQ4u726G4xEQerHdOWODKKq2WRzHmNNWq07EKoS4urqKBvEuz0oD0jjJy3I0GFBKZ7PZdDoVbdMfjquqwh4bTsZiea+UKrIMAJDXhRcGt7e3GON+nBxMpnlZK6UiP0zTlHi0KnKtrdY23+ZxHDNGOece42EYbrPdTslhfyC1woimYSBFHfdCSriQsmpFl2GjlBZFxjmHBFuBhbIMMQiREAJz2gihjaSBp4w2jWEAUkyEEMg6DCAAwEEQJYl2+u7uLgmjwWAgNfze9743GfT/+Mc/fvTg9NmzZy9evDDGLBaLRrTj6cH+/v7d9bnWOg5CyNB6uwxg0OXWx4Phcn4vjYZKAUKNg8vlMo3irnCTrTdayt6ozznPlut+khRF0VQlRqhLZxpoy7pmANzflb0kGY1GRb5bZznFTjsNMGhqSRnXrV7c3UdBGIbx3nT8i1/87DuPz3abXdNKxGhVFwhTSjkjdLG6gdZhj1HuQUw8wjrTU6/X22w2WkgvDDqtsdb65uZmOBy+ffU2TvuiKLLVTjdKC8cIgQ4sl8uHDx++Pn9trQ3DsNVVlW//8G//UZSkzONhGJamrdsm5Z6DqBFtEARPHj4uy9JApFoBAdDKamQp5X4Y3NzfRWl0eHK8W61DPyhAPru9+dr3PM9DoUeLopgvl5TSgPEQ0YEfMQd9z7PAYYw9yhgmiJIWO0FA1lSI0VrLvK07Y6doFSEMYAIw2Ts+DKKwS60hRJxzRulBry+EsBDkoilE45xDEFJCyrJspSCcdcCDrrJslCaEWAgscABBzrkxxmPcD4I4SQCESiklZeD7XhistptNkUltOA2Mgg4RL4o7jb1PiVaiqtu8KGXVOG2wh3JZWWuNhkXZIsYBALtss1ysAQB1vjOiGU8nxGPrzQJAFXAWhf7B4VEHPBkOh3mxk20d+97BZDy7uY487hHMCWbMq+o2K0pKeb7LkjgWQiRJsltvqAXYuLMHp/Gw7xjpdI5NWXHKCGet1Y+fPNntdtPJPif8zau3Rtl+f4gxNdq+8+TZ1cUVQ9RnfLvZXF1d7e9N8mwdx+E/9S/8d+ebfHF182h/vx8Hz58/K4uqqmqCsBRCNqau1HaTcc7rcvfVV59Np9OiqHrJkMNgt8o5QBHj43462h8YamjAuedHUdobTNP+QBqdJAmnTJXNMOlts81qu1nPlxwjKSXC+PD05Nm7z588evzk7PGkN/VJ8Oar1xcXl1nVbKtGGnl4uP8bP/pNacGLV1+VbUMcQZpQ5rMwrmT79vry6OzBcreRzkCM8rY2wCFO71cLFgW9ybhSSlpHw9BhhDhlHvd8v2lbY8zpwzMLUdobBH7kHFTSxFGa7QqOudXO9/0gCNI07Q36p6en08GIIrwts2CQYIw7lIIy+sUvP2nKaraYb7NdSHnqh9/+zkelaByChOHGmKyqPUo4Zb3R0I+ju6trZEw/TkLutXV9cXHx+auXo73pe8/fLbLcArfabvZHk3qbN0WZxsl4OmlE6zBURjtnKAQcI2PUbLOIBomRiiOyLXNh9YPJ/vtPnz969Ah77H4+a9v2zZs3o9Hoi69fFk1dlmUchIiiVrXdK6Osq4cPH+a7rNxl8/lcKHlwfLjZrK/eXohGrrcZ5b62gBKipPQ4BthoJ3yfD/sDyplWpheFHCMj9W6XI4LLusLY7e2NPI9k2ToMw/U2k0JbbaHR2FljjLa2bVtKaed1QgCqVkCCpVZREMpWGOOUNAhAq3UrFPO41hJCuLe39/bNhRLyO9/5zmq1IoRIoYIg6FL/L199vc0yCGjbSAPBpiwd9bNGbVdrRqlSRlnSGCMMcM71k55xyDrUpTAWu40fR6audV07ANdZFodBU5UOwHQ4rKoSOhB4XhJFk96AASSKAgFgCWohqKVSyqRRCpULCDoYDe4uLuIwsg4qDax0e3sHjdKL7bpuq6rOfMqW89XRwfFsthDCJcko9KPtas0Y++Krz73YN9YeHz/wqBcloQVmvdk1QpatePDw7HBvdHNxHocJwlQ59dH3voM9BKFrmgoSnE73NeO7ssp2BdL20eHR/qiPrUKtQgpoYe/u5zzw5pvlcrv7xje/RTAII6/VykCEHPIwRdhqIEWtoEaqbgNKpr0eAVCU9VcvXiCtrXOQMUYYLeoqK4siy6VoldRK6rpuEKUGuLKqLHaFrC2GBhgIYRz6nPMuyaCUwhhnRVGWZbci715eTWfyars7H6Yel0bXVcUpc8514RBjDPM942xna+yKbZ3MPvD8uqz2x/tt227W67zIKKXdgLsuyqZpPN9HBEqlMERJFFPCAUAdNdRqjQD81THEgiiIhRAYQ6laLVshRJ7nnV+mQ6FRQqqqmi+XQgjnzPz+rpcm19dX13e3/eEoSuJuo3BxcSFk8+knL8bT0ddvXt/f30NMqqrCCPm+L5vWJywNAwvMV6++TtN0PBjPrm+BNldXV+mgP9mbbnf5O8+fCtHMFvN00P/0i89323yz2mJI0jRFBK+3m+V6BREQsh32e4Tgbt3/3rvv7jZrI0RVln4Q/cZv/dn/4vd/v61LTglnxA8CrSxjXpr0lLLr9Rpo8+bVy9nsDmHIfa9uhZLGWndxcbldb5A1CNj72W0n6pVKOAvTKBZCrDZbAEDAvU7WQQjb39/3fb+Voj8eDcej+9s7LeR8ufjqq68662kjWiHUcrFeLpfdFPuXLz6drzeUsfV2gyABzhkHOw7frwblvh8EgQGuaZr1biuVeve996U2y/XK87wwDEM/MMreXF13/diu+vjyy6/SNH785OHB3uRgMv57/sxvPXpw8v7zd3/jBz843Nvvxwkn1FrLCBFNc35+rpRClBRFEfkBQZh5fLvdIgwQQsvlPPQDo/XBwUEjZBDGs/k8r+rZaq20VdIQhL/369/nvi+lPD4+tgDM16tXb950wCLm8U8+fUEYBQA0ZXV/e0cQStN0NBpt1xvOuRcESS/tJstJFMdhVIomq0utrVJm//Dg/W9+kPYHCJGf/PSnF5eX9/NllhVnD07P37z1PG+1WcdxTD0upDTWjkaj0WhEILq7vW2qSmq9Wq2m0+lscb/eLMMwHI1Gg8Hw5u6uN+hPpvudYJZ73mQ6TdO0a9JOxiMpWgjc0dERgjgIoiiKtNZ1UwohugRRHIZWGWANQ8gaLYTg3q8G6AhAZ6zTLgpCY600WmrVG/QBsNrIR48edaJXP4yZH/SGA+RAyFnnV8rrShmDMX791cvvfvRRU5fPnj0FAHDPj+Le1c2tNcBYWNUiSXrOwqJuIKVGO4wxBmiz2XieRxm7vb5eb5YAgKqpIUL39/eY0TCKsqIsy1pZ05lgRdMabUUr27aVsrVG101JGC2rJo7jyI+yTcYJjcMoz7LBsLdcLv/gD/7GYDAqy3q72n703e88evyYEBJ4PmMs9P08z9M05Yw9ffLk5PCoa5WHSSy1qkW7y/O6bVvZYIY9z6sbQShP0nQy3vP9UNQNp97h4eEvX3zieV5/kO6KvFXSj8Mnz56ywD95cPTeO897cbJdLXfrjZEijiLG2Omjx++9+40ojCnlb99c9Hq9Fy9eMI/3er22qgNGwyQ+ODhghLZ1szedzu/u33nvXWl00osHoyGC2KtK0VSVMQb5Pu/3LIWUMQwJpT7lYSmkIxQwoo3hlDitrJQhIcQ6XVW+xwjtPhShT0k/jgJCsvUKI7AttlEvTYb9qqkQo865tqqRdYEXCCGtc1ldEoK0lmVdSWcY94x1QojBYOB5nGCkmjZiATDASfvs4aN+HCktOyJC3EvDMMQIGSU9To1RxqjI9wJCGKIe9ShmXUkHIdSFMn3iAWXSIIh9NkojCgFjZDQeU0qjKErSlHM/TVMtZZnlz588nd3fxnFIPLors7ptureYH4Wb7bY3HLTGDPYme8cnZVUDBykmURAmcTjupdjayXSorQzi6LMvvrQWiEb2gqDNyzKvhBDrbNefDIQWWZYlSe/V2/Nvf/s79/fzoiqjNGml3OV5vss4JcvlvKqL8XTsnMu220dnDwmAgzgVVckY+f5v/9YffvynnucNe/2nz59Wsm1aWZQtAIAiTCHADvhROJiMfd8/ODgY7U8r2cRpPBgNoyiqq+Lh8fG43+Ocp2k6HQ6KLN9ut0mShF5otUOEOoiYF7TCWIoaY26XS6ENtkhUolG6cVpY2Rpxt5j/4tMX9/f3ZZYP0sHlze1mmzWNMEIbIZUVFpkk8A4mk+vzt1YKTgnGUErJCY29iBiEAN6ut2kUj4eTgAccsZj4fT9kAPWiqBfF1hgMYVHkeZ7lux1D6PmjxyrLnj047cXRaj7bHw19TBgA9Tb/8pNPX3/96n6+uLm5q3YFFIZjroR+/fUr7ntBGHbss816+fbmslLijz/+6Xq9YTxwAJdlxRg/ODjEmBit3759GyVJ2TY3i9m6yN791gf3ywWGsMryRreL7XJ2c703GiqlHIQWuNlshqyDxmKAugReU9VG2SIrQz9AhEgILCXGuVfnb7+6OP/xxz+9WyxbqcbjydnpYyPkg8PDo+lBHEYQobpp4l7q+76UmkByMBwHmAKEbub3rZaL1bLIc0ppWVfnlxdCCYxRo9TtfB6E8Xhy4AXJbpeXu2LU60eejyHyAj/ppQAjzGjVtNusiOMUAowhka1aLzcEYZ8xpwwjFEPkAMCMtlJpbYBxwHa9J2e0Dj0fEiy00s528DhrtR9FbSusQ/d3i7OTBx7BVZY/PD17e3PFQ/83f/RDXbeXb84n/XFRZMxjwup1lo+GY5/7WhpGad2qu8VyXRYGgZD7ppFd/hVa56wpm3IwGHBMKCabpg4nI4hRlhWjvT3Mue+HShlKGe1OddBJKw0wrWySMKiLHAOXZVnb1h6jRkvKuLBuV9U09A+Pj3u93mIxIwRdX5yvF0uj7KA/yYrCArC/f4gM1EW1u5/ruhWNbJUum9oRMJwOt+XGMedFnjEm2+4QgA6i2Wr75fnlycMne3sHPmVtXi5v741UzsLBaDidTrfr1cd/+hPCsDAaAHB/fx/1+g6jsJdwnwmnVpulR72PvvntMi+UUvPFOgwjChBQEjiTZdl8Pr84v1ot53WZF3k1mIy/evt6crK/qvLh8QEKowRTXNd1x2Xc7XaUc+2sMabMq7KsAcJd1tXjDDqAIPAoowg1Vd2Z2HzGIQDAuV6aFlnuUeb7vtGOELLZrsqy6BIRlFJoAHKoa5wiSqzVTunA87RVDqOOIUcpz/McOIchskKN+oMoCrq4jmhVN70BDsVR6nlcaxXFYZJERZV7Ae82yEII4FxVVcZZDXQlWw1cI4VzDlqopUzjZDmfWa1Go9HF9VVW5JTS9Xo7W8wBAJ2atcsqrNfbXq8Xx3Fd10kScUKHw6EBMOkPVvN16EdFUQAMpdOE0fVi6RMmm7qpK9G0EEKHcCv1u+9/IwiCclc8OX1sjFHAUorv7u6yLDs8Otpt88iPyrIcDoez2SIM4qqquEePjg66Oi7CeJdnt/e3QogyL5qqifzA98hXLz+b7O/tHR7+X/+D/2AwHn3/N379ww+/2YgWYyyFXi6Xq9VisV589fLlL375yS7Pyqa8vr600B4/OFFKzGYz5xyBKPYDoA2wxmNUNPV0tBfwcDlfQIiE0lXTiqYxxlze3n3x5s1ys72fzebz5WyxXK5XT549u7i9fPH5i0q2B0eHzrkuNdFP0qoRu23eVDUjtKiLMAkfPjr9xc8/vr+6YYTWZVnX9cnpg95w0PUzB2kfY+x7HkV4b28yGo3Gk2FdVgSiblLHOfMYIwgJ2bSifv7sqayro739q9evm6b2PT4dj4ssp4Q4a8MwdM6lcbrbZqJuDg8Oiqra7HYYY4/x1XLZNA0wWik1nk4aKZIwenh69s33v2WtPdw/Wi9Xu/UGOXB7ffPw4dnd7H61Wc9ms7/8l//yxz//WbeDyXdZdwUZD0eyaafj8W6zWa1WlFItlZXa97zONMQ83t1FjHYBD6ajcVmWNzd3TSMgRhChOI597lNKfZ9fXl54nuf5HEJotN6s14vFAkK43axGg2HTNNra9XajlY2C0CjdNM12u+WcD8cjPwz2DvY9zxNKWoSqugk8L0l6cZoCBCeTkccIsqCtm2y9AdZwQg/39+fzpTHQAii14Zw3TaWU8nzWtTQsBFVTU87ysqCUGmWY51lrOaFt22pjrHMeJQyTuipkK+qsgAYiSB6fPX58dsox2R9Pi10+Ho+l1gSig+neYDCwwF1dXmopQj8YjwZ702lbVxjaNE0AAtz3KKdhGBijEXLWSIKBs+r4+HAwGLx+9Tbt95XRq9VKCZVEsXPm6u7WC4NO3bzdbtM0rfKiLsswDL3AJwS1bdu2bZIkDoKirhgnWuvlZusFIWNMShkl8ctXr5Ik2WxX6/XaWiukXq5XUqurm2uAoNCKUloUxZevvg6i8Fvf+laaJAghwmk67FNKt6u1z7iDoG3bNE6Gw9Hh4dHF1ZWUsqnKtqxi32uaZrfbLecLAiAz4L1HT40UPqcOQQPcYDzyo/Do6Egq1TnalDKXF9ePHj1qpVDWxWmfENLv94MgoB4HECOCD472kyQBCHZ22TTph1Hy+s05gtAhhIQyvh8OmDf2gras1osVcqAXhqEf+NyToumliWmEqpok8BGElWyDNPGiCAAghOCEWKVW84WDQFrnMLEIcM5H/QF0RoomLzLnINCAAEIoV9ZoLQOf95PYZxRzZhFEADPEGPO69JVTcjga3M9uFpv5yaMHL776spYaAuIMCLg3v7sXQgzHA+ucgy6MI2G1ZRiHXqcATNMYEYwDj4R+KRoeBpARZY2zcLPZjQdjQtjdfPHg7OF0Op3P59oa5nkAwdlqKYX2gqiRinh8u9mVeTka9Ju6PD4+xBge7B999flXh+M9mVUMIN/njsFKNf0w5g6Ox+PHT59cvLnAkPb6w6KtoyRwziReyjAXzvTHg1GvPwhjqZQxbjIcl3mxWswu3r72mJcEURj5d3c3xmmp1WaXI0S0041u++OhUGqzzcqydEAbLVIeMMRG+0f/t//wP/jis89+60c/+PDDbzqIMGZxMjAYVqp5efEmb5rldnN+cXF5df75i0+qqkj6PS/wT45POeEM0mlvmAShlG2/30+iCGhzdHTEPHZweOx5XuIFQKu7+3leVNlmjTFOpqPn3/7ms+fP8zxvmqZuGyX0m9fnDoL9wwOrXVFUlFIv8Ef9EcPMGpBnJef84s3bXtJ30lLEBoNRUTX3swX3vf5w4BDo93tpmvRHfaGFRna+Xc42i5OT0/fe+wbAcG9vD0FcZIXR7smzp4vV0gAXejElnHJCfLper7WUom4wxkESGWfrsuwnied5d6tFodtaiziOv3jxKYEojeLJZMI9bzyd5nn2wTvP94ejr19+OUiSKts9PDkptts820Fnm6oejkdStn/2Rz+8+OLLLunfVLVRGhjYj3qb1Xa72iohjw+PsING6c4OkcaxM5Zyn3J/tpoVTZUGaUg9bHRESRLG/bTvYdePPGxcL4pa2f78058fnBwLJW/ubx4/PGUOPjo6SXh4f3f3wXvf2GxX14u7F199wXzv/ffeow5S54SUQRgeTKYc09cX5/P1qqrK8XC43q73jw6Addl2F/SS/nRctY1o5LuPnqYs6IdRxBgGAFgX+KEF0EFEuVe0pRf7GttaSw2sBsACJ63BnAVxZIALwrCTkitllDKUkMD3e0lfNPL26ppgnHj+w5PjyXjIGNluNou72Scvfnl8eEgdLDe7JIrv5rOv3r5++PRJwL3EC0JOx8Pe+duXAErPR7tsQSjAxBFgVFNiDzSmYsT5FGrVfPXl5zfXd4TRu80SIHQy3mfKhdzzQh9TjChmjHgei9O4rPM0ijimVumqKKuyjOOY+958u261itJEGI0Jifxg1Bs8OXt4/vbt6cOzd959brQ+PjzyAj8e9gVyBkPGyMnJ0c1iFg57pagsgy0FDTDFatVngSpF5IcIYAzJMOo7bXrDwfToYDWbM+Ou3r4d9vubbNMf9t5//nw6ngCMvvHNb03Sfn6/hI3c3c3rda5qeXF99eryfL1YPjg6TqJ42OsTCzxC5/N5B1bKqtI6ByCWzqx2W4twXlSNFHEvXW920ppatACApw+eLm4Wm+Wmnw5QkMZRL8WYzG/uRFEibYhDcZgYrRknSotss438gABolU6DaLfc+twLo6hVsmzqKIk9xpVQYRh1YsZGS+VsKyXGuCpKBHAURYkfWSkCzgCwnWbBWtuUTdM0UpmyLIG1UjSyaYts56wOuC9apa2NemnV1Dd3t14QBlEoterqjpRhAtFqsRZCaGcBRh0zsq7rjkLXJbGaprHO+EHQjSC7k0gQho3SmDBMydXVlRDCAqeU6mwhSRQbZ7so7maz4Zy3bXt3ezscjHe73c3N3fn5ea/Xa9u2rRshxN3dLAiCMAwZY5RSB+xutxtOJtc3d7e311m2RQhvt9vtdjudTm9ubu7nsy6vijGO4/j27qbf7x+dnECM67rSVi/uZ0EQ7R0eSaOdc8PhSNRCVqJbU8tuXd4oCsjV67dPTx/+s3/pn/mH/uJf/MlPfvLX//rvffTd73lBgAkRStZKWIS5H94vF8LqUjTC6MV65awepMlkMnn06NFutyMI99I08cMoiuI43K433fUIYxxHwXgwVMaEYXi4f/Dd736UlZUB7uDwMAjDn//yk9////3B3d3s5ubmxaefdj+ru/ns8vqqKz10LO/lYqu1fv78eb7NjTFNVU9G48DznDZ1UYZh2NWX/DBopciKvKrKyd5kOOpd3lxyn3300YebzcYYkxV5U9eDXn+73a7X6/v5sm3ky5cv9/cPgiiOwgRiNN3fj3sp49w416VRm6bp4hPW2vF4jBDCGGqpNpvNcrMuq/zq+oJTOhoMf/Gznz1+/AgAcHh4kISRQ9DzvN1mW2RZ29YXFxenxyfr9bqqqt1uBwAoqhJDOJ/Pp9OpH4ZVVc2XC4xxlmUQo8VqWdYVY0w0jdUSIRBFwS7b1HXd1CVnhCBEEYqiyDnj+35Z1/1++v43P1hvN0VdxHH4yS9/bpQ+f/MWY/zgwYOPP/54t9t4gb9/MKWEf/HZ54wgCCFwzhpT1/VyuXzv+btaKgSAM+bZk6da6zROTo6Ordaz2Szg3v7+/m632263GGNrQej73drJOaeUCsOAc24csMBRzrS1zGPU4whCY0xHFSzLssu8e0FAOaeMlVW1WC3zsmiV7oxddVF6lJ2dnkLrDg8Pj4+PG9FOx2PZttvN5uDgAGIECD7Y2+8eMUaxz7jVruOOOGNF0zZ52V3xCcRKSIIx5QwgeHBw4P4bTRCD2Gnz+ZefK6XqqpBtvdmslW59n8tWNFXBCHLOEUJ6/X7Z1EVRDHt9q3Tbtl0Xp67rxWLxt3/8R996/4O2bQ+ODp88edKd8YtdloTRZrNx1iKEhNGbYks8Pl8swjBECKyXK9G0xpjlYt3dnyghFOFilxGIekmqpPQZXy8XGDij5S7bbDersizPLy52u93t9Y3Pvd1me3x4ZK09OTv96KOPOOeB719dXXX0HoowISSO4+1u9+z5Ow/OTrt9civFYDB48OABw0S1onv77Y0ndVktZndOyaPpvpUC1boN0hhooGrhjGIUO2kpII2oNTRhFE2nU2fAZrXpUDiD3qCuG6WUc5Yx2r3iCOVaWQTJNs8gBtgjwuq6ajHmCFIMaUAY1BZy6EUcYaeF8KnHqEdYWDdyEPZ9TAFw/UHS7wUB58AhzH3k8XWWUeIBCwGwzmmMoRfw1ra+71thuaMEES0kRThfb7lDAaJKi7zK/TAsiqLnhz4kqhVdQwchtMsL6ZwEoNJiuVz6vn95c80DHzM8Gg/qvDBKOQeMsTcX149PzxywiKAwSqTUr95cRnEaRv7po9NWi7CfrIs8jhNdqWKdsZDTiAmlEMF5VUf9NE7C6XSMEc/yNurH691KigY7rAFcbncQwjzfzVfzVivr0K6seuN+1VZpmvaT/ma5WS03m+0qiaLtbOVBOuoNiyzjBE4GAyBcjIM0jN59ePbFn/5kL0n+/F/43dfnN3/wN/5GFEUIm0ZWmPpKg7pRAOJFsaOJfzu/ffz00bMnjynCRqqri+v33/+mMrZpGt/zQo87qyd742TQF8Zo69579mzQ72HGEPdDn++Nhnv7hwhzDvH1m/M4jgEid/f3zsIwjqIkXsyWGOCk1y/rhiJCMb2+vSccn52c7k+n9/f3u13ueb5S2uc+ULYfJ0AbDJEQIgg8QtBkMgpCLwy8m+vLxWzx4QcfDIfDg4ODyXjfKPPw4dloNEyS5NXXb+7u7r/88svZ8u7nP/8ZRgRjMp3uHZ0cp+Ph0dmD7rXrMFTOYASdkAM/sq1sZXNwctyI1vOY1loIgRxYLGaU4W999C2hWsyJnwSrbA0Zkk598dmL9Xy2mi8++vDb/+WP/9a6LnqjYRQnytowio1zg9FIGLWtChJ4u7JwGB4+ONYIII/5cVS3FXJai/pgOtFKcB9D5pjvzderVtSL9eLi5vZqtliVGaQ4y4rXX78ZjUbOudnt3dHRUSVai2Cr1Ww+h5Rw39N1GxAm66aoyrjfU0ZO+sOj6X4r5dnZGRSKaMsxtdrcX99YqR6cnTZNQ7T78Om7PveUkdij4SANksQCWFS1xzyCUcAINKqpCo96FDGGuZE2igMpGtW2GECnNEckDkKEUJqm3T7TQVA1bSUk5h6kJK9KaXTa70kp4yB8+ennSgjKyMXl5a7IhVL39/cI4zCKZouV1IYwXjXtYDItajkZHYQ0hopyGOrGJH4cx2mxq3TlApoaSaREGuBG6appJuNx7MWT/mhT7Cx1XhQKIR6fnAKpMXHOmbvb29FweHCwJ0SjjRHarKoyK6vxcFKss/3hmEOsWsE555yfn59H/bTDObx8+eV6s4EQImt8hJ4eHYcQPTg6zvPcOk0IGYyGw0GvWqwiRI2z6WQYhPHt7b3HgyTpXd/cQYiTMFrcz7JityuzIA4QBkkc7jbrPM+MM5zz6WRS1u3Zkye9yWR0fBQO0snB/suXL33GMUKfv/g09HxKqTC6S5lThJ8+fUoxgcA2dZmvM2xRUWR1VUQ0iHjsYWqNkXUT+wGl8OhwUmYrWWVIOt20bW84KOsGY2yV7j6pAMGtknmWNV1ZNEm643bH7mjb1gCntemINNpZSHAt2iRJAABlWVNKGWPW2qpqoIGbzSaMAqXEdrfuJTFjrG5aoSzG1FrQWXicM1VbtW2rlAIApP1B1bQAwbIsu7x82TTuvzmRlWWBALYWQACstUbpJEmQA1LKbnwGAGCMtG0LrUMIWGshhAAgjChGlFIqhT598NAYc3JywhjjnO92u+Pj4yiKhBBe4B8cHZZl1XHKKCYXb88P9/ehA1EU3d/fG2AWq6XWGgPIIaYAVXnWHw48z7fG3S3mo/H4q6++tFq/fXvx5vwccbrON6v7OQEwr9uvzy8fPnyYxHHbtr1eL+6lddskadrrJdfX10KIJ8/e6Qos2+3WWevzXwXIqrbyff/6+nqbZwa4P/3lzz/95Sc/+8mfHkym//Q//U+PBuOqqmoh4zit8sIY65wrqnKXZbPlIisKrdX15WW22Q4Gg+FwWORlHMdpFBtjnHMIQEaobFvkQOwHs5vbUa9/cHCwXq+Lurq6uhoPR/ku09Yoo8u6dhCkaZr00iRJrLW+70sp5/M5YbSTEw1Ggy75+8tffAIBYp7HfG80nVR13TRNdzRbr9dpmuZ5HiUx9fjh4eH5+fn9ze3ZycGTp4/KsgyCYDwYBtzbbreYkn6axmE8HY2DIBBK1m3z5vXr9WLZNI1SKkmSfr+vlCqLlnJuIOiUQwjAwPOdc1VT+0nQKskJla1oqto55wB48PAsL4vFarlcLqcH+2cPH/7kpz8NkjiM46qqLi8vN9nu9ds3t7d32+22bGqhVZIkvu9vs+zo6AhR0hsOnHN3d3eD8WgwHn311VfGmCRJGPNWq02e587aOAiLuqrrOgx9jtE7Dx9Hnl/VhTQyCsKTg0PR1pvNptcbrDc7g0A6GkACDXDM9/KyTpO+0Y4wHCbh5y+/ksZC54QQcRxfXV11/dsOzxB4/oPjkzdv3pw9etip1hDBrZJZmdV1fX19LaVkzPN9HyEghKCcEUKqqlqtVs4Ba20H+PUIJQgj46B1ZVF4nlcUhRAiiEJltDCaMJrXVVbV0mjK/aSXDsejR6dnHmVxHFd13RoltZrsTSHGt/d33ZNljKmlWG02cRzfXd9ShNfLDYTYKi1bKVtVt4r7AcbYKgsRsQ5o5YaDyWq9boVq2zbLss4RZp32fNa2re9ziAEieJD2lrP53d1dlz3HGHeE2qZpKCZVWVpreeBv86zV6sOPvn16etrUJQKQMfbm6sIiqIQUdbOeL/cm0+V8YbSus8JqtZjfV3kRhWFVVZBAaQ2m5IMPvuURCowd7001cJ7vx3FMCHHO3c9uCcOz1bLVKit2aZr24qStakIIQAhgFEbR+cVFI8X+eLJerfr9ftxL7+/vi6Lo9Xra2bu7u5OTk8vLy+VyCQBI09QYt1ptjAUAYgQwsiCKIgBAHKXOuVq0i82661cjgKCGzk9T7PHu/1vnsMcMAqvthhFslIQYCWOLplFG7/Id5ywIQo/6utWBH0W9iHKCCIzTSCsrWtNLB9YAoUUQeIkft62UGOSigcD2kzjbriEGygHm+W2lPOIhRLjvGaukalupCed1XS9m92EYRlHkB0RbaQGJwp6VdrfeEEQQwpAhwokDgHpcGS2UNMQpp/b29ozSRgmHoKPYYMh9L4qitm0xxnEYM0hU1Y56/dvbW62sVdZpp4XOs/JuMU+HgyAMr29uAKGbPNttM2QBhjD0+e35OVASKAMM2BvvhX4wiBIgVeT7IWeb5abOKoZZXpVRP90U22ybjQbD0WiU9NJ33nu3qMpxP/nt3/oR9QMN4HA4vru+cVJHQXB++SbPd6KqA+pzyhxy17fXmJJeOuTcu53PLHZSq6ptG23XVaGpTg/7P/nqxR/8nR9DP8ry6uOf/Mny5uqjb35wdnIa+GlZdC55V6tGWJ3nuZIyiSJG2Ga32+bZfD6vmzIKg9gLPM/b398fj8fOuV4UH4xGsef14xgZJ8pa1fV4NPSSaLHbDdPk17/z0f16aTnx49ALfD8MhOiUh1IDV7dtR55Q1vlRvJlvpuO9KO11e6Q4TngaLqtdLmvs0byutLWjybCStYau0bJum5/94ucvX768vZ0/e/bs/uZ2m2f7+/t1seslycHRYb/f39vb++6HH+7vTff29pRx04P98XAIgTNGJ0lyMJ7eXlyJph1NB2VdYUYxZ5s8s8BkxW44HPZ6vc4pCI0NEU3DaDgcrvPdxe313d2dqBsMYOj5n3/22Wg02jTl1XpOGSvLshcnVijVtJQQj7JB2oMWNGX97PHTB0cnzx4/aau6zIvJaDy/v63rklCEMZ0tN5td5SDb3zveS0dEw9iL0ii2UnELQVbGFj48OlrNZ/00DAO2nN1HUTSY7GV1K5yJ0mg0GgCrs7y2jqyLYplta9cWovDD6MnjZywIpVaLxWIymUCMsqr0Av+dd59P9/devvoaEbxcr3gcroqsapvtdguUe/7k6ZOzh4HHg8CTWjqINIBZVUvrNHBB6FlrKIac8GJXKKHaqvY4xwhxzimlxljOvW2WaWsBAK2SjWgBxX6cnDw6Oz47DaKAe2yxXr69vqSRv9xu3l5dbrfbXZ4JYLd1+ebNm5cvX7ZSIII5wA/2Dzabddj383oLMOz3+/uHh1leAgQtsFHPV1AQTpG22MAk6d3O7mnAgjiM4+TRg0eTdGCkkEZqigqhNIDGKILhYDCyCgDnCML9KFFCWufiQU9p3bZtVhbY54ZhQHG+3iJlmqK8urgURs9WC2kN4awbAq83S0rxdDSUZR0xb7NdFaoGIZ1tVuvddlMVQguoDIdYGblrci8NRtORki2GjlBaiTaXzfT0+Ol7z+fz+WQwrrMqSXpHRye73W4+m2ltsUVPjh6cHhxVdV1WVVkUh9M9ClFdlE+fv3M7nxFCjLPHZ4+Hk+np6cmHH36zknLv+Pgf+PN/f5JGm6J49u57nbzeYbatxWD/WCCKwjAkhLRaQYpbJSmlfuznZeYAGo/HaRx4jDRNgzFmHvfCoBtUVVXTRdG7oHpd13lZaGscBJzzbLtt27bz+BijrLV+ECBKlDTIAUJYUVTU43lZdp9vYRiu1uvAjzjnYRyVVUU4gxCuVos8zzCGAFqlVNMIDDCDVGvtENxlWaOktqbMcs/zEEJ5VfaGg6IoZNv6vo8coJxKZ6qmzrKMMWaU1loLoTjnUqgOQEYp7WKOjJA4jL78/IuiKDDG68WSE/740aMkSW7u7p4/f356elLlWZL0Qj/Is8Jpa5RyzjVtRTgjhHzvu78OANjtckxIfzDq9/tplC7WizSNV6sNdKjf79/d3azX6w+/892u/fvsyZP96UQrkaTR0cHxy5ev2rbtKAt3s3upDKV0f39/f3+Pcb7b7YbDIee8bOpWislk8v7771ulRdtWebFZLXeb7XS6X5Q1ZZ4WymOEcoYICcPQKOvzQCmzXm+FEFm25YQ2VVHVBYRQKdW0EmMKrIvDqGkq6OxoNPI8b9BLOSO+F+7v7ydRsMs2l5eXlDOEkJZqMplkWVZVVV6Vxhjue1VeEkIghNvtNkniuiizLMurut/vdz4s3/cRQtpZxtguzzoC6Gq12m22n3/62ac//+XVm+v33312cniyv3+IMfrJH//xw9MHz548whh7ntdPe0rIJA6dUT/44W8cnRwPekm/l3z5+Rc//elPf//3fu+zzz5rRKu69oMfdlxGCxwAoMjytm4wQqPRqDOFNU2zXW96vR4y7mC6l4RRVVV3d3dNWe12u6qq8rIQQqxWq9vrm17Sb6paKQWc22631phems5nsw6n0aGEOv4iACD0fARgvzdIB32HoJCqaZoiy6wxbS2gA4zS2d3cKPv1y9cffPDB28uL/+w//8875/CrV6/6vV7Ag9Vq8/HHP9cW7O/vHx0fG2drJTabnbUgSZKiqrjvAwBC7jljwjA8OztLolhLlW13e5Ppo7OHFBNMSZwmhBDfC9Nef7XeOOiiJOE+b0RbNR2yO5zP56EfUEq1bCkmnLLA8yljjLEuay+VWq/XAADOOSFEKOmc44QaYwhEGONOFeBgV9Dnt/NZq9XJyclms7HWnpw+eHX+BnP6ne98x2rT7XUuLs7LKvdCz1rNCaYY1m2d51mShkK0bVPVdcl91qqGMDIcDxxGQfIrtGQURXVdG637vZ6ypm4bwpgFzgBoLOjokgQSRilCCCBIGM3yvLMvTMZjhFDkB77va6211nEvfv78+bg/QAiFaXwzu4cYWeA83y+KQjZtP05ms7vBaIQo2Wy3ab8HoO2nMUX4zfnb3W5DCAn9YDabvX79WhtjEQQYRXE6nIyrpp6t1mEUv339xhjTiPbi5rrM8raqfd//4IMPfO69+MUvsywb702fPH16dna2Xq8HvaGU8smTJ8v16jd/8zcfPXr09u2Ftfbg4ODZs2eMse12SyDq9/tBHD188rgoivVqG4Yx8f0n7zxHjFAIYRD5hw9OmlpagLQzeV1YbWTVtE3VNmUUhBhgAFDTirpV4/EkCnzgdJgG2kitLIQ4CKLtNgPAVnWGsZuMBtaCoqy10xZaCCFBBGIiDODMd8Y1ZRHFgXQCc1yW+WgwqlvhEFZaAwSl1AgRQrAfcGAhxYwg4LTi3OfM11ppqxGnAEFOmbNWCSmljOO4bVtndBKFWirOuRKdbIx0L5SurV6JNpPNrio694KUsrMQREFILBglPWtMmiSPT06xcXVZbXZbFni3y3mlxGBvIrVyEGRZBjDaFBnitGybStanp6e/+MUvvDAAEFLqX13dCaEMMI2oJ3tTWQtRyeHeXqsVgaDOt71BP+6lUus4DEXdDOIUcxanSV23w96wyPLDw8MHZ6etFOUuM0q1TZOm6dOTB9TBpmwSP373+Owk6v99P/jhN955dvTg1AuTo+MHhBAhKz/ghBBrQRAExirsUMRDjweMeXlRWeAIRFqJ8XDgeUxZU9W1c240GpV11XUx1vmu0bJs6iAIDibjoR/0vHC+Xl3e3ly+PTe1WNzcjIb92+ubzqMyHA47+apzgHOulErS9ODgoFurJkkipc422/3xSNf1qJdGHjdaYgR2651qFUfs5YsvX/zk59U6/+bz57/73/odo+1sNoui6Bvvv/vm9evdZiuaVik17A+A1dPRWGvdKRhvbm7u7m6qulgsZpssh5RBjIUQPmVNVjCAsANdYdKnLPXDkPha6EZJHHjD6WQ8nsqsjpHHDJgMhsvN+nY+M8b4lLVFk/BoNBgnUdpPB1cXV53ExlpbVVUHFXhwerpYrz794ov33/+gw/lCgOu67U42PmNNVWZFhn2cycpAW+XFZDwuimpblCAIrOcBwn7ys09efPmKBgkmPnD4YDJKQ38QxFCCDz74tTgdjSbDuinG4/EwHU0GhyHv+WGIKfQ594n36Pi4WK99zkaD3iCKdF0PeonP6c3bt8V6TSAAzgADTk9ONmXeP5jWVud12W2AoXU+4wFhgyjBDhAHo+BXmFzjrDTaAEc4k0ZbCNJBv3NxAIQoZW3dlnnJCPO5fzQ9oABfvD2/ubqqRPPwnaeFaOI0OTg4GPcHx/sH7zx9lvR6m92OMXZ0cGilHvRTSxQOcDxInj17/uj4LPQ496AX4jhiJ4fjNAqNcVADCF2pyp3MatMI1VJAHp8+vL693OQraTSAmDGPEKaE8Dyvktohvlqu18uV7/v3s1nrNOC0bdu2aTBCwDlZNbZqI0rHaRpEfqnE6/O3h4eHD/ePcGuDKPn2D76PGI3SJIijOE2Mlnm+o76nrQHKeIgAbZuiqjfZ4v7OH8QSW+wsFCLEBCOAPTbfrlsl8zxPeUQcury5qY3qDfpa60bJwWRECEnTdP/46O31pUGg1+tVVfX+++/vinK53f3/efqzH1m3ND8PW/M3TzFn5Jx73vvM51R1DT2ymwTZ7CYpyyAMmJItwTJhGfCNLkzAsHVhGPCFAV97ECBYgGBTEklRTVFqdVUXu5tV1VXn1Jn2POScGXPENw/rW4MvosU/YGNnZEZ8sdb7/n7P0+30wzCs8mI2nVqG+fLly1/87C9ky9fLTZrm88lt6LmvX78VQnFef/vVl19+/sVHH3xom8xiOFnMksUCibqJPI8SZFmGHXinVxeGbY1GI9lwICTn3DCMdLXZFv80BIyxwPVsx9JaL5dLCRRlOM9zLVW32yWMbr05SbJhBGkguRKNbJq6RghZttMKWTZNFEW2ZVEElRbr9RIRXNd1w7lp2Nvp2PY7syxLzrlruZpL2fJOFAjBuWgQwZZtm7bFTEMp5dqOkpIi7DCzLWuDsbqqti01izJeVgRhqDVBmBACAABQ9TtdQkhTVQZllNLJZOK6LmOsbRolxKDXxxifnp7u7+8ihObzue/7cRxPF8sPP/x4NBrE6w0EmhBs23bDuW3bges1TRVGfr/flVJqqQnCECGA4FaeuTvaSeK4FWo43vU9dzTol2V5dX3rBf4mSTCm/dFOWeWO74zGO3VdV1XV63Sl4GmcQAjHo53VYrlZr4qiKPO83+11oy5B+OzsDGL4+PHj8e7u4/fen8ymr9+9jaIoyTPbcw3L3oJYDcpUKzab5PziijEWBMHueLS1miGKCMOtFEKo2XSBEVBS5nm+Mx7btokxzPN8tdwsZpO6zPM8H413Wsm//NWvBoPB6enp9inAObcNkxDC68aP/LquedWMd3Y6/Z4b+GVZAqm2qfBkuR71ulpILaRlWRDCpm3zPD89PV0ulyYzAIDf+c53kzRrlaSGmeVJFEU7OztaqcB3yyKzDGoYxtOnT+/du5dlyWw2efPmzbNnz5qmYYxprbcyINs0t3/xpqqFENscy8HBgRBCS6mlYpapNCyKKtvEUKjFbN60PMmyNMu2PIzLy8sqy+uymkwmvu9XVd203HLsqqoJpbZtJ2m6iePNZuP6XhRFs+VCKCmUDLudTqezSZMw7MRxslqs9sZ7WVp4vp83lYJAa21alueHjZCL5ZoiutMbDAYDhFDbiLevXiMAgdJVXg56wzwvs6w4f3vO6ybLsiiKZKsMalKMl8vlFhw/W8wfP36c5/mwP8AQZkmitR4Oh4PBIAxDqHQYhsPhcLFYIAgWsynD9OjoqN8bOI5zsL+HMARAb29jCKGmabXWEKPtdg0RzEW7PfkWVWlZFiGEEKKlsg2TQGQQ2os69+/c9R23zov9/f2mbf0w9H3/L//yL6HW49HO2btTLdXu7u5yubRNS7diNZuHnr9abqhh3N7e1nWNGZ3P56vN+vzyMo3j6XRa8b/qx2oNqWksVysNZH/Q8zxHSgkJ1ggnSbJabZI4k0L7rhevEkIYF5JL5flBVVWj0aiqKgxh23DHcVohXM8TQjDGBp3u4nZqW1an07Fc509+/KPnT5/ZplUX5eOHj/qj4WK5TNPUM+1PPvmkG0YGpbysoFRYg9lsRhA62d0vktgPPS/wDYxNhKssl60Yj8cIoSzLGGOi4U3Fe92B0AAQ3CiR5tmWbnZ5c23aVqvkizeve6PheH/v/Py82+2u12tE8O7+3h/+we+XWdpy3jbctezAsW3bTjax77gMk7qskiQJbHd3Z/zw0X3GWBiGs9kMaplsVsihBmhlkSSM0tHuXiOEakWZJJZhmsywmOE77rDbq7NiOz8JPPv2+jpLMsu0TcdhhskYJRhQgihGABFm2ghgk1mubYW+Zzoms0zP97XWCmqhJRe8aiqD0CxNodLdbhdRDBnudDq8aSyT1XWFiG61oIaNkVlVVds0WIO2KgFsuaoIZVzoNMmbireNqIvKpIxBHE/mFsAAACElIYRgrCo+CjqKtzY1NssNaKVrmj0/qNPMZQYQSraizAuTGbzml5dXtRDz9TrqdjzfN337zflpnKy6nTBZrAxsnOwf/9N/+k+ff/u02wmj0M/jDULINkwKSLZM+lFoMjydTn03GA8GdV6Mx+PZbOa7gWXYz55+HUQeQOTbp8+rokQArtdrwqhGEFDcQriKE9e1N5uV7brd/oALsV6vgeAGxp7nzecLKOX9u/ewZdSi9Sx7dzhc5dnOw7vANn/19JuvvvpqPp18/uWvMMYHe/sEopzXcZWZjBqIVFXFpYi6nSiK5vM5UDqLE4IRl7yVYpXElusI3oau51tOXZSdoFOmWVuVJsG27UqtuG5fn71BCEVRxGyrFHXOayFlXqSMkaqqFouF4q3JWNu2QknbtnpRhzI2GA4f3L1nmmZZlq2S0/msquu6rCzLcmwvS4vler3YbBZJvM6ztK6O7z/44ptvf/XNt2XDMaPnFxevX79GmE5nCy1axzQ2mw1AxHb9r776Ko7XT59+w6VAmGIM66pQomUIO4appUrLvJKtxhoRuE7i1Wb95bNvN3kKAMg2MZIQa4CEopgsN2sj9CTFF7Pbuq5Xy2VRNVlWOLYLAFQQpkUxX60Rpn4nMl2bSyGgNhy3kep2vnj+8nVeFA3ntu/PFsvBYFAUhcEsIVUj9O74uIhLlzqbpCga3h/tSABbqS4vL6GSO53O0PPS2bSON13fVVW5N9pBiL55czoajaCWTZ54jEVe0At6uzu73W53b39kW5RpfP/wnhJykybAYu9ur7r9Xpqmt/OZGwZcilardZ7mTdXtdhfT2Xy98sPwcLDDWu1hxjQus5Ig0kqelVlSZxzKSraVlNgyDc8r2wqbJAwDwVvBW4Oyf2Ph2EYnbdtu21YpZVrGBx9/mBeZZRrD/gABuF14nhwcHu0dbH1qeZ5nSeJQo+eHUArXNpGSBMCd3nhyOd8d7bqet0g2Xr/n2JFjBszwNTBKIbK6xIwCpeu0wQKjFiTL9eX08nJ6Qy3bCUIIcRj6mBKMcZ03RJN+FO7vjYRWlZTENDjnJsBtnLuOo7TeFBkHqpR1was3Z6cY493eMGDW8d4BNY1JvEIGHXrhn/6L/+7i9Gx/f7/jB9km/uZXXxqEjpzQlDCNE9O20zyDGJ+fn54cHk0vr5fTaRonZVlmmw2UanZ9y6saQ8Ir3u8PZStev3rl+36jpaYQKuk5dmc0CIa92XJhuU5WVi/evYuzvOGiE4Sh5/ud6FfffP3s2bNuEFqMRp5rEZMAfLi/W5d5YPunr94xRiDUZ6enyWodBaFSquSNHwZN01iGiSzCMIAWM+I4hpSEvX6Wplu5YtXUVVUlmxhBDaBijG1ZpnVdP7r/IIoi3/cV0JZlBa6XZel2abs9GUGp0jQtqrIVwjDNmjf/hgwTBIFlWU1Ve6bdNhxCKJSsebN15jFmeq5LCDEZ2Yq2uRSdXhdCWNc1NakXhFtHqwQaEbjFx0spGWO2bUOCVSuiKBJtq7V2XXeTJFDprcmIMSbaJs8Sy2R5mh0dHQVBMJ/OCCFKqSAIhsPh/v7+ar22bds0zSRLdsZjx3Gaqn507z7B0DAMy7Yd1wUA2LYta66FlKrVQG5fu2pV27ZpvNFSBJ0IYtqUDVK6LMudnR3btqui3CJYt7+KrQ+IILy/u7fZJJQaWuuqqaSU+/v7EELLMV3f3d3ftW0bKEEIubq6sk3r/Px8vV5DCCvBbT+4c++u63u2be8fHEAIsyzTEFDGttFjixkmZdPbSVEU22jQeH/PsEzf96WUbd3EcRzHcdM0Z9eXy816Op2mcZKm+XS+aNt2Z293tlj4YcilWC9X3W5XQ3B5c/3eB+8fHR1tORjbmP9WSN1UtWmao90x0BponWXZb/7mb25L5K3Ut7fTMAxXi/XNxSVUgBCSFnkj2vHh/vG9+9e3N+eXF0mWvjl9t4o3k8nk/OIiydK6rgFABwdHi/VqPp8XRUEMVlWVYRitENt9jwKaELZtPCigCWPUYKbnKAgG3d5gMDBNUwOQ15UXhVmW/dV3jOPYvud4bm9nyCxTbgG5QAFGDNeueRP1upPlvBFt0Im2v0DP8yCEZVOvk9gwDMe00jRVSnme1+v1Jtc3FOHtu7flkhlW27aLxUJKZTteWqRxFpum6bqeY1qcN+skNizH913Oa0TRZD5DkIRuWFVVVZQP79z1Laeu+HITf/rpp+vVSmsFtYJS+aa9WCwePn5UluVwOHQcZzqfDQaD7cpqOpl0u93x7m6WZWmRbz3Aq8Vy2OszQkXb7gz6lmVtNonneZ7nWZbl+B7AKCtzCbRp25RSAABU2maGFrIuq0GvTyACUm0leRhjCOHewcHZ2dkW/Hd4eOjaTts0BqVASKDUdlFkMQMKFXgeQejl8+eB50dBOL2ZDXZ2OG+TTfrFL38FIBYKaAUEF2VZSa0JZm3blmVpMMs2LKTh7s64qgtqGI7vCCHLrIAQrtcxIQRjwojhe95yuXz1+jXnNed11dSGYbiWvZUHQAC6nY4QwrHsrRXu3eU5QPD+g7u8qY73DmzbVlq/evXKtWyhVcWbpmm4FKbv3s6m28FjGIZctPu7e5TSVsn1ev3w5O5Ot7+1UDDGoFaz6dQxzI4feJ738uXL9Xr96MHDpq41gswyDcNo6hpI9f6T99I0Xa1W3/3+9zr9Xt3yzWbD6+b68iqJ4zAI6qKUUo53hoSQL7744s27dwiBbjeKV2spZS1bx/c8z+t0OpskRgSPx+NPPvnk4cOHWZyg9XJFES7qSgC9zXLxpu10OohACRRBlDEzKTJiUSFEXXPfDzjnx4eHdw4PN5v1aDQsisJ1Pddybq6vG15hAk1KCIaibS3LgYTGWSqBJpRqIQkC/6bEJKW0TQdqoJTSENqOSSlN4rQsGtkKoJVqyzBy8qpM2xqblNpGnjVIE15LyzBshwktIAaQYGaZmzzlGAgE6pYnceyYFkIo5RVxLTcKTMcGADQt11ABpIu2cgLn8vJytVj0h0PH9cIo6vf7yXKdxokG6u3Zu7oqtVTz9eZ6Mu0Nui9fPZ3NJvfv3y2UfPrubTQYlnXjWAZvS8iAETpBFCJIpYAAgP3dIYAijtM0KU1Go8DbRhoCxyYIdXo91/PPzs5810EavX3xiinRFoXFrMV8NRwOF/MpxZhSCgh7d3Fp+04jmvObi7DT2+vv7I92O4O+lHIv7PWIK6p2tV4XdSWBJJRSSvuDkcEsoHRT15skxZhuQ2A7owEEKohCPwqvJrdSg/lkrhppYTq/vt2C8oFhmL675fjXQgqEAYVnZ++urqdZXl9dXSyXc93INm+yLPv2+bNNHM9mMwR1keUaAsIYgcgxLWoaddMQiPZ2xv3R8Fdff3VwcHB7NVkslvPl+uzduazbtmyABJv52mZ2FHY9N9gGT8PQv766yOLN82+faohWm/WLF99iit6dXVxcXLmO7wXufDlrhSqqtmokMylCAGFsmDbEqJEyKXKFIMZYCAEBNpjVNhxIBTWAGNVYr5si7ERt22Z1/vL0tePZz559+/bFK8+0hRBxkizStbbJIlmHvc5qtfJ9/+NPPkmzLM9zwRukNBTKMS2o9Gq5XC2XvuVt5uvVdN7WDZZgM1se7+6vlks/cCVoJ+tJOIwMg7mOlRWxhiKNU5NZRZlzoJa8SYHOWjlZrZ6fn2kDYwJ3RgPHsBgllmkORsPjB/c++8H3nr94oZSimOwMRx88fk9Ldbi3H8fxsNNDUhOEu91uHMd5klZp7pl2leVZmiZ5tr+/f/r6DYZwuLdreK4deWEnzKsCAN3r9Hph32IOFEC2jWubgefItmGYSa7qonJs22QGBMBiBkKo5VvYi5BaSiVMy7BtWwjhuu7X335zfnl5eHhEAC7WaeD5bcOfPn+uEHzvvffevHmz2axs29weOKIoWsXL6XLeGfTbVvzg176PIaYI9zsdqDUAWkpBAOw4oWqVEKJWNXNZWiQHBwdtK6GCTOpRpxuFnSDqNFUja96J/HfvXluBF/SiKAwcizGMOOel4AKBqq6RBqZGhEtQNGWcOo7tdvyXN6d/9Cd/rHhraH082p/cTJOmyZUUUr95e9rZGaaKny2nOw9OLjdzyTAgeD6fAyGhVBKigteaC10LajDDczQEEMJ+2KEIAy0NgoVqLcsyMAGiTTfrzWbV6XdMkw1c79nPf9ELg6oqqqYUgvOiOhjv9gZ9Sul+b4cKvVqtirr41z/9+VfffiOJbhT//Kuvn714tSriuEw3aXKzmC0365vZ9PXpOwF0nmWCtxgiZhko8DxqGlEUpXm2TuKgEwEEsyKHRBsWA4zUoqUG24o7tIJVy+/cu/sXf/Znb1++uHf3ZMs/SfIMEmyYpuu6JqUWoQzibrdb1IUQHFOCKUYIcc4pZnVZ8aKCCCAMLZsVeeZYNsVYtEpJTTGzTYsSTCk2TZNz3tsZIoIhBgAABXRR1UBpoKAQAkK49bY0TYMQaqXYbgUsZiRJAiHc/nhxmiqlCCGMEQAUxMC0jVW8Qhhsj1fr9TrP89evXzuOw6t6sVpBpAeD3r17d7buzbdnb/3Qe//99xeLBSHk4ODg9PS0ruvtTqIoy7qu0zQ1TdP3/Yf377959TrygyzO5pPZg3v3rq8vLcMcDoeTyQQhVJfVdDLxfT+Kupxz2zQ7YYQAmM/nvh9kRb5ar/MipZjc3t4Od0adbngzue50OlmSxJt1mqaE0cF4DCGEALi2Z1lWGIZV1RCMGWO9Xq/KC8VbohEGpFWSGCwr8s1mJWW7mE8JRVHUXSzXfhgRRDEkrmVTDJMstlynvzMOux2pFSDEjyLLtt++PR2P94IgWsfJV998W9c1I0aWFmVZbjab8XgsWyEkQAhoKeuq2t5L4jje8gl2dna63e7hyfEPf/PX66aN0zQtqtlyNV+tzy8uicG6g/7u/p5S6s6dOw8fPpRCRFFU5Pn2msgsc7ZcXV5eSq1dP2havlptgqjDRculFFpDiCHBRd1wKTBlpm0BBC3LQgRvrxSu6wZRWDcNNdg6iTWEGoK8Kouq7PX7TdOUZWlbVuD7lxcXcRxrCDjnZV4oCLYIa9mK169fl1XOGN2iSAghW/wWIUTyNk3TbZinripGaOD5SZIM+4PteZZSMl1M/chbxysuBaUUQJgXRau10qDX66026xevXiKDVoKfX1/9yU9+/LPP//L8+iLNstvJZLqaH50cLteLKIo+fP/9vfHYteyoE9ZNFfW6CCHf90ej0Ww2274WIQRCCGoNNRANNwwjTdNut7u/vx/n2SqJ0zyvRdPpdQkhnU4nSZLA9XjVEITzNFFCAqU3qzVC2DSMbcaJEEJNY7lcIoQQhEopjLFlmI5lzyaTThhWRbkzHEkpb25ujvb2p5OJAvrOg/s7u2MppWzFsD84v7ocH+xTy0yy9IOPPoyiqCzLbr+X5fl6vbYNEwi52WwAVJZtYIgcy26bBkMUJ+tep5um8WoxXy9X++PdZBPPZrPlcimEUAqIRigpr66uPv3s4+ntLQSqrFKMoRAiTdMsy7YRuO0zQUs16Pc9xy2y3Pd9LwxM28QYC95O5zM/DKJBTyP46OFDyzSvJ7eu69q2/fEnn9x79JCYhoZgu7wp66rijVDgdjqveYMpk0CPD/Zr3gKoGKHbS2Sv1zs5Obq+ODcoA1rXda0RNCxzPp9zzrcyuMVsblJ27+7d1WolhLh3795sMs3TDFHihwHB8OMP3x+M+swyv/vZd1zX1QDYgXfv4QPP8z75zifT1YwYbD6fz24nL54+y9NECY7Kul5t1q0UlmOvswRbBmLGerMZDHp+6EHbnCUbAJVBKMS4UaISnEsRuj4QYHJ5y5tmS/lJsgRiFHo+BsjAlCGcZhnEsKpz17PSNM2rijFLSRjYgWVYCGiMdVVnGrRYK6wAxoRSgyLMy0JrWdYVoUYrQVk3WZZJKZUSQeRhCgzLbJpGC+07rmjbfqfb1g1FGLZSNy0GsG452W6HNIgcz7EsCOG25KKUZIR0ow6ByDTNbr+/hQGVZe17YdTrjg/3nzx5cu/BfchImieuaWAIvvf9H27y4ue//JwQRpVaXF+7ttnpd9KmKWtuYSMw7dFoSCl+9fpFVVXX17f9/nBnOFJSJHlSVcXB3jjy/HWc1Jx3I39n1E/TtBYCG6RqeSMVNYybyaTmNSIYYuJ7Ia/rlldFmmzWCVDaYsbx8eHl7U2LFMLk+etXrcusUbTaLE1CtyVyy7IwQn/58592+x0TEdhK2bS+42NKDNdsdUsp7nUiixpt21JqIET8MIBaM0K3+2fPtE3KlIZRfxDnaVpkbSul0LIV8/kUEAoYWydxUmSc89VyI9o22axr3lgW1lJRjAACnu9alrE7HikIHN9bLpfbr5+d8fiv/fXf2z8+AYRWSu0cHZ08eRSX5fnV9Xy+LMv65vJqcnP76MHDg4MD13ct116nyWoda6HX65gw4kch523TcIIpIsRw7Ua1mBoEm03LuWirplZKKaXbVhBCHN+DBBe8hiajvrPOEoTQuNuXZaO09jvR+dt3O/1BN/Dv3797evq2qHLHdxACTCnU8I7vVUUZBMF6vYZAhWGIDCqAxhRTk9W8ghTu7u2F3c54PPY8z3S9w6OTTRLfvXsXIewH0dYlcrJ/4tneKttYgeO6PueqO+wgA8V5liRJNpvBLL1/cryYTaGWhKIatjS0n757fZusfvXqGTLZP/un/2R6efXm9att7qs3HGzi+PDkmIs2iCJNUFoVhmsrpQaDXlnmpm30Rv3r2xvDMF6/eAkAgBi/fvs26kbjvXFRVYZhaQ2owaSWrRKreM0sg2FmMUu1SjXtsNsDSmoAqpZLoKVSrRQAAMNgEAKTGQjApqpd0zIw/ei9DyxKHMNM42R3d7eqqsFgMFstk6qI02S1WoWu4wdu0TbXq3ln1Du7On/x4lmv003XGwyRFwaT5Uy0XLbCsMzhaKxahQBACHHJFZCj/mByfeMxJ7R9KNT1+dkPvvc9NwywYWzSxKSk5/uOYVCGZ7PZ8c4Qy7ZpK8CQRjoIgpO9g8ByIABeGGzqotR8kWy01kjpap1gLgedbloWJHBaigQlrutTRKu8+OT9D4+G45EXDa3w6edfT28nW19jv9+/9+jhYLwDIaSmoSyqLNq03KTGsxcvBdBpWSggEUJ5njNGzs/Pe70ew6Trh8f7B03Ln75+eZttUOBcr+bjvV3dtPF8+fbdu0aKZ98+z5P8/Pa6v7vz+L1HjJC9wXAzX756++Y3f+e3GQD3Do8hRte3N+dv3g27nc8++8wwjNFoUNclxtAyWVtXBoSIORZlTGPsuq6oK88yDcOglEkpZ8sFRMj2vU2SEIOsl0vbNEzTWC7ny/UKYqQgXK03dcOBhn4QCiG4FADCOI5Ny7J8p1F8Z2e4nMwcxwn86K9m6JzneQ4hppS2bdPvd9M4KYtCKVVUpQbS9d2/cvJWtRSac267rm06jNCirjSCEAMINdQqWW8ghNPFXEHALNOgzKAMI7r9j5qmIRput5dNy4VWXAqtQVXVL58/l0AvN+tGtgooADVvKtsxJ/MZIuji+vzNmzfTm1uo0dnZGaU0zTMp9WAwCP1gPBxhACmlGgIu2qjb2T5YlVLLzYqatGnbqNvDmF5fX2KKjo6Pi7r65quvgdKu7zct39kZTiY3nTC6e3K8SVIvCl3XT9McIbSzO26lTLJsPB5XVUUR7ne6UsqirBGAi8UiKbIgihBCvahzenpKDWYYtKnqnd4ICiUE/9Gf/snNzY1JWVHklmFYhrHZbPKy1Ajatt00TV0WgvNknTiO17Rt27bb6erx4SGCuq4KJaRj2VtTeRRF25Cy51hlWSZZmhVF3lSGbfxV1F3KOM0gIq1QAKCq4QihJMvvHp+IRoRhSAhhGLmW6dlOEsdhGP7Wb/zmb//ub/+dv/MHv/M7v/X48cNPPvnok08+mU6n7169mswWWZGfXZwv10tEUd1WlFLDMAimQEMAgJDtttMQbzZtKypeW64jhNxsYghwXtZlWfC6QQBuA+lQ6W29Oc0zxtj+4WHNmyRJpFbz5QIg+ODRk9VqpTW8vrjmrWy4mC1XaVFSYliGXeWFUgpCoMBWMAtbISWEgJA0z3u93s5wFATBp5999tmnnw76/fVqxaVYb5L5am25nuHaCkHe1p1Op2040HKz2SRxluf59fVlmq23MR4CoO96dVkhRLbXHQnk8zcvLN9eJfH+8XGa5w8ePHAcy7KM5XL56sXL2Wy2TFY381svCJRSVc0pM7f8nJOTE0qp5wWnp+cYoi+/+NWWKT+fz5frldb69evX9+/etQzTNi2CcNNUfuB2Qt+gGADQVJxiQqkRx2tKqdCy/B99sAghZhppnhmGgSGEWvd6vS2kMF4vp7cz3/fzsri6uhr1B3VdLeMNZqjbjd69eXMznTHD8twAQmw6rhv4QojxeFzXNZCKGnRrdtvf3QMA1LxBCO3t7VmWtc2eFkXhWHZdVVJKz3UH3d6f//mfE0IMywQAFHmOCdryWMqyXK7mgnPGWJZlXApCSJZlshVIo6Is3dAzXVsADTECABZFUdfVi9ev6qa9uLw2bMcwqGmyPE2KPH335tVysUjiOF6ubi+vZrczXrejwU5d87OLy/Vy89mnn945OfGjsJYKQkgx3v7AYRhijE2TeZ7HuegPB1GvizFGEO4MR6rmge9atjGdz+7euxcEwduXr5qipAwjArv9znw19zzndnrz5s2b6e2tFmq5WHz4wcd5ns+XizxPt4qF5Wq+nC/+9E9+1O905/NllmUVb64uznue13Vt1EreKgkRg4rYCqpNNghDQkjLtcEcChHWupUiK/JOGBCg0/Um6vZqJRd5yiFsNSCIKqHTOEOErdMkrfKj+3cEAYXk2DAllw6zVSNk2zCCKUON4sigtu3keQ0BzYuGMMOy7VYJ27MaLXNel3Vl27bvuK5tEkYRwnVZY0AwIIyaUCvbYlC1GCrLIK7vEtvAjBZlLaVuG+HbPqWUQOTatuc4UgFmmi3UVSsUNFyvS5hJGHV6XbsbKqxNk7mOaRgkb6pZvG6a5oNHT4AACJDx3j5XejlfBU5wMNoL/egXf/n50fFx1Ou2Ug6HfaFkUVfMdlqtNYSdbjfqhi/fvlrGK0zR7sHuOksubm6Pjo76/W6crAeDXt5Uvu+HrnN5dqoAjPMyDCOkYb/TU1Jfz6bMsefLJSWsrrmUmjFTKI0po4a13VjWVbWczX/nO9+TWVa3HCBtIjTqdWfrxTJZUkqlEoZFNJaIAcywZVAtJMGmYViUMsbYoL+TbBJGiNAiK3JqsLOLi+3IWwpe5xkvK5MxAEC33zMsRk1j0O9qKfI0cSyzLMsyL5qmmW82gDGuADHcqm0hJk0rRqPRZ599t21bg9BuGIiWr1dL17Ec01pMbnXL0/WSIo2ViOfTo73d7376yXc//vQ3fud3OFTTZJ1UBTKw1OLgYG9vb68oSiHEts2wni992/7Bd7/71377tz98/73DvYNer2/ZrtKQYMO1HAyRQYnLTIcaWGolWyDVaDTqh70yK+eTuWm7yDJK2XZ6g263//nTb4LhECHy9u1p1O0R01ISYMIarVsIDWr2u13bdSRQCMG6rhgzIWJFI45O7n785MOPn7wfRn7dlJHnAs53Bn3f90eHh+ez6fViUYt2lS7Dfuj59r07x6BRXS/yHItiFLgelAopAYHmQDVaN60AGjFCh8OhaLhnO4Hr9Tu90As3y1hqhQi9e3KvrTkhtNPtuZ2O2wmzIo26HWbYeVHbhvn82bcXF1et0mVZIwl7fmgSSgz2xVdf39zcMkLPLs6btr65uiBKmhB1PS9wPYPQNE4IwqKpKUFaS0QgB4ArSRglhBCIZKvahiOEbNetmlq2Eijg+v6dB/cfPrr/l5//UmplOe5oZ5cQhiG0DNrwAgC1tzPa2xvfLBfUsvaHe9evziM3Oj6+M5/PiyIbDkfXk9soiggh49GgbSqgdNu2QEMpVJJllJmG5ZQ1p5SGYQggjpOMUbPf7SEALEp120ZRuCmy3nikWkUgwqYDmeEx1wJUNtwPvazMJAJl0WRJweu6qsokyyE1FsmG2OY8XnMpbm5nuzv7n773YT8IZFP2BxGmqDPsQoqCfteOAojI4f4JBjSNM8Glb3m+7VTrzfL2Vmj1+IPHw36/GwV3Dg93d3YowpzzYbc3m97mdbN7fGc2XzPTLavm1dOXHiaHYU8kec/3L0/Pnn7z7d7OyGDYwHCzXrZQfvjZR3v746bIl9Np0zQa4laAZJ1MJrNMNJPNcr1cdbxgvLu7jjdACp6XWVooQJ48/vDuyd1ivcBNhSopEWPJJvYsGytYphmFyCC0yAqLmfF6QzAmhDDGpGjrqnJsd7VcQ4IBhGmaMmJML68nV9cQMWyYluVoAC9vbtI8M5lhYppvstALoAaU0uV6yRijjGkAWiEYNaQGUmrDMIDW27O2ghpgpCRI46ysqizLBJeEMIZYmVdaqiJLWsG3m3qE0CZJCEFJksyWC68TKoSxSSrBRaswo5zzqmmUFpRSqIHnOKZpbpKkKRvbdEzTvLy8nM/nCKGtU9x1nTRN0zyHBN+/f3+b/Qo8rxNGb169Wq/Xk5ub/f390/OLF69epnm22WzKvKAIG5Qt56so7FLTuLi+2t3dtW17s16blCGMR+MdPwym85nWsuEVQfj6+tp17W63u92wY0LSIt9Sv13LVq1wHAcAkMbp/uFRUhRSg8Pjk6IoCSGdMJxO5kiDxWyipRKiRQhdnJ4t5vPL81PTNAHQRVEADLjkhJCmLHzXIQgXWVkUlR92qrKpytKznU4YAKmioNO2crgzqpumKApe1Y5lubYVeL5hGC9fvnRdF0KY57lqRScMt/0Dz/M454yxuuJlUQOAAEAYUyX0p59+ihDhnAdB0OlGtusooBHG/6MRCZZ5XhTZ44ePgNKL2XyzXDFIH9558Nd+5/cs212u49dvzyCmAJHBYMAI3bYK37159xd/9mebzebt27c//vGPHcve3Rk/uv/g4OAAUcIsEyDCTLOuay44pSQIgmFvyBD2LSeNE9u0tQa27VQVNww7CMKbyZRZBiJ4Np0P+sOL80vLsJqGG9SQUm39vVVVKQi2L9axHIoJo/Th/XuDbs81jVGvRzFmCKXL1SAItZBHRwcnJ0ey5bu7u9fX1xihIkuyLJGtsA1b8HYwGPWHA4TQ1l8Wp4kAOi6LphUAQM8Nbm5uBG/vntyBED958mQ4HEZRtDMaM0K3KpX9g4N1lkxXM4CR1vri4oJLMRwOiywfdHuc80cPn2Rpnuf5fD7HGDZcJHFq21aWZXfuHA8GA601AEAr8fb163QTQ6V930+SZOtU2NYFPM9Li7xtOIYAYzzo903TrHhdNyVCyDRNjKjtewdHh1JKixkaQs65VmIyuaEGC6KIQNKWXDRC8NbthOs8xUDvjYar6bytW0qN6XT++L3HVVMLJTElCqDVJvECXwiBMGx4TSmRWgkhgiAo66ZVGiB4eHykgNZa12XVVLVrO5TSoqoms9lqs0GEVVVVFBUASEpNKb28OseMNkISgwEAgNKdMIrCbl4UhJndbldKSTF99PAhI4ZFycnurs0MKNXe3t5gNJJaZVm2f3jw4PGj3qB/uH9gEMOzPctyjo/uZEleFEWZp6Hnm4y1jeiFHc+0m6ZxHOfi8lIDZFnW559/DiEmhNw9OvYsU1TNaro42N1fL1eu6453RwAAwdu6rLYY2vlysVotjo+P401iW946TvrDnentpMoziJHfjXw/lFIDTFzPS9M0y5P+oPvhR+9TDCnU5y+fMlUiQC3KbAqVko3phbfLNUGYaNBWNZDKtiwIYVFUaZozy5RAC6UZM0HdqqIObbfexH/zN3/rP/z3/v1VkmZNKxrhUCuP8zLJdVEyLiLLy+LECzxIkBeGotUEQEpwmqeAYtNygYZVUTZVaRsmVFrUlUkohpQyiysddHsGs3jFFQTMpL3INQjSUgsJMDUNyzEsWwg16g+BBDVvAEO5bgQFGsM0KwDCvK37/a5satSoNq17nVCJen+8q2puKIhatb93ZFiuVHA4HHeDMPT8/nD0o3/1k2W2thzDJqRYrTerleM4H3300dvT06rI9nfHzDQppQZlFqaR49mQ2MyAAGRxxivuOT7WqErLKOi4tleUNaaG64eSS8/xKaIGMVql66bBEOzujiHDOa/qtvFcB7ai5we+aSnOPc/NiozaxiJezZdzwZu2agRvMUIKAkmQQGAxm/W63Z3jg9PLC6pwaNiGYUgIENmONCyCKC8KA0KtNSYMY1rXNSOQQH17eRVYjqnRIOgQgvrjkda6KAqlVJZlnHPfdTnnbdtuCQ2EkDRN87y0LXe1WkkpeVEZmBAAZcOh0m3dHB7sjYej7T8hhGBKESGEMS5FVhYAINfxRsOdOE7PLy99P3j48HGVlXmStiX3DOf3f/f3/8H/7H/58M4H19frL3717Nm3LwxMCEQUIMNgNee/+OIXr96+Wq4XP/7xn6wm02ITu451995Jq1sBlGE5QacPLKZt5rgBVPjXPvqsyUqGaVEUeZrBRnStyNJGmuamaZSbFWobJQXQ2rO9Jq9H/ZEWsuN6gHNmUkRRlmW2YRnE9GxvPOjfO9z7je98sjfoLKfT9WpxtL9T5RuGpGgr17ZsxiKLPTo+2OtFPdclrZqc3tQlrxp+cHQSdrrXt1PfD7WGCFNkmtFokPGKGHQ1X2MAhVItl91u/82bd4PBQCm1Xq93d8bPv/2WURyEnhN6FRDRsEcsqoCEGjmW4wXu1e2lArrfHz5++KTIMsNhmzyWUC5Wy+VytbOz883zp8xmaZysFsudvd1VEp9fX4Vh+ODe/TxJszQdj8e8bce7u3Gctq1sqjqwXYsSixpN08wXi1aKbUQYU1q1XBH0e3/jr3d6vXev3+yNd7WUm9WyF/r9Qed0cq0o2xnsT67mvuUPRjs309tGN1HfV7pJ4tWwPxgMx9iwtNaGQbgQRyd3nr9953a6AgDDshBCYehrLbUUQvKaN8y2q7Zt2zaO48VmneaZlJJigjHesgUty6raJsnSwPVcZnIpJAQKQY2JwpjZFqIEEcwIrYqqyFNmEIohr2qZlZHp9PygrYpu6GZpYkAcueFmnZ2fX3/4/iee4zZVPZtNHN8iFm6aar1eU8N0glBTenjnrmNY7168MqiRxUWR5U1d37t7t1VyVVQSY99xH9+5vzMc7Q7Hg05H8HYw3o8Gu2leu75n2KzmNdAKKpBl5bA/5lX9/Otvp/PZ7XzBFVxsUtP1nz5/9v57j0f93mI2LfOiKAretHGSG47fyNbxXQzF9fnbt6+eqiYbday7u10EidW0InDcPI4N28HMyrKs1+nImouqbZqmqTkljDGWpmmrZFnzLCu2KRHHcSTQ/+gf/e//V/+bf/g///f+F4vNBmIax+lgMPK9UHAJtWaYCN7+Fd27EVrrLWXbjyINQM05pYZJGQIAAMCYYRhWVZQQQgQJBripOIVItlxDoKDeJEnbSoSxZVlC6oZLxljTNGVedDodTCkXDTUY55waRqcTIoMww5rPl4ZhmczwPO/66gIjsFwugzBsG357fd3U7Xy+xIRt1sl0OrUMEwBwcvfeu9evAFSNaAmjjuN8+OGH//yP/ujRo0e+79/e3rquq5RyLGvr9lQQdDqds7Mzz/PW6zUAgDHm+z4C4OuvvgJK94eDJEmiKLINcwuMDoJASnl9eWVQuow3lmMTQmaTqWwFxSTdxNv9/pZpt50/bo/zjmU7jpVk6Xh313EckzDDMJabNZeCIMxLnpflNuls2zaEkBBSlXUrhWlbCkGIkBCibXgcx47nSa23FQHbc7fGK9u2kyRxTGc0GPGm3aKEhBCMmRhTw7BMw06SJAwjy7IhxFprDNE2OwwAwBhvB8cAgN3d3efPn282m8ePHy+Xy52dnUGvb5nm9fX1aDR69eqlUmq9mLu25TmG5CWUvM6zy9N3f/gHf/CD7/3wcP8oz/OmafYO9u3AowZTQMdJcnt7G282i/n87evXVZav54teED15+CgKfM92hBB+FI33di3X+eCDD7Iso5RWvNUaDnv9qqgXkykjdH+8m+f5aDgcDvtNWy/XS4zhNsispGyretDpbsMngev5rtfr9H/rN357b2/Ptc3Li7PI9yCBddNcXV0RQnzf39vba0RTVUUU+t1OCLQc9Pqe4zuOVxZ1kZfT+dxyvDt37kihEWFSw214AwBAKfV9p+Wc182wP1wv1lmcZWkOIRZcYoQ8yzYQWS+W19eXmJLTi1MMdJbEdZn7ruMYtC5z2zaTPHnx4kVRFFrITie8mU42SQY02myy3/u93+v3+3Vd379///Xr157nHR4fWY7zi1/8You8v7i42GacthCFKAiAUoSQmjeEUcezIYRIg63tiDHW7XYBADdXV77r9TodhFDNOQQKKrmON3VdCy6rtIw3qWlYe/tjRnBVFWWZb8GrCqBnz54JIYaDQbxer1arLX0IIeTa9sHBwXYV53kegUhL1TTN9uOQ5/njhw+73S7nvKwrSmng+Qaltm37Yeh4rmj4FkAPAGiadrt0aZpGylYpaVKTEbJtnLRNU+XFznAn8iLXNk1Gvv3228Vs3gsiwdvb21sAwHqxpAg3ZfXe44dv3ryRUv7ge9+PokAp1bTt+ODQspzDw0OTGSfH93d39799+pwxdrh/UFWNE4Y7+wd1WbVVLaXM8vzbZ08RwUG3PxiPIcSMGNPbW8+0BRedTq/fG95eXc9uJ2EY9gaj29nUCwMIITPZ/uGhUPKbp9/ev39/y8PBGCulok4w3jsoysrx/OvbW991bIZ7rmmpCllO1HKFIXJtp6wqy3WePn9GKZVSlmVOiVGWNaVGXTVaKQwAxrgz6FdAllpMNpvdO0c/ffH13//3/91KcC1UzRuNUV7VXCpsGZVss7qEEJqmhTR0LdN1rFYDzBjCFELYNE3bNhRgDEiaZq2SGDEEqWFQ06DbvAdD0LVNBSQxDIUNaruc8zxLBG8JwpJr13bLqkBIM4NAAELLxxoo0VS8qHlTCQ4B0QCVqi2awretQa+PLWOZpoWojk5OGKXDwc5qtVFAe0GQpumw06/i7P7RCVagkJxFHsCg3+9GneD65vLg4GAwGAClfNerOZcQVEA63VAAHUVRL4zqvNjb23t7erpVfu+Pdx3DrIoyzbMvv/l6ON5p23ZbTRZC7I5HdVnYJnv9+uVo0PNd+3Z6Qw3iD6JaNggBKAUUKt3ElmG+efOm0+ls+a7bUvW7d+/aqmaEZrxeFSk1WNu2hBCEkMNMx2BZlhBGhSa1REmdR4PO3sEuY8wwnV5/hG1DEg0MpAngXAAAxuOxbVndqLMdxRCMGaH9ft9zAyXBZpMQzOq6Nk0rjtO2bTGlCgCAoGFZaZ4RQgaDged5ZV7IVlxdXA77/f3d3ZurK9dyDUTyeHN7eRm4LkXwYH93cnvl2Eav545G4c7AC1ycxLeH+4PpzTsgi7/3B3/r7//9v9/fGV8upjVUgBHb95hp1RWvs6JOMt937z28VxWFbvjJzu693f294WCnP9jt7fY7w/6wp4mGBr1ZzGzXKoqiaSQhpGmKKHAwRFVamZY1Wy2+evUUOWydbrjmyECEEQyRbFWvN+j3RncPTz5+7yNGMaPY9/2iagbDvcUmW8UbrqUXdRoBz2/nhud1+r2rm+vBcA8ChiA5PDkO+4PuaAdTlqb5neO7J4fHruv98Ie/jjREADiWPer2IzdIVvEW3yaqdn4zMwlzbWc2nT5/+gwIydOCanhzdtHvdTzXFnV1+eYNbIVqGtFWoi3ePX8amCwM3KoqmqZCAAIhRd0wTCg1bNs/PDgWjbq9niBCu91uYLtlXlycnb87O22hLpp6Ops5jmPbbl3zLfdttVi6tpMVObVMAXQj2m1EEgplUZas1icHh01V/3f/7b+EGMV51khh2lZR1ru7+5EX+I7rO8ann368zjYSaBPTzWwlufzgw0/X8Wa1Wfc64ag/WK/Xezt7umkBbz2DDcIgsGyLGkCoi7NL23I5FwRRIKRv20oIZpoIoen1rWxapcGT994nhN7eTrTWdV27tr27u4sIg5gqCWzb7gRhnVWoVZ5hSSE8x91sNgY1eVUjoF3PZhbjrYzz9PXr12kaKwkgxr3BCACQpnEUBUoJgzGM0NXlTVU1ruu/fv06TzMtOW9KoLRje0mS1U37+Rdfmo579/FDatmT6wmSuixLgFE3CEe9/t7eXiU4C3xBycXN9ctXL5JkYzMDFLxjh5HXndzOOmF49+5dPwpv5tOsrAaj4d64H3Xsui5bKd6cX3342ferWiwWq37UEW0dhU4cL8/fnULELm6X4/27tu3tDUY2BIbkyLYdJUHb6qqqt4QZ03X+8osvDk+OmWlkRR5EoWnapmnzuuFNI3jdto0T+i3UlmU9ePJeocU/+j//x0G344X+tjEopKx4AwCkhtWIFlOSJFnTtFsRgW1aCOCmKAnAgWNTQloNqOUwy2TU1BIwzOqaayUwAo7BmrpseSOlRIRxpZtWbPOqrRRbD32RZrs74yxNizS1LGsxmxCINJAKgJI3GoBWybYVRV3ZgScABAif3LmXlQWEUEj+9tVLwetuN7q9vW6Kktd1nmaB6799/Y5zDgB48+aN67o/+tGPZrPZd3/t1+I43rKTsixTQmAIXdddrVadTmez2VxcXHS7XYgRIni8t+d43s3F5aDXdxxnOp2GYSikTPKsruu9vb3A95u8PNzdAwD4vr/tACspOeeE0qwsgigcjUbXl5e6FaZp9vt9xv6qlgkhEEJpCY6PTyilAEE3COuGQ4xc2zEpIxjH68SyHEwNgDCljHPBCGGUBkHEOe/2e3meAwTTMquaOo7j1WI9n0wppZZl5WkmeJMksWmak8mkyDLLsjzPy/OcMWOLBjQMY2u0aZUEAACMmMUQQjc3N23buK6bF2myic/enTJCq7y4OD/3wiAriiRJLdMt0iLyIyXly5cvv/jilz/6yY/+y//qH//sZz//b/7on/5n/5//9Jc/+9l/+9/882fPnu0fHLz3/oe249Rty0xjSxM0TbuV6mY6+elPf9rpdCzLujy/AEpz3kRRcLg7HkXdOs9Pz952+t2Dg/2mqsbjUV2XcZrs3TmcrRez6ZQybBhWVlTEMq6m14DCqBcyxjClhFFMSVXWvV6PEGYYxtHR0WazOtzbRwgnae4H0eHBiecGUadfc2nYXppXk8ns+PhOvE7yvGTMuHNyUlSlUPrTTz+FGkGhNFeT65vry6udnZ1Bb9CNoqIoEEJ7Rwf7J0d3Hty3DBtBiAhppVgs13Vdf/HFF5c31whjL/Bd123KanY7Odo/6Lg+g7ioyhevXlZVs1qtfvSnPy55M5lNv/z6V59//nnTtPcfPoaUGBYTQpi2NdoZY4xns1nbtnmaEkK8MPACP62KoNdppZjP5wjAIsu3GXMppWFbhmEwxiCEURTVdb0tMezs7OzujD//y1/s7+/nVRln6fvvvz+fL8qytgy7qarn33yzPx7P5rd5kXHe7A7HWZxxrtpWCqHahhOCKSVlWUqtiyzvhcH+7s7Vxdnk5sZkrKmqTqcDMHZcTwgJIZStwBAGQZDnuZYSQ3jv4YM//tGfYIwjP/B9f/t8n02mGgKpFYTQtt2qqiilBOMyTWzbghi5rl+WJQGQYkQpXq1WZVlTzKqqePPmTVbkQup/9a///JsXz/qDLuf127dvX7x82esO7t99sLcz5kWV57lSyjTZzeVFstkURdHp9n/wG7/ped7Tp08RZZiQly9fJkn2/uMnFCLG2Pn5+Wwxz4pCQXR0565lmQjow6P9pmlG/V1ecaHByb27hJDZbPL4vScN55ZlmaZZ8oYrGfa6QbfbKnV7e2s5tu/7VVWNd3aGvX4eJ3/99/6aZbI4jt9778n3f+17f/5nPzGwjlwTBaYV+tFinQHEHIsGoR31B6IVP/vlz2vJw06wjcTxuqEQhLaNoCrKtGp5xdt4tnKo8ebifFMV/+H/9n/NCDQRaKtSYYgooYg2eSWkhJRsL3qnF+daa8jFwImYxKKoFW8BAJpSxSgzLAxh5IYYEim1kBoRUPOyaRuplMmMNI4RUK5tCS4NZoWdQKiWIGARxos69INOFAlemQbRQGgIpQa1aIu6wgC4ts0Yy+uGuO6mKH71+a8MRFTT2gZ78PD+Jl52Au/u4aFDaSfwF4uFYVujnb2bmxmV4HA4Xi+WhyfHR3dOfvJn/+rt2alpmpRSirFvOTwrs0XcdSPJ223jiTBTAFi1YrZYTiYzrSBUcDzei7r9kzv3CGFpkkdR1zKd26tb3YoyzbTWju/VLd8KOizTxBgrrTv93tXtjcmMfrcnON9iEizLGowGhmFsZyDbxTKltOa8Epy3Qra84wcnR8c//OFvBEFUVZXSbZ2n7588+Ojuex0vyrLMDZyKl3mSQ0Vub26WyzkvKoeZW40cQdiyDQCUazt3797d29vbTnioYTDT2HZkbINtLeRSK0JpURa2bW+pI0EQHBwcfPjhh3/3D/9O6AdKSMswB90egkQCLDTWgDW1Xsxyit1f/PTLH//xn5+eTb/89m2S15hgIaHrBRjSV6/e/OKXX/zLP/oXf/6jP+V5zYumLZtkFQMAKi20QZM0ny9Wv/zyi6AbUYP5ndBwzc4gGvSCYr2okqTYJNlmLXjtmkYar0yXhqPOIk+M0MMGcl1nvV4nSWIxoxtGrmtPJjd5kSIMJAQSAttz4yRL8gQZ+N35O2qw6e0MSbicL9qGF1k9u14gTX0vnE7ntuUny3x5s1ov1t0wggKsZ6u90U7oOrppv//pd96+fHX65i1WCAOcxDFCqOWi2+1R0wg60dvLcwG1ApoQklWlQlBCMFsvdw53L2fX02y1rtI4zxrR1kUJpXr76s3t7eTydlI2erpKXry9dKPuKk4Mxx7sjHLeAmYssjjoBu9/8LCs4kW8RAbBGK5Xq7AT1aJNkoQinGxS3wvPr29qIU3bqqrKJNSzbMuyuGgZJoLzyPMJRJxzwmgw6FLLvHP3rgKac/7xp5+kWSakLPNi3B9mcYEQpgAZFK8388dP7hMCbq4uZCsP947fnl4gYuwMd67OL5hBvNCL84yLdv/wYLVZC60ww48fP6zrep0mjRCNVMtkI5Xakk6EUut4s7e7f7h/iCF2HOfx48fbDx2jBoHYtR1CCGEUILitbhHTqmWb53kURZtkfTO/xYw2TWMywzFMpZRt2+enZ+vZAijd63Vevz3NqlozYnlu0zRZniqoHr33RCjZtm0/7M1uZ5ZhPHh4r86z2fR2Or39/PPPESJJnO2Mh8NBL3C9wWBkBcHR3TvPvvny3vHBdDHfZGm8XpmMnJ2eb1brtkhH/SjLsrJq1nFyO1uOdseIEUhgkiR/+pOfjEbjwHIIoq/Pbm7WeaH18YN7+7sjx2bz2U1Z5lwI3w6G0ZBIkq43BKjvvv/w1Tef//d//N9G3SgMHNGWCHDx2aeffva97y+zIq+KsswBhdHhHjBokiRNVVNMKEYUQwAAxtC2qG2yrWI49Ny/+Iu/ePjk8Xyz+o//D//o+599yOvatC2tJWOsKSoGqee66/XasWzBWwhhWVWQ4E0SE0Jc18UYb0O4UAOogRKyzFMC0RbSu0kTTKBBmWkYiGDfd5VstWwdxxFCzJaL7qAPCTYdmxBSF2VVFYZhIARs24YQF0UBANBaYwKVFrZhylZUeTXoj3qdaHc4MiiTrbq6urBtM0k3y+W8rKu2bcs082zHDaNOr++Y1snh0fb0+uWXX47HY9u2V6vVYrGoOeecY4Q6QcjLqsjzqmwwNV6+fYMpSfNs7+BgZ3e8WCwUBG/evY3jGEKYJWmZ54PB4OrqKs1zgKDtujdX157tFGW5XC6zddwJwrIslVJJmlLGeN3YppVk2WK12gJzbq9vtgwvLtqSN4TRtuHf/7VfY5bphQHUYDwcEMKiTqc/HHEhOkH4wZP3Pnv/w37UqYpaa101JRcNhPDl8+e8EVrrTbxKs5hS6jjO9t1fl5VUrVbi4f0HAIDLy8ttC3c7KW6axnNsgqGUUrStYdCmrIq02BmPIAJN0zx9+vR/+NGfrNfrrZgmK4ujk+OwF7mh198Z3E4nV7c3cZyeX14bltMfjSybmbZjeg41mJBSKhUEEWMs6vU920mW664fVkXdCqURKutGY8SVnK2WQql/+l//13/x03/9xZdf1ry5vb2+PDsVvARKrxdL06DdTnh4sOe59qeffhwEHiZkHW9Wm43UQvBWcrmcr6qiVkIb1FRCioZvy415mlgmhUoDJQEAs9lMK9XUtWHStm0RgE3F0zhmhEityrpyLXdvvHt8fFjX5XI2F7zlVVkX+bDfU0p8+OGHhmFwzq+uroIgiKJoewG6c+eOQRkj1DXdR48edHu9/cP9sNsZ7Y0M0zy/ugx73buPHpw8vP/zX/5ivY6jICzS4s3puxevXl5e3bw9P5cK7e0fEsuaLRdxHF9d3xJCuRB+FD5+/BhqtTvekVLGcdxU/Pb29ue//EXZ1GVdzVfLpmnatu13ugCAmvNOr0sNNpvNLMuCGmwpm0mStJxvH6M3k4nl2L/7N/76crnsdDoAgIOjw+VyCZTyXLesq81m8+nHn8hWXE9uN8nadd0oirYtFmZYAIBt8HG5XvcGXYRQkiQ/+MEPyrJcrlYQkU2amLbtOE6e5wgD3/dt2y7rSiiFCHYsBwBQN43luW9evlpMZ5ZlZWWxdSlvBctN29q2TRjdBuQRJErrxXrh+/6w19/uYDQEFW/Oz09bwXu9ASEMIr17sG855pZ0NF3M58vl6buzbR+ibZuf//znk9trBLVhGLPZDGPcCcKd0WB3b2exWk5m081m0zTVwcGBgkBCcHC41/G9b7/4otvt9kfD1Wa52WyQ1svpjGISx3ErRFLkXjcqeDVdzbjkk8kkCALPC3zX2xkNTMo+++yzmjeu656dvptcXgaOraWybZsQ8u7du5/9659FQbjt3J6+fYMhEE2t2xpK3vMd1PP9fJPczubINBBjBa8hxYih/f2DIi9Bq4mGFEOMkQC6li0zDCUkFgpxyRi5urn88quvoJT/l//T//E/+Hf+gVCqqCqTUAdhk1AIAJUotN22KkLXQQophWJRF0RyqvO2BsSwTAdLSWXbFhXFhFJsmNikrCxLxw240EBqoHRRlwqryHOQlq1oCKO97qCsmhbqWrdN22xZ7ULwuq553SChbGzYmKlWKKUQQrwsHESpBNPTCxMRoOXh/hHnvBNGe3tjM3QWZbrIssVy3et0rs4vbqcTodXO/t4vv/rcsizR8N/4wQ/nk+knH35kGgYXAlNCLKMRLUSobduj/aPQ92/ns95okBVFv9/Py2Kz2Yz2dx3PrWS7SjYUk9D1Itdvqrrb70GK/V5fY8KYuV5u+t2B4NKkJtFY1G1VNVrD+XRBERn1h93BoGwaIeT0ZlLXda/X22Rpq1XY7UilFtMZASjwo7Is+1Gn4wVSqbPry3dn7/r9vmj13Tv302yTZ3EWJwiRIIi2pV9iENv1zs+vt1vBsizTNEUIxPGaMnx+fkoIogyPx6P9g13XdReLpZQCY2RaBobIpoaJSNcPbWpgiAxGsiQt80IptYWtj/bGUb8LKaamMVnevjp7Fu24VkAX8fTi5t0vv/klQLrVslHCcGyupGGZpuuYntMd9faP9geDQV3XnU6v3x9SzDpRxEwjq8tutwshaqUkltk0XAhZNfzi4vLbX331xS9+9S//+I9/8fWXb0/fFXX19OnTi7PzX37xOWPsmy+/evXsOdKgKsqybZhjlVU+ubm1iGkgo0orWUubOVpDZhDXMULPRpLvjwauaXuu63leGHrdbui6Thh5jmt0e4HWrdZtrxd2e8HxyX6SrtJsE0ZBtxcK2XTDIIoCrlvDNW8Wk4+++8ne4R6EertUlFLmeX55ehna/mcPP/CJ0et2g9ALA+9gb6yFtGxDSBn2uy/evD09v7AsByNaVc18uSbUKnlrWYYQ7fX15bNnzz7//POtQi/ZxN2w67vB/GYmuCzjos5qi9ie6d1e37z36PHWKjwY767ihJmG4ziy4QRASllSFMBkkBJeVkgooPRWnrzd/28bOt/97ncvLi5evHiRpel6uTIMw7Kstm0hQo/ff3xxc7XJ08F4N86LspFuEF5c3Sw2qycfvYeIfv3utQDq/vtPTs8vyoYTDLUS6/Xadv285lF/uN6khBBGqOuYoWs3dWEaDGNMGLNtp8xy27JWWVI29e5oByo9n8+blgOMCCG242Z5AbXe0vYV0K5hU0wkAnbot3Wlm8YgWCrVAFXJNl6vJ9c3/cEAW8Z8vTq/PI96EbUws0zTsV0v6g/G9+89juP49evXm3gRdrz+sFO35Wq1ms/XUdBr8nIQdSvecN1WVQEQvJjcXM+n+/u7qhUnewe6FVpI1/M2aUIp7oUBQ7BtWzeMioYjg96uZgKL5Wq6mE46nc5gMBp2h6JtX716cX1zxnl6sj9MJpNyuRKN2KziyI8e3rsPtW54vbO/2wJVV5xQswUGMd33Hz3suo5vGVgqNJ9PV5tlp9/zgyDLin6nv93zKK0Pj0+ysmAGaZq6EU0QBHlRbZJYamUSE0glVNsbdp8+fypl+1/95//56ZvXhmsT0wEANE0DoWYYmabZDSPOeSN427aO42CMEUUQo6ppKGOmabe8RgBu3z1c8iRP6rp0bacoCttyPcdtygoRmFel1lIpJbVUQPK69hzXdJ0Wao2gkFxrjSExqKl46xp2HidQg8ALRsNxXXGNoG2bWzG841iSt6enp74fdqLo+dMXCODxeKwA4K2wbdu2Lcf3+uPRV8++tWy7qKuKN4PBwDCMV2/fOK57sLenhEzTlBisabkXBqfn5wJojSEzTQhhWZb7+/vrOG6aBhGyJWI6jhPH8Xz+V/Tt6XRKLSMvi+V88ejBQwwhAKDT6dR1nWRZp9Pp9/vdbhdrkKXpZD6zA29ndwwhDH2/rmuIUaPEtvTr245nO2EnqpqaEZolyWAwiLqde3fuyra1XIcwttisFdaIouvry21RoOQNJqyV0rAsyszrmxtqGpgQTMn2uWNStlqtzs7eGYYBESqKot/vYYyLLN+m0LajoTzLgNaUIC3VdDqllDquhTFuOEcIzWazcuvGQsixTd+1yjymJv67/9O/9/F3PhNAA4SKstzkGaWUEsO27c8++eTx40fj8ejxowe/9p3v7h0cSK1aKYbj3Q8/+Xh/73C7eDBsyzRNz/Mcx/F9f2dnB0My7Pf3j44W66zkzWq9efX67bdPX8xny6fPXpyfX1JEZ5fXomqklBfnV7P50vf9thWbdWJbbhh0lFLdbreqKinbX//B91azabZZDzpdxpiQ0nGtDz96HyHg+67UcrGadYKAUTJfzJIsiTq+1LKuS9s2i7qI4/Wg37EMlueZaZt+4EkpgiAYjUaDwaDT6ezv7zPGTMOQvK2LUkt1cXF2dHTYlrVDjMjz+mHn/v37UGmkwdHB4Yfvf6SU7g+GUumsqgGh67QgjGmtG9EalkMtmzDj4cOH9x4+2GxWs9nszp07x8fHlmUNel3RNnVZ/vznPz84PprN52/evcWUuq5b5nkUhBBCAAAzjaqpXd/7NzNAIYTtOHlRUISrovz4/Q/2d/d+/D/8ye/+zl/bEoBDP2CMvTs7s1ynqsptUMq2bQiwEroTdPr9Pud8s1mPhoPVcmFY5mw2S5KkE3SGvf5kMqmqynU9irBtu4ZhFEV5fXnpWXa83hiIlOVfOZmbpjEN4+bmpm650Kpt204QDofD7W1ymysLg0AIobW2XQco7TuubgUzjaqqQter80IpZXsul4JQ48H9R0dHxykvKs1He7vrZL1cL6WUGMBe1DFta1t5JQifnJx89MnHeVVWvI7T5Ml771UtP7u6vLi6zovUNliRZuPxOIoi06BRFBR1lZUZYXQ+n3/17VetaD764L3Dg71O4PC6alpelLXv+3fu3KnqomkqjUDZ1HmerxbLqih4VTctZ4wtJzPAReC4UID7x3dO35ze3t4mSbK9AA16ne9957PbyfV0MdcYep5XxRsTqJ7jqrpGw5M96hp+YP/WD3+dALNKGlhxBxFimNiyS8GJxSBDvK3rmkdR13Z9LtQq3hR1VcsaGSDN4tVy/gd/+2/9w//gH37/139rmWaQEk1A2dZcC4103lTYMlqoAUF1XXqQuopWWc4Yq6uqLEtq2S2EAigBFDQMbDLDJAgD27SaptleBrfdikaCRkipBQQaQ5DGcRynTSsAAZbjQKmQhFBA2MLFdHbn+GRnOPbdwPFCTXBeVxkvOqPO5exysZ4himrOW67Wq+S7n3wHNQpzeP/4wd7h0fnlhRd5lKLLy4tOr9sbDoq6OLpzBDBCJluXmYD63cvXPjUZxE3TVLxBJltkcaPlcr0OOxHGeLXZ5GWhgN4kCSHk6dOn2/JIEIXMsV6+fbNV92ZFKaQyqaFb2dRcSFW33An9rMgJo1Kq2e20F3b6nW6e59PFfBPHQiuKSTcIhRCUsqLMPddRSmzpTlXZYIx3d3c///xzyzCVENvQ6mKzDobDTVkSi+7tjasyr6omKWpImdBV2HXyujHdoKyaTqcDofYClzHSima9Xsdx7AZ+VVVhGEIIlRa2bZgmk7IVQhiWSQ3SCoEIblq11UZvP5D9Xo/XDYYoi1OKmYkND5nFPMYSjrpDxwt2DvaO7t1DhlFX3Pc6CJAyyVXNmyJfz6awFbOb6/FoQA1y98mDw8d3S9lIKb/32Xc++OAj3w0C2/ctj1Kj2+1HfrR9UxFipAUP+kOv2/W7oRtGgBAnCA3LjcIBVDhwPNe0ZC21QlKjtKzTuizaJi+LyWw6XS1evHm9rS/9F//FfwEhvL68+qN//s++/fabz7/45T/+x//ff/bP/snTb7/9yZ/+6XozB1CIuuoGPtLKMlicreN46Vh2solbyQejftM0SrQE6NB1oJKyqUeDfpanTdOMx+M0Tff394+Oj4umHox3TN8djXfyPH/v4SOiwOHO7s5gaFFmINL1guXtdHZzC6TKqvrNxcVss8pr3gKSV7LkEmHq+CGXerB38PGvfZcaxDDJ7/zub15cnad1qQk4u3xX18XOzjCKosl0mhW5ZVkYwvl0TjHFhGqI/DDAGMtWWoY1Xy5M22paDjE2XBtQDCHcHY7+8G/+/s/+/C8oIRjje3fu3l7fcM4PDg5M1xZKmIzs9nuXZ+eeaR8Oxm1atmWz0x9km9gzDFkUA9/nZfX44cN7ByfLm1kUdDzbm9zc7o52Qs+v07TOCt9xHz98NJ8vXduDGm0hNkWW8bppmmZ7UQg9P47j6XQ7DGmWy2Vd11qpxWLhh2HN+db9UmY5hsigzKQsi7PA85VSaZ65rtsqqSD66um3aV0anuNFwWC4QwilmJRZniaJZbIg9AiGbd1MJpPnz59zJSWEdcV/9fVXLRDIZKUSzLU92zgajy3DfvPiTbmKHUqZZyLPend5Ndjdu/fg7mRybRH29Re/Wq1WiEDCzKjXVUotp5Oe4zKAur0htWxmGFK1vCkog6PRKM+qjhuZ2JrOV1lVrVabD9//gFnmZD6rqmqn35tcXHz+858FgQ+QGu/0V9Pr5dWph1RkW55hkMvrq9OLs4/e/6AoisFodzmfGsyuyzrnwveJ4ziz+Tzw3ajT443krSh4JaWkACulpBZpGoOyKNbxf/Kf/L//xR/9d+7uXUgZpKzIYwRUwyvTNIu6tAI/LwogZKfba5NKtBxDFHhemRdAaYRQkqWG5ygIhJJ5nDmu1RY5RBhAJZSGBIuqNrGBCZFcMGqKtsUQU4wIJo7nJqtFi1qTsrzIu2E3LfIHTz5oROt4bqfTidcbAAVjDFEEGSk3hWWYQknLtouqNAidTmZCK63h7fXNfD7FSFNM8nwNkf78F7882B27rvvu3bvJZCa0aqSYLxdBEJR5IYSwbVsptVqtPM8zbMtxHC3V1cXFg3v3tNZ5nu+Nx2kcj3qDqqm3p3VsMMdzm6aJgvD+nbtpkpRFQQhhthVn6d27d5hpbsULAMJWSakVhshihu/7nX7v+vy8bRtCUFM1vKrvvPfBt19/I4VIkuTw6CTZxBdXl5989MHf+Ot//eLyMsuyYb8veAuURJjEWWpS4tsOhJBSvP3hXdvJ89xxnK1k4/zqchh2ENCMscViYdrukydPyrLs9Xr57S0AgGB29/jog/ff/1d/9mdpnrUaOI7bcCGk9H0riqIta9cwaV1XvV633+0FUTSfzhzD5EnpBp2uG5VC2rY9nU5/8MMfLpfrH//kR8k67vg+QTDZxKKuPvzgPaDR44cPJ9Pby+srajCNidZaKvXs5YvhcPi9731venP75s0bDcHjx497UefN6btnz76dLFe259+/f9egWEvpmLZS6sXrV0VRJVmxv7sT+Z7WOs5ypUAjmqqukzxzPDevcoyxF/hay8Um7vgepkaaFTcX14gQZRpNk4uqYYQyZl5dX1ObUYTfPntNKdUUL9Yr0Tae487nS60hMek3L1/u7+5d3F7nZbG3t3d7eyuV6g9hv99v27ZtOASgKSuE8HK5rKuqE/hHd06A+iucSxzHy/Xm7r0HSdpEQSh4vVisFuvVbPkaMQqV1lrv7u7WZfPkwX1CUNk2TVsLIX78458oXv/mb/76Ol49f/6cYmM8HmspNvHKYbbjOJssDYLAtl2lVJ7mUadT1FWn103TlHPeCcI4jnuDQdu2qm0BJqt4gzUOHefO8cnzpy/yPP/ud76ntS7LmjEjjTM/DMq8zLJ8GIUX7041xALqpmmaqi6y3HAt1/dev31z9+j45uZmvVypUFqOnee5y0Lb9Yo0a9v25vJqNBqdnJxspaaj8fh2Ou12O5xzkxmEEMsyFVCe72/hQkEQZAitVivX9957771nz565zGSMaQiYY+maZ1nWtsLzXJMyrqSiIE02hm1Uom4qLpqWtzKMun4UpmmaJARjonVd5hVU2rast29f37lzp20bgKjW2vfCs7ML07a01kmS5FVp2NZwZzCdTh/fv/fmxXMocRRFVVWdnp7akZ+keRCEGAGh+HKRvp6//vSjz2bLqcwy13URQqOdwfl5blFWaIANk7nuKt7oljuOnSUJRjQKOkTj5Wqz/dPM5/Pvf//X1Mtnq9XKoPTq5oYhuIzXI2/3/Y/e28xjAlobt7u9qKkT2WRofzj6u3/rb48Gw7IsoWkXGhRStxq5pkMVcBhVraDUiJMs53y62bRamYZlYsMxTEKYzewIsGc/+/xgb///+Z/+p//Wv/1vb0OypmlDCLdHv63alBLimGaZZZoAhSBCpCiKpqmUFkgjRgxKKbNMrfV4vIcRdVxPalmKptGy4gIK6FAHS80Qi7yOVggh2PKGAg0bATUuygZj0ul0/E5078mT+x990tk9uvvo/VahqqkxxkACk5qLxcILgv3Dg6IsTcOIgpBQqk3DjiLiWI5jPbp3/3i0m803UOuTo+OuF3TD6MmTJ1VVSd42WdENQgihRrCUnDhOWTVEolHQGwUdJNRmsVRCIACjIEQIAQB81/vogw+vLi6UkFEUxWliuo4XhcvlstfpwlbyoiKUSgiWm/Ui2UxmU8/zri4uwzBUUJe82js+oLaRrNfv33/QSoEN5rpuU9Xxam0ZJobINE3LsoIgCIJgHW+4klc3N0AqqFHb8NFgeHJwOAxCU4Oe72NAeSOyLJlObw0MDIxUA6uUC8EpxVy0CJE8z6uqWa/Xe3t7g8EgjuM8z30/hBBJKe/fv7/tm3z62cfvffiB47lCSWIwodXRybFhmVmWJUky6PXrsprc3CohXNsZ9Hq8bl3LX07W6Sq1qZWukr3x7ma9fvjg3qMHD4HWbcNVqyxmEkIuz84HvY5J8G//+q9/+Ojx3mDYd71hFJmuafm21GI6vXVd+/Hjh7uj4cXF2Tcvvt07Ofrt3//b+/fvYAqyzapJSibx9PI69Pz/yR/+Ww8ePPjo0/c1RueT2c189eTR43630w87h7t7nU5HSD7eG5sOE4L3+/39wwNqe5oYZzeTYDjEnqsowI5z58l7xPNaqJnrDncPnaBLLbsz7DPHOp9MZuv07cXNIkluVovz+fxnX3/9R3/6o5vN+usXL/7JP/vn1zeTd+/O/vi//x9evXy9Xq8vLy9V01Z5cXn6bnc4YIxc3lz/9Oc/O7u6mq6XXjdKy6I76J+en2FCnj59ulgsmGl2B/3h7tgN/B/88Id/+/f/1sM7R3/4N3+XGejoeL+tirsHh7u9nkWJaZpff/215/j3T+5byCo35e7OvlaQGCzJE6R0vFpXRdnv9w8OD7kUcRznaSZbgSFabtaGaQIA27ZlCJuEIg2287duv3d+eXF09863L583SnS6XcYYxrgpyn7YWdxO67rtjHaI77y+vnBCT2OQ1YVCMGsqxdjlfJ7WTVbkhmVmvHp7c8kB8KPOy7fvPv7s0w8+fH8TrzFBz16+cDshMg2uNDINgQBXUiMVZ5skyzRQouG9INokseXYjmkJ3uZ5vrOzAyHwfW++WkqtHMdRQANGcl4XaWYyI+WVMmnTNlpJ2EoDsdli2e/394fDB4cHJiJ5WpR5tR1GIaBs20IIQq3SZOPYdlFUGNOmbqXU4/H44HDvcH93b3d3Zzi6vrx68uTJd7/73eV61SLQCHl5dr6ZL09PT4XU2SaBQhGDXd5eJ3EGNdKyzePN8+fPwzBUQluWBwhjrnty985ovFPWheM4BMAyzYosbUWVxqtW1Lwtnz/9CrTcpqQqCkTosiySll/NppPZ0mIWA7BKp3t7gQAVtTHJktR13Rcvnn/4ySea2hLoszevXdOgGAKo0izzPC+JU8/zBcCGFlBJChEiiBjMs5wiixnC8WQauP7b12/+yT/+/wEplFIAACl1N+rkeZ6lKbMsxlhb123bSlkHYUeLNk1T2zQIIUqov6qQAR1aTllXUoOyrAHGge8v5yvHdDzXmd5OOkG4dYGapunYRhzHSKGiqBA2Dg7HVZIAiAGlf/Nv/93zm5nSS0wtz/OLLJGCAwWqtIx6EcY0K5rVamMz1zbNTi86v7lywwhBXabxJE7vHBw1LVdKJ3E2Ho87nc6PfvSjBw8eAK6O7955+vrlcr1umqaqKqy0YVtIgKool8s5JNj3/cPd/S9uf7meL2zfa9vW9/0vfvFL0zT3d3cdyzIMI+xEy/Uqz/NCg1F/eD25paaBKJG1Hg4Hh4eH6/Xad73RYIgYzYpitlreTG7bpkEAQkKaponCUGs9HAyElKvNJk6TnZ1xEsfzON7f33/91VfMpLfTuWVZWyVFGHnT21ulOEJImaxseFbGlBLKzLIs67LpDwdxFhNC8rwc9gdMKSXazSaxHHMwGidJdnJ8N82z+Xz+8OHD9588wRq0bTMaDDUmmLBn3z4riso02aeffvr27Wtv/8CynPVytbs7TpJEaj2ZTDw3wpR4YVTkucFr2zaN0LmZXIdBkK5Wd05OhoPBsy++ilfLqo0Nk0AIf/bLX2it0zzrdEMAVV1Wk9mE2Van318tV0DpBG4Mwzg5PgYATKZT17FEUf7Wb/3W/OZqdTvlZdvv9xkjk8kkiMKTkyPLseerpWjB7dX1X/7q86O93U8/+YSZ9Kef/+Litnnz7u3f/Xt/J4+TpmmoweqKG5RtV4W+bea8Totivtg8fPR+XRbdXnR5cX1ydOf69N1qtTq8f98NoyqrEUJKi2yx8m0fMQth1dS1boXp2i0X19MpY+z1u7fz2cyx7DRNEUKWY794+RJCONodB350dnmhhWg1qNr2/PmzTtBRUpqMTafTqNs/OjmpX7/99MOPDg72+v3e7fXlarmYXl8nm02/FyIlB93uZrmyDCPN4pcvXqhWjcd7k8nk66+/xhTv7+3N59PADzSCRVlfvbgO/FBK2Ys6SZ51+73NZuNSD0OEKTFNU7atlLLX6aZp/ujRo9vbSdPyo6PjLCuvzq/uHh8Nh8Pr66uDg4OHDx9eX9/mRXF4fPwnf/an3/3sO9/94IOn33z76t1pn+4M+4Pzy4v9wyMHomy9rqoq7ASjZlQUhev69x8++LO/+HPXNKWUGsL9wwPOue/7jBEhhBv4TVNt/bFh0JlMJqOov1gsvNBHGG9VIUCqKIqaosjz3HdcoVVVVYhgw7KkFFDrqqqYZSol6jqVvCHYxpgqBRzHA0Ks57OiabdSX6VUxautTfD29vZgvAsUSJIk2cR+GOwMh1sIT+D5Z2dnOx8Mq1au6jrLCgoN23WEBkVd1mVzfHiU5aVo26OD47ou15skTtOjvUPZijxNO8M+Qfj68sYz3aKq7ajBECGAN5sNhPj+w4ezyTwr0tD3CCFcSdO2CEFJnkGpbNOimJR5UTV12OkaBn73+s393WOLNw/vHC4n53d6HtKCvHn3+oe/8YNXr173O6P1KnctR0EmIDWwXCcrZODJatkJOqIFQAsshZa8VnU02FmnmahEJ/ANrVdp/C/+mz/63/1H/9F/9v/6f/xf/2//93/5z//I9xyN0Hy9AUoDAHhTGgbLqyoIgqZpiioHACilDMPCGCdVwhgzMJFaKwSLprQwRQBLTdKkdi1fCZnzwg18pRSlFFO0Wq2avHQcBzDTNQ3DMMoy/43f+u1ub9Df2f38i68wM4DSshb37zxEGrx789KizGQMSRj5XdHKR4+e5Ou1lm3LG991N/F60O1ijI7v3e32hz//5eeffe87r9+87Pjuu/Ozkwf3FABpnpVVJTEECO4MdqAA02RdVVXOxd7BfuBa2wThbDLRdfvJkw9enr3tBuF26QoASNOUIIwhkq0QQri+hyFKqsILA6UUYyzPM0ZIlqQUQyU4wfDLLz4fjwa9QdcwjMvbq6ZpbMNAAHAphJJxknAp/Cj0w0AJWad5348sQi4MI87Su/ce3UxuAVCGZd7MpkWaHO+PtVTz5ZqZJmEMYLTKMq0hYWidx1poACWjxnq9tiGhDLdCGABxLtq2/fLLL7v9/ntPPjg+OazrGkrV1GWapmmWLaezo/0Dxszz81PHsrtRZzgc3txMvcD3Al8oudqsR+NxVbazzcrpBcwms+s19U1TW6Fn6qYOBoM37966Qfj7f/gHP/3pT6ermRB8XpVy01Cs2ZW1I3b29vZGOzud25uiKDRERZYxxoKo2+t2V4ulQego6v7qJ//aDJz+zogXldY66Ppvr975vr+zP37x6kW/Nzg7O7v34H4jWgWGByd7q/ni9duXjJFut4MMsn981Ep1sLM/3hkmVbFcr7DQDqNJXoSdbt1IhEicpfE6GQx6Vdkc7+9FjmcdHA92hl+9eP7k0SMg5GKx6A5H59c3Quq9/fHTb75+/P5HBNE0ja9vJo5oXdd1TEu3vGma8f7exfUVtYzv/fD7y+ns6uoKABQ4ATLw2fXlxx9+PB7vDaKuRUm8We8MBzVvfcv53kef5kX66utvb1xr/2ifYtSNup7nXl2fL+n87sk9P3CytGjrFlq6LPOfff7T+/fvm40VeN705rYThMS0TC03aTYajYok39vdbYVqlY7TPC0rxzYZxnmeU4xbrVohepa1M9hpqvri7Pzeo/t5kjKIW6AD243TjRf4s9UCY5JlWZInUa/Tsb1nX3zpKLTabISSpmlSoUdelK7XRyfH+WZTZPnJycn1+dX08vLJBx/6rndzPfns00/7vdF8vgo7wfn5+XA4uLN/cHF16fdDjDGEhBJD8K0nDn/6/mfPXryo67rOCsMwXr96Ffa6ZVlYlvXhw0fPnz9XhkEIaZqm3+3OJrcAKtfx58ulltyyrCSupAY7vSGG6M2b1ze3V/cfvQcxaSWPN/HWu9IJO69evTKZiRHUQkedYNjr51kq2qYuQODu+FaQZ5lhGO89+Yhz/md/9pNH7z+5TjaW4xKNXde3LGuxWCCpm6YJ+5Hje2mey7ZFCBZF0euOdnZ2vvnVl50wsDHOk1hRczzcT6viq29faCBbpOJsIyFiQRgXuWp5tiw6jgOU5nVTNbXtGFiJ1XL90QcfWBzdnj0XWhwfHyFeEITQ93/z168m06ppnz19ajLDtZ0PPvgIM1bUjRd1dg4PsGVIiAAmVVVhAj3XDsNgFSdCA85FWdRxkvleWBVFulmfv339/qOHZcMt24eYagBaKZhpUEp50zDG6ppTTBihEMJ+v5tl2WKxMAwDY9gUFYUoy7Ig9GTbqJZbltVUHANYV5UQnDFWyzar6zTPMaXRYNAb7RJmQUS6vd53vvs9ww0kJrfzRVpWoRcejHcty9IYmZ6joFJKaSmjIDx7d1qWZVmWrmtL2dqmBZTqeH6eZoQgqeWvvv6qO+iu12uMMYQYErqOU2YaO+Ph5dW5b3lQo8V82WoFMWraOuyGUbfDlaxF27S8aZrtUNWgjFLa7fc2WWo4dtTtXl1d+a673axukoTZFoRws9mYlpHEGyiVZZg7OzvbcE5Zlq7tMUIxJW9P34VhOBqNoFbxesM550JlaTrqj2reZEWZbuIHd++UaWYaxs7u+O3ZuRDCc1wEsGPZq9VaKFC3Is0LoVRZV5bluK6/v3+4f3i4NQqFnejBg4ej0QhCZNgWgsQ07XgVR35Qlc0W293rRvPprEjSbhCWSWZSpqTshNF6tVqvFv/uv/MPbm5utsxI3raL5fJ2OkmytNPragAIIePxeHd/7HgOxCiKojTZ3Fxe2IxSjChBw16EMT46OvqDP/iD3/nd3wvCjteJiOlc3Nx+/fTlv/zjH//xT37yzfMXs9ni4uzMdV1KaZxsnj59en5+fntzc/b2HdZAt/L10+cUIt/3Lcf56NOPvCB4d3Z2eHh45+7JRx99yHmDoepEvgJqZ2fk++6g3+8GYS+ICMRKKCDk1ellk5eR6zuGaRumEtI2zJ1uv+t5J/v7P/i17476A8e0TEKHw4HtuUqDg4ODy8tLBABQOk/Sj9/7AGsla/7w3v3J7QxjnKZpEHpR4AWut783Ho/Ho9Ho7oP7d+7d7Y9GmyQejPqffvrp/mh8MN79tU8/+/jDj+PF5njv4Ors9PTdu6blW/NGnGxuJ9eTycSyrF6v9+7du9PT0/V6vVpvOmE38P31atHpdE5PT4MgSNO0aZoHD+8yRglCURQNBoOL65skSyfz2f7hIQDg8PCwKaskXnu2hRBijEAILccGEAqti6beOzgIXK8TRpPZ9Lvf/15elm3bfvjkPVFX33zzjR+FSZZKKYUQ/X4XU1w35aN7921qZFnW6/U4519++eXOcNQ2XEoZhqHrure3t5vl6mT/sNvppJu169ndbvf87JKZNqW00+kgAJNNopRijKhWa6EZM6qqVkIQhPM83/ZjOOcAo0a0pmWt1+v+cFBV1dNvvoUaxHGcZRlBaD6fU4NRZpZF0e/3KTEq3tqe27Ztr9MFWidZajuOlBpqwBiTreiEUTfqrONkNBrHqxhq9P+n6b9+bkvyM00sIpb3a23vP++Oz3NO5klTVcxiVbGKZDtRJDUYQZBG0p3U0AjQbUOArnQlaCBBGKGhG3XPTEuCmlM0zSLLsGyystKczOPP5832Znkfa0XoYlP7P9gbsfaKiN/7Pk+z2TQMw3PcIAh0XScEvD09SfJsOBqlWcbJ4vHFmWYZBAKJlwIvRCy3WNrj6QRjbNt2mmeI4S6ur3Ja8qJYlnS1dPI8NyzTrFhFQcZXN5qqzpeLOMd37z0YT2eQY9/94EkBKC9LCCFZU10/qNQbSZoTiMoS93odRVEwxrIgnp+dnZ2+5RjQrJgkS4scMwyHziaT/89f/yXgWVwUk8lEEvgsiWRZ0us1N0tPLi71ShVTEmWx1aymBLux78UhS6HIcDzP2q7DiSKE3GKx+O/+m/92Y7MLWUooygvIQMhxnKSpaVEUJQAUQooYCiGmSRAihGzPFcW1lCOBFDGAgQXhIHIXK1kUORYlYVC1NIJTQxEFlplMJllJ0pLIqtIZbAS0nHhetdL87u//wbvvf3Bw9+7FbPHi9NJNIwrJ8ZvXBBfnw+tf/O43Xx1/DThKEMC0CJKgWq/wMscwkDAwxllJKSBA4DgGUkhRkeO93a1m1ZoNryEuW42WZdZc17c9V+AhKHKUFEyGcgaFtMgJ5nlW4jnbXmakQDxvVWvj6UzUNTcKh8MRgkylUXeiICclK/Kyqkznc03Tzi4vUpzUarXLy0ueY3kAmpZVr9ZVVSMIZEW5vbdPAaoYlj1fTSYTXhIhACUu8jjd6A8s3RI5oUwLRIGhW3GasAJPylJVBJFj25sDO4oYhqG4YAlMvHCjP5AkaeWGN4vV3PMFRVNkQ+Hl0HMvT89YytQr9VqryQmcKAg8y60cO85SXTXiMCkxSeOY47gsSxVJLLMMFSQP4qOdA1VQPDfwAj/Nsl6/73meokgAElXXMCCURTzLNWp1PwxTnCNIcRBmjpc5vszyoKCWXuOQsDXYJgWBpGQQVUUOJ2EeeINW46N3361b9e3dI14y57Y/WTgXk3mQ5TPHC+NksVrOZrNapXr39u2dnZ37jx7Wul3CMZBBjx496vf7uqrhJFUFRZGkTqsVBMHx6enZ2Zkk8tPxJHDcPAwlkbesquv4ZZK1rdpmoyVC5DqeZVVUURld3BQFAQy69+C+pqsAJ6YiihwMnGWZxYiUmqaOJuO4yObeyqxaEIIwjtrt1mo++/LTT3r1qqGIlmVBlpnPp+/cv/few4c7GxuZ70NcMgjphjYZjrY3twguUowl0+xuDFSB32i1SJzlTtAxqjgIBp3OxkYPl2W93dErlh9HXhpjDj07Px459qvXbw2rYtQrVqNaqzX63UGRFb7j7+3tSYoiSVKr1YKEBp5DcU5L/Ovf/qMgK7PFClC0mM1kWXZdJ0rj9QQLJ3HVMA3DGE7GkGcpi+qd9v133nn3/juXJ8eyrjUGXV6RTMuAgLQaNcBByMB+t5fGianpAFKeZ0eTIUBUUpUoTcIoVmVVEiQ3jsxG3XGc2WTKS3y72wriiEBgOw4viZACz7UZhmk2m4vFzF6uWMjXqi03CAoAiwxDysRZCiFsN1tJEKqifH1xxTEMwzFu4KekYAVeUmRnZVcqFYxxWZJer4dYJogDhuN4UaEUIo6dzmdumFLAhFHEcigOnCLLm+2eYdUC14v8wJ3bLCZsXkZLh1CqW6YqCpkfLSYrxw4kTedEgZQoy7AfhVBgckBenrz99e8+oRyDZPXlm1N3aaOS6poxnE413RxsbKU5hiyT05IReT/yndAvcaEpiqSIb87erlarLMsVRXt7ejkL3aCI0zT9+BvfqlQqKc6r7abtrhDGbF5qgsQARjWtHFJBl2Kc+GksabokKAyAcRnmIG7UzAIneYEVw0TLlbd/cFQQKsqCqitB4EkiV7MqumG1O4M79x7uHd2qt3uQ42KcEFAClpMVVRTFLIlMXVVV2Q8ixPPz2VLX9TxJv/ft3//+97/vuC6llGUYXJSIYURRJgRACsqyZFiW4QRJEDmASFkKPMtAtGZ/UwqLHEuSxHAcBiTPkrLIIYSu55Vl2Ww0Op1OrVb7/e/+wTe//e3/6f/if/mNj7/9/X/2R7Vqw116z756IYqiqCrj6TxM0iAKJ8u5bdvD4XA9maSIiLKEcWZWKzzPpzgvSlqtNYbDYaVSsZeLIse6rmdZFkcBy1BKiixJfvfp5wLLGZYZhuH18Ebkhelw1Gu10zQHEK7fTGVZnl1ehFEU5akoS5zAUwQJpUmSVCqVs4vzNQOS47gkilu1+uX5ua6ohqbTsrQqZomLTqs9m82m02mWZZ7jFTmeTaaE0PF4rGtGu90ejUaCKGdZFichLjJDNxmGEQSBAlBgXDGr7W6nAJRS0h/0ZFXJC3xxdclwnOs6SRLzLCMIgh8GgiR2e51areq6bpzmcZTWKlWe5+M4nk0mtm3nWVwxtVa35YSubS97nZa9nDMQCCKHcSawDENJvVINgmA0Hkdpsr29HQRBinNeEhmOhRAmcRbHcbVahRSYupklaRAEDIMEUUQsk/ghCyggtFqp8bxgVqqO7ZVl6Xme67qAFu1WPYtjUJQ8Yna2N999990H7zz6+Jsf37v3oNPpdfsbcZwixGZJhhDyovDp868ZBqVpChhQbzWTNHM89+TkZG9vDyE0HA7zPC/Lstvvm6bJIjSfTNuNJiSUhYgSmObZYHMDUGjPF5EXDHr9wdaGG/qAUAYihuMODm+dnJz4rscwzNXV5eTm5uH9+51ao0zT4zfH9XodcpwgSRzHS4IAIcrzfHtjs99uT0djjmUn09Hm1qBSsXzf/8UvfsEx7NHBvqlrCMHz83NAyXw6qdVqmq5fXV25rlsS4tkOLIkqybqhloDmZeH6gShLw/HI9b2zi3PPd4sCNxoN27b/+I//OQNRq9G8urgEAFxd3kzny+vhkOf5+Xyep1jkBVxSq1btbg6iLN3a2zeqla2tLUVRVFVdR6TCOCopiaJIlSWKcWC76xYhYNAHH3xAivIv/uIv4iQ5ODjwPK9RrX322WecwHEiTynBGPM8X7Uqtm23Wq0kDhVFgSzDSeJstYIQ1itVd2VzouAGnqaow+ubjV6/KArbdRiONSrWqzevarUqx3FrDoFpmmEYlpQu7ZVumaIoAoBYlpV4IU3TMIxEQQIQrtm/qqxs7myneYbLglJalqVr2wzDBIHvui7PctVqNQzDPM/zoozjGEAoyHoJWISQLIs3w+HXX38ty3KeF6Zh8Bxn1WvdzYHIcFmc5HkuiqIiyzzLeq7LMOx0tfDjxLbtdru9ublZr9cJBFatzstKGEeGZa6Fsa1Gs9Vu7h/sAYBkWaYQCqJou06j0WIYTlGU9ZosKUnTdGtnByCg6lq92ZAUBSA0Gl7rhipIMqHQqlbef/KezKHVZMRSCACwwzCjMEyzk4tzyiJBEsuyJHlGcNyoaDgNNvv9jY1+GAdIF1REoK6ZbhRgkNv+EgBg6HoRFyxl252eZlY13cqzIggCTpCKEi3dICnirIzSLNJ0VbG0VeCpZvXrL1/GS//5bz+vGhrHAlEUPddHhFJC/DDEZUEARSwTUpySIg9jGbKQEJZlojgQJd6JPYxKhmGKgthxzMkKIJQDqGRhd2trc3tHkbVBu9tqNmv19t0790WG++DDJzHJzkbXzty9s3XwL//g+6Yk2LY7Xi4jSi7H12WR39ndpWmpCJIgCFmW8LzoruwyIzsbu5KoCrzMcdx4PNJ0BSIqyrJp1TRdV3VNUsQ79+6yDPPq1assSzDBoqIvXd+yrCjwFY4TEStxPCUkL4vB5kan3yOUpgUuAIUQlkWxZqJSXG50e5amz25GFVXvtzo8Yn3b2epsIAKur691UwvjhOF423ckWa7pJo3zXqNNMlxvNRmRS9PcNCrVeq2kxPbcq+tr13WDODJa9VqntZovCC6iLMWQxkV+cXO9u7FV1U3IcSWkqqVUG/p0OqzVzTzPESVZ4N1cHvux+/r8LQGA/ydODoIULGfTIg1Xq1GU+EZVqzaMMHJYjooSCwCRZdldrELX40Suvdm3Oq3L8eji6jyKgma7lRfYj/zRZCyrmmP7DCYiRTISijRXJEE1lBDHGJZmsy6qyt6t/dF8MhxNKEC8zOdFblZr1XrdD4Mw8i2rQghZLpcMg4oyKYqo26x99OTx7b09keW+/Y1v9dq9g+3D3d0DtWLJltHqtJfLhaLKrMB3B/2SAk03fvXr38zmiy++fHo1GnlRdH5+4btur9UpExz60WQy6/QGS3uV5ni5svOStDvdzcFgNpu8fPsyKxPKEl7hIMe4vrc92OJZHnKcVam2K43J6VXmxSLkmpWG53j7W3uz8fTrz79q1BqkBJZVsW2XF8T+5tbNcDjod5zV3Fnavhchlqs3mq1Gs9fpIpa59+AByyJntSSwjJPo8f3HLOQgz9pJiGnZHrTn/tLNY8LxeQlWK5tnec9xBZY53NxCafb48M5Wo/v1F1/iJKVJ9v6Dh0+/eHp2ccEp8sJzZ4sFwaXveI7jFZSOV6vffP75zPecOKAIfvnV0/XNhud5iqYKkggQTPMsTzNQEEUQZU5o1OqHO3v394/evnzlJdH24b63sjM30CSZUloysLu1AQC4vryCEFIIFvYKIdRotJbzxXK1ElWZQsAKfJ5lhq6/fvliZ2tLlxWKc5EXoijClBSAKpoiSVIURaZphKF/fnbCQIAA3dgaMBxSRdFzHMgyRVmumYNBEGBACkgAAyVJnk2XWZY1Oy0CaVmWmqgxkAEIGpbuu24SRTjNRJ6XBVHkOVoSyDJ+EhWQ8jyPMa626khg3JXdqtXnk3FeZNer2cVsDDim2Wyu7cq6rpdl2Wg0eJ7jREGQJYRQ4ocCRYkXSLzE82KZl41afTK+TmK/2W7EabRcjEJ30WrUCQGsyJaU0BLMhlNFUJz5klcEyoCV6y2XNkBU0ZUkT1gWbfU3Iy9UFMkPvTCOV443n850SdEEvl21NF1yAl9UdcDwAHJWteb7/tXVRRB4NUsXirSusKrE2atZEDoQUbS3s1e1aiyHAIMc371z70632/ZshxZltVonJTx+fer7oaLrHCvEccwJkqJpYRxwAhcEnud5rusXBAiKslwuf/KXf3N3Z69umWkWcYKkqjoDESSU4zgCgCzLeYEhhAwLCSFFURBC1rs8x3ULQhC7/vAYl6uVI7CcKIrdTh9xfJLmh4eHB0eHB4e3rofDv//pT7/8+uvJZDIajQRBsCrG6fnZj370oziOJV4ocYFJadWqiiiNr278xSJ0vLKgiOMppYIgyLL8+eefIwaOJyOGYRqNxnw+v3P73nJhu64bRcnF+ZVjezc3NxjjXrfLc4zA80UBmq3Oxv6OG7kElIHvx2GiyVq93izL0rZt3/MQhPG6n5Wm6/7eYrVcrVbNemM5XxyfnKw7b+ttu2vb25sbZVlqhrFYrZrNJgT0s99+yiKmLMuyLG9urmu1WlmWjuOEYWhWrDzPzYoFWeR47ngyibNUEATLMDVNw2WRlrjb7bYazU6jeX5+Djm22+/hspQ1OY5jjoer1WIyHs7n8ySOe61mr9uyNOX+7cONbqdTqykCv1gtc1IOx8PffvapUa1QBkEIVUnO4iQNI11W7t25OxpNHM+7GY+iLLm4ukQIPXr06MGDB6AkJS4AoQLP5XnGcVwUB77vG4aRxgmLUBAE60H6y5cvK5WKaZoIoSSJEUKqqgJCSYkJLgguAC4lQWQBZMsSkWI0vGIokThWYvkPP/zwwYMHhqbfOjiklG5tbV9cXERRNJ5OR5PR2cU5AZSXRIZhOI6r1GtrnRbHcQxiF4tFrVYb9PqH+wfn5+eaphekFGSJZdHZ2dmzZ88QYCiC7cFguXJsx/NdTxaltSMsiqL79+8vl8u///u/H41uzIpFQKlp2vnZSb/T3d/bIYTkRXY9Gt59eM+NguVyLgp8mWNAKCXk5upaVdWTk5Pzy4tnL57zonh1c93v9x8/fkyKYqPT4xADIWx2e/u3jhRNDcOQEflnr1/2NnpW1TRNc3tzc3tza3dr9/LyMovip7/7HJZlq9UKQ//k7Zvz05PB5mBrb8ewjG99/K3t7e1ms/neB++HUUwRLAF49N57lUq12+7EQfjh+x8IHLf2MUVRZJomASDPc0mS1hOysiz7/X6n2fqbv/prQsjH3/vO1v6u57jdRqtiWpVK5deffEIArdVqgPwTMHwxnc1ms3qlyjO874fddkcUxcvrq63tbd3UZVlGgCIEWISGw2G73eY4DmNsmiZCwPFsVVfCwAOA3L59u6R07SUGANCSxHFYlDkDISElRAgXBUAwSmJRUhBCURSFni/xgqaoaZqGflDk2HXdWq2maVocx5SW69cYz/MsRBhnCJA0TYss7/V6oih2B31VVU3TdH1f1bTe1kaO8c1oWKtXcpyulo6iaYCFXhJgQgGDdF2PQj+Jgla1Wa82BEEQRTHLMsMwACDLxSzPIp5jt7a2DvZ3p9Nx1aookmqoWrvd5hgGMijKUi8OKaXb29sXV5cUkjD0JVF0povMj25ubqbTaUU3+u1Ou9nSdT1OkyRPeA7BsjAUNQ7DvMBlWYoCh/NclqXh6GqjWTMEVuP5KPSzJBU4DjEMN5vNFEViEUUUzifL8Xg8XUytVo0TJJKSO/t37t+/32y1AGJ0owIA8AO3hIgRBEGQOIbr1LuQMGmeVSrmr3/ydybPfOe7v1dt1gsA4xxnuEAAZknCQBglMS1JGcdMSSOcZQwhELAsmySJrusIoSgISEED263JlV6j09/c4mQ5TzHB5Dvf+/7O0dFoOScsevLBR4gRepsbk+lcIDyiAEhwlbo5y6+cGBbZZqsm81ySZDwn3b/7wNI1gV3zrYgsiywDw8Ct1yxK0iwNgyAQBcUyG2dnlxDRJI2iOOF46fDo3mQ8RxRcX15td7t7m9tZii+H4y9efemTiLBIkCVKaRSE1xeXHEBVzdjf3IJZwQCoaZrreRXDxBgnOJdN3Q+DarNhdZqXs7EoikWO0zxnGObq7PzO4dHKccIkMXUNQcqJfL3TKiG4mY4kgeNYFEcJIXR7cysJgzROtjY2a4267/t7W9tZFK8ZGgwFznxpajoA4OT125pZxXH+xWdfICRCJHKcYFkGQiCI/BgTAlgeoKamdS2tWzPqpn734ODOwU6nVS8R66U4TrNatf6TX/7KCeNGs8+zgi4rpqqkaUpK0Ol00zTL8lTVNdsP/+TP/zN/5bjzZa/Tr1WrvWal2TBfHL9BmqjWDcoBLwwqVk3npEGznUVxGIZlQWlJsjTWVJngUuTFLApxllaMCs8KDKX72zv2fGEpmiXLCgOtiiarosRzHIuG45v5as4LbNW0Dje2NU4cbGwOdneSAgdJcjkev3z99pe//LVtu41Ga2/voFmpdepNQsjKdWaOnVN6dXVd4rJRqYkMhwCcTcYIlIaudNudTrMtS8bp28vLi5GmWAJknz/96u3Ja0kRFIk/PTne3d+9/fCeX6asIe/fPiSknM2mo9FwPB65vksRSIrs7cUpy7OgKCqqjnG5s7nTa7Xu3jrYHPQhi/wwlFUVsYzjOCcnZ29eva3r1V6j9eL5177vLu3Fm+O3L16+Oju7xBH+xpMPCMWT6bXvLaPAi8Pw1cvXJWR2j25vbG1JqlICPHdmrX5bUMXJbIyLjEHgxVdPJ5NxCcjnz55qNTOOoyyMSJabktxUrf3Bduh6jUZjf39/tVptb27VKlVVlmuNRoHgKg4zRNtbg4Ojw+FkTCDqbW5RSnme1wz9N7/7bRhGjXpTFsXA8wfdfpZkURSLnHCwvz+5Hu1t7iqS6i9t33Z0XZsv59ezEWVAFAVRHPQHXYqo7TqqquqyMrq6LnBWrVb9MDAs8/d/72OZ5yeLaZxGLANNTY2iSBA4hCADYZZlAICclkjgHNttttppniimulqtZFEGBOR53u62NENPolgVVY7j8qIoKGF4jkAgiiIlBBKqsLwqSLqsCDznLVawIBnOKcd4caiqasMyFU6I45hQOl8tp4t5HMdxmqzSUKlZRrUSuEGv16nVKizLRlHk+z6EMEmj5XLZaLazDFcNrdtsUgIm0/lP/+Enmip3ag1vZSdJsrG95UTBvccPG+2OYVYURYqiQKvoJSFJksCy4AG1NJVleUBR4gUVVU/C5Mtnz4x2e2N3D+b5VquuIiBAUjGNw/2DjUZbZbiizBmWZpHbsHSaZ1VVN0RZQhwaTca4KHgOLecLkpWhF06Go42NDYZhnj//+vTs3PX82WJxcHBgmfUoyvI05Rm2Xq9DCkRehAUguECUMBBFaaSa5tvz0+O3rz7+1kee53G8yHEcx3GSIJKihAwDGaRKMgsRQAAiBAAghEKEEMv5vi/wkiCJ/c2taqPBcyIBlAIgy+p3vvOds4uLT377KcPyGBcvXrxqt9uff/r53/3tj8IwHI/Hv/3dp1lZTKdT27bXZ4J6vd6qN0RRnM1moCjzBGdJyjBM6Ed5VhBC0jRdLpcAAE01XNcXRXFdXrAqlXWCSpbldrvbHfQfPLh/c3V9c3HJspxVreRl0e51zarV39xwPZ9heY4TsiwbXlwhXOI029nZEWRpspwXRSEKwnoPoqrqydmpVa9u7GxdXV2JomjVq0t31Wq1JpMJpbQsMMshjmMYhnE9r9PvFaREHGvpxuhmrAqKvVzd3NxwHOf7fhonkiDIksAQwHEcQkgSxDSKGYg4hmURAwB4+OAde2Grqs4LIqGM7QSqqmcZvrmZXJxfjYaTs/OLX/zmV59+8dlPfvzTX/ziF4vFYjyazFf2eL4qIbtYuRAy/f5GmuYsyyuSGvpBvdnIcC6wbJ4m8/n8008//Rf/4l/Uq7U4imRZ1nXdcV1O4Akhm9tbXhgCBE1LRwh5jnN5cRG4nqJI6zzryrEJIIvFQtbUNE0RggjANce4LMvr6ytd153lSpcUVFKS4dDzKSk3+71Bp62IAkDUde0sTsIgqFarVrVSq9W++90/+MPv/4Bl2YcPH0KW+eKrpy9fviRlefvWLUPVoiiq1Wp5WTi+t1wuPcctslyRREWSG41GWdLpdPrs2bOaYW10eru7+2ubT6vVMk3z+vo6juOvv3765dPPcZmLqnx6cfrLX//q1fGrw1tHhqV7gZ+mseO5iq5EaUIRVXVtvlyUuPS9gCI6mU+m03Eah61WS1aVOIzu3LpdrzdM08qSbLVY7e/umobu2jYCUNe0Nfn5+NXr64vz0XCIKEjj5OXzF7bnMrwwWiziEr86fmu7jqprW7s7nCgggXMcZ3I9rlcbpCiDKKy1WrbrlXkBSzq8vPZdX+R4RMFqsYyC0Pd90zRZll0LhuI4XhsFDg4Otra2/vZv/9b3/bsP7iMKaF4kfiiKIkLI8/2yLAWOJ0VJCVE1bblcVhv1td/16xfPOY7TFTVPM5EX+t2es1p2Op16vXpycjIYDIqyvLq6QgjpqramklUqFYyzsizfvnn9xRdfVCqVsiyLAgsCzzEshHB9kuB4BiCYl1iQJV4Sl8tlQUvD0NvttuM4SZIQXCxXC0LKPMcA/BO4dG3fBSUpy3LNnxE5PgmjPE8ZhtEUVRTF8Xh8enbGi6KqqgwBIsNV6g2jUvWWq63eoNVpp0UpCFwYhm9fvW6322melQhUG/WlvbAsC5KSRUyr1VgsFnmeE1xCSheL1Xg0CsPQMrTlcrm/u+Osllkc7exsnZ2ddQd9WVOzKPTsFc2Ldc4nThI/8m1nmaQpwzCyLL96/gJCJk7weDoHCPX7G85qyUFSr+hnZ69tZ5HGiSIpCEBD1bp1S+MZDgIImBIXlFKWIEoRXSxmNcMCJV8UtGrVkjAqQ/8Hf/T906vRfLViy6Ku6xWjEXmxJLJFmWRhoIlinhIOsPZ8wvNMnoSGYbyZL/73/6f/sxe6oqppipDnBU5yUZJIjjmBC5MYskxGS4CgzMmUEJxnoij6SQpxMRhsQsgoopSmeVTmkEWWZjx58iTD5dOvv97Y3aMMOtja9QL/qzfH2zt7vXqHI4zZrKiFfjW8UVjlT/7Vd/Oy+Mkvfy5KSrVScZYrlecXI4fkgIc8ZcsSYyTJtISqqi6XS8to8Dzv++Fked3ptVRVgog3Ne3lq6/qrWYQ+wTg4XhOhuX+4U5W4Dyj0/Hk7r3Dy+srhRMkVWV5Li8LRhYNq9qsN0hBQy+IWGQmkVWvNTstZ7nEUfLgzl17udrZ2REkaT6fS5JkVSs5pF6e1BR9a2vr9PR8zT0XNBmXhSpwfhINp5P3Hz6M49SQpTTDRVFAxEIIBZ53VqvQ97vdXn9r8N//8Ifvv//++m8oDEOSYs8PrUrt0TuPWYb5t//2//4nf/antVYnT/I4zG/t3vnu/+p7f/O3f/3FF0/jkhAGFvmoquk4iSlEeVF4SUxYNIqT2/fu3dncTv2gLLGoKiIjXM2XrKzmZV7grGrol+cXh7t77z98JwrisizjJHHjsL2xEeS0XIU8K/KQkKTMI2xP57ubG72t3rNnz+I0kTWV97xmu0UImc/nWUkEWeIE3qxbc3sVJLEgiYqpM1Oe4bk4TUyzEoZhu9aYz+fHb1+ZitRp1QEtWp2G6/KL1WKxWECWYSiTh/HRYDv3/MH+zub+Nk7SF8+esyz7w7/47weDwaDT9X1/e39v5dirwFN5MQ7CWrO+mM0ZAnFBC8jsHh1oLP/s+VecohqWWalUX7582ajVu81OlmXvPnnv5fNXplkZaEZGsO378+X8y2fP3nvvXdcLIWQO9w9fvHherzdkSX129uJgb3+1dGxnqVW0/mbPUFScFRAwYRgKDHKXq1qtblrV4+PjTrtZ040oCjYGvSRNm43GfD6HBJI839s6YCivycpsNrt1+06M0+vpeDV2ZE5oNWoEAj7N/vbHP+v0e3sHR6PL6/3DLdd1xZpoe2611sC4bJiVullxo2A4HB6fnVJKW50OIWQ8HhuG8cXXX1UqFdf1er1+WZZ7O7u6pqdpuruxJYpit9VOvODFV081XuRZjuN5Pwo3+4PleErMklCa0xIDYnuuWauen5y+8/iRkOeL5cr1/Fq1wpcgDqKP3//w/z0aYpyNJsNHjx69evVmMpmoqs4I/PX19d7+vipr9nLV7nbyApdFcXhwcHpyvrW1ValUTs5OFUMghJQFFXhoWdZkMuEgp2saKYr5dMoJIgFU0VQEgLOYAwCsasUNfBBDVZOLAsGC8CwfJBmrCUmSCaJIKY2iIM0zyDF5WUi8QCGomObl5SVkOE3TEGLv3b1br1cuLi5iXrYaNUHgoigaPHgHY3x8eQoBYRjOqFXmy4mqqpCURZanUdzvdH03SJcOQmhvZ1cRxDev3qqmYUcuBMRezDEuQz94/tXzbrfdrtYdZ5UnGYcYz/dlTUWQ4wsxStLB5pbn2AnOk8Xy8cNH5xenOMymyznDcl4cEkg3t7tR7LjLsNPuAwCy+Wpzu1MES6oIGQS6KkOGQTjPGrWK77qaptVqdQBApVLd2t0xLL1RryNCsyjutFrVal1S1KKEZUlpTiSWVyV5rRvleaYo8zRNSwgwz6SAMAwT+17gu6HvAUrSKA7DEKcZIJQWZZqmfhjkWUZwEWcpobDd7sqSWhSk0WyLsurH8ebuzoff+Eaj1dk7OAII3b1/v9FoHBwcAch0O/33332v22zY81XFsi4urz0//OCDjyqW9Yuf/+zs+C3L8hzHhWH49vWr2XIhKfKaHUiKkpaERUxZFq7rVqtVQkiWZZqmdLudkmDHcTDGV1dXFDJJkoxGI6tWZTlOkESMS03Wdre3Wq3mcmlHUXJ9ff3Vl08RQhCitTHy7Zs3HAPt1eobH360prlCBsmyDDKcRTEAIMnSna0tUzem47FlmOObIQtZQRAW9iLLElNXa7XGeDydz6f1en05X9RqNYRYCGGR5RuDXq3RWOuBNF1fk17MivX2+HhtR6IQxHFcMcw8zykEkEGLxaLVau0fHfz6k19HSSjLomVZnufNZrP/2f/8v7h1764deEGSYgrm9jItyizNbdsxVfUPv/e9//jDv/gv//X/ejmfixy/JnpnpOAlMUmSerPx+Zdf/Kcf/ejw8PCdd95J03y1Wuma2esNOI43DAMAyPN8VTfLHGdJvpgtWq0WpXS5cjrd/lp9KYuS53lJFBuaDiFN09RxnOlobBiGqqp4zeNm2TTPOUHCuIzDZDmfVwxTVRSORYFjiwI3nY1dzzYMbWPQ1xWV5Nhb2iTPCC5W8wXDMCnO+/3+/u7ek8fvkhz3mu2dnZ2vv/6a47hGo7HmSp6fX6wcJ0hSyqCNrU2M8a8++RViWUHkWJ55+vxrP44IoIjlLq6uXr583e3193b3wzCcjqfOcnW4f9jr9TiObzQalIAkjPqd3mq+mI2n29vbXuDXGvWHjx49efIEIXYymlYMczQaubZD8nKj24vj+KuvvgSEnp2cfvH0y9XKGV4NJ6Ppj370I2e5mk7HRVGcnJwKglCpN0pAL2+u3r59DQF5eP9Bq9WyXV8SlQf3H7YazSLLsywXRcmsVgzDWCwWhJAwCLqttsKLk8mEYRhBFHf2dkVZ4nmeUtpqNMuyrNfrgiRqhr65vaWIUuQHq9lcEcROp7NcLk+PT+r1ummazWazXq8bhlHmeM2PS+J46dhHR0cEF0mSDAaDTqdzc3NDCNFMI01TRAHHsCIv/NVf/ZWu67KuXQ+HgEGr1aooirXuJk1T13UFnl/M51s7O5VabTVfmZr54MGD9UhGU9QizwEAJcYIIc/z1/OnKAqyJAvDMIqitbjYcRxZllVVDcLQqFgsz3ECr8lKlmXzyVRV1TzLeIH1ArekBYWA4bjxeGyapqZpjUZDUZR3Hz2+d+s2JFTVtC++ejq+vmERipOEFfg8SWFJNgcba4S9bhpFUTi2vb6XVxUFwX/SgxSE5rhkKYOT9Prien3yWC7njVplMZ3JslyrNTYHG/du35F4odNoarIUBZ5lWWkWB3GkGKYsi+Px2PdDy6pu7+7gsiC4mI5nK9vlJAkgRAi5f/8dUVL2Dg7jOPZWS41jDQnWNZFjIaJAEAQEKLJk+e2LVzgtPM//7OnnkqEEWXJycemG0S//4efYdnZrLREKs5Wzc/tWXOSIEg6iNEiGw1EQeDnNOFlEosiKkh9EDINWq2VZllmSshTCshAZriRYUCXIM7qsyCz/T3pUyLKi1B1sNDpdhqD7t+89ee/Dja3tRq/7r/+3/+XW1la3271178FPfv7LJMvyPLcXy8P9A15Vl64ns/z4/KrVaHY7/YpZcRxHkriC4JXnV2qtd955h2PRfDi8fevWynYTXIQ4dn1H4ASJ4wEsmq0qz3N5nuU4AhBLihynaRQmWZblec4wXMVq0pJnGH40mpSAtFqtVq0NS3j69k2zXlU1g5cVxLElLRGAWZy0a01dVQ8ODkzTNAxjOhpfHp8okqgY+uvT44O9/c1ufzSbAITyOP3iHz9959ZdkaKqoOi8RCkNonC+miMOybIssFyrVm/Vqq1aTWQ5kRd91w+TyPXdJE2XriOpCoUgw7jRaS9dBwPi+B6EcLaYFaSIcRqlSRRFZVmyPHdyfnbv0cM7D259+dmvytIHTN7f7v+H//AffvnzXx3u7xq6JDKlLABZ5sPIFVi43ev84Dsf/9k/+0HpLf+r/+P/gaLcaJqEEMSA6WqiWkq33bq5Gn7+4uXW7dvf+Pjjpe0sHAeJUlFC1/Z0UV4MhxrHtitWmSemrqi6Um02DNM0zFqWl1GQKJLKU8hQAgpiyFro+hRj311JnKBIappEosTXKtWbqxtCiCwrCLHVSr1i1SRRmc1miIJmrW5Z1ng85HmWgYBloOc5PMc26lUGgTRJqoZhSLIpK08//9yyTI5BzmLR73QEkdcMDbHoYG/vcH/PMHSKYJzlZq0eFcn15FpRxWcvnm3fuS1VK0EYTyaz+WKa0/xkePmbLz7d2dt759F7SZL84te/Oru8EEVR4QRvusBRUmb50dHRWl2LMT48PKxWqwrDd2qNm/H1m7Pjs7fnztSmGJyfXa0LbhzD3tzcOI6dZ+lGp/HO3buA4d0oyQvKALTR20hxLiiyqEqOu/z6+dPR5EZUhRLind0tjjAK4PylBwqynM8Xs4kuia1KJY2iTq/z8uTNyncODw8Od3Z6zaqpCLiIvXAVph6v8EVR1Ku1xWxummaj0SjLci0M+fa3vy0Igq7rLMuuo7FlUdw6Onr99tVsOVOr1uV4CBk06PcLnCdx1Oy0Lq8vSYGzKOIonF4P7fmC5zh3uVIEUWQ5gecLSgY7W0ESz+1VSoowSYMozrKs3+0FjhvY7rsPHq6WDiCw3WgXWfHsxfNaox4EQRzHFxcXq9Wq0Wj6YeisHBYylADX9dIw7Xa7mi7neQYoUiWd5/ksS8uy0DQNFzSOU1bgMMnWhNQ0TniOQyKfF1iADA+QrKk5KRVJjn2vVquEoe/6nut762DFZDgiZRlHEc+wq9VKkiTIs6P5lBH5rCxOTk5c17176zZH4eHG5mazJbI8KUrLMJMozuJkLTy5d++eqWqT62GYJla9xgB4/+i24zj97U0oCEf37gz63S8/+51ZqVUazTBLWt12mWeopKAARUFEkS9xTnIsspzjLHCZel4ginK12SopEDgu8sNPP/liuQg8Py0wVgXYb6gCzWkegyIDtKAEJ3GInJW9v7e3ub3t+p5uqcdnbzqddlmWl+dXvhtMR+N+t/f48eN6vVmr1f75v/rnGSkYgZckSeIkxdCykgRhlGVZSQjG2FnMOQQdZ4XLAkKGYbgwiUVZjaKIljRLUp4VZE7odDo7e/u3bt9VTSvN8He+9wcffvCRrps1q1arNyfTuakbT58+TbLUcf3j07PLy0sOMQAgyrBLz/vy8y84hmF57s3x8XK5VGRxOLr2PEeUlNliMZ+Mh1fXeZ6zLLu1sWWa5lrvuTbOrO3pmqZxPAMhrVat4XCYZVmlUuv3+4oiMQwktMBFVpZlkiSKJBmaPry6dhzXrBjnV+deHC6WK0EQkjSSFQlCyCJmuVhNJ/Pl0oYMlGVZ4HiB45MsbXY7QRAkUcyz3GI2QwDUK9XxeFwUBU4zieObzSYFiBeFKIkDP7q8vAaEFnn+6sVLTVE5xLiuCxBsdltFiWVR2tnem8/nXhT4YZDneYmLXq9XbdQRQhnOddNQVHmxmLVarbUAdrVabfY3Nze3/u7v/m6xXFbrtTv37v7DL37+q09+QyFgWTYIvChNdMuEIvutb39LELh////8d/+7f/2/aVZqm4MNhmEQIFmWbe5sz137s6++fPr8WX8w+OM//mNIaJ6kplWVJGlzY03MCPIk5xGTJUkYhivH9aMQIej7oWEYzWaTUjobT9Y5X8swOJat1Wr9fvfWrVuiKAqCwIuc5zmQQbphCIKAEMrzfLFYlAgwPCdJiiRJK8dLk9zUTA6wPGI0SS6L3DT1ssiqltlqNeeTqaGoPEDfev/D41evAaE8z89mM0pplmWaokJKdVWTJMmsmN/85jdFSUIce+fe3RfPnpumOZ5OEctGcTKbzhvNNsMwaRpvb2/joji/PEtLfPf+3f3DA0JKlmWzPJ3NZs+eff382bP79+5+/eXTRrUqC6LE8yLLBZ7X7DSqjfpyaYuirGkGQiiO41ql8tlnn7GI4Vmu22kVRTFfLhiONawKx3FZlqU53j84ohDgstB1fWtr6/T87enFaZJkN9cje76Y3Ax77dbRwWHFtG6uhhenF7QE9nK1Dst7gb+9vX19fbmazkc3w6plmaY5nU4pKDmGwWlm6QYCcLVa9bu920dHB3v78+ns/PTs+Oy0v7lRlKXn+6ZpPnny5MH9+zjPG43GyrbXhlWZF+z5QpblerPp+r6iKDXTKtJMFEWI0JrO3e/35/O5Hwau77G8wAq8Kmt3bt92bYfgguM4x/N4nndd17KsOI4lSTJNUxLEJElwXkiiXGK8Wq28wP83/+bfbG5uxlFEKazXmyzLJmGkqxrHMBhjTuDX+HhVVYMgkCSJAoiLrCiKFOeCIEAIbcfRNG3doQ39gEOMKIpx6MuSFPpBkiRZlvE8HyXJ8fFxnuc8wyIKdF03K1aa56vpvGpVirKUFNkwjGa1jiiQBBECsFwuJ5OZquoYY11RNU0DAPS77bPTY45jHMdpNusY4zRJvvrqmarojuO02+0sTi5OT9IkyXB+fHJ2//59FqLVYhHHsSyra62FIouU0uF4PJ/PR6ORaZphHOGyUFRdkBSzUpO1yp2774zGEw7BDx7c9mdDhmAWllmeIgZQSiktkaoa9UZnPJvmJa7WK426dX1+cnt759bWbs2q9G8d/eSzf5w5jufYi/H4zq3DeehhHgEAKCFhXjKCDCHHEEgJSeJYZkScxLzAsCIHIEeRSDh+6bsIIIWXGYZTZE1RNF03oyyXdWN3/7C/tZ0S8pOf/+L09cnNxfVXX3z19s2bqmVtb25GQczz/M7WdqfTKQH97MsvPv30d1mOVctodNobO9tZnm9tb64LvkVRCIJwcnr6j7/55OOPv/XuRx+9PT27ubi4tbtb1SsYl6zAC6oaRnlJGIAgLooky09OLxuNhqZpaRoDAGbzKccjSUb3HxwCUBqKXFONjlULgqDd77z/rQ/Vqg4Rw3FCFEVHB4dRFCEE1+rqW4eHeZ4zAj9dzWVRrFWqJycnjuN0Oh0GocnVzccffiPwfC8MmoOuUrNG82mG88DxQUlUVe0PBq/evNZUfXfvYLVa8QKbJakoCH4QqKY22Ny4urh058vY9WVRKgBtdjt5nESO12+3izybTqcsy/a7vclobOkGoiRJorIs37v7oGG2lrPsm7/3L0ogPH/9lhXFzd3djCDAyEGSM4JKGWHmB2Pf/a//2//m73/xi9Fsfvveo6PDe3zJwhSroqxp2sr1fv27z2dxWN/sPXn3IYhTHiBFlGiRySI3vDxrN+thkkKO872wzIlhNqq1lqFqPMuu3zGAlpIkqIZOIYNLWm02ojgejkZuGMVpnuYZy3NxjlNcZBjnZeFHYZIleRozLBRVZRV4qqqWBYCC6EQxiwRVVCzZmA3HjWqtwElRpJCl8+Ws0W68evGc5BkOgl6txnEM5FFMMlbkICV5HNn20vddTVeyNK7W9M3NTrNW2xxs7u4c9NqDuwdHaRBhQPp7214Q84xwZ+tws9W9Hl8M7VFQxC9OXmVZiots6dlIZBmJP7hz2Bv0fvbTH2/2epnnKZDJfd/2XcpAiZcX07mgyKPFbOU6PM9TXMRh9IMf/KDdbu9sbSHIfv32TUoKCMhkfC0rXKNTv74ZvXl7WqlUfNdjkDgZze7euXOwuyfxWhoX3/jgw26vlaTh8xdfrVY2gKzVaLGy8uTJh4f7tw62DxqV5qtXrxXN5AW5YlSuzq9VQTncPaybNURA7Ae3jg41TWVZNggCistPf/PJzcUFLYv9w4M4TaqtRorz+WLxs5/97OTVG6aksiBaulHkeeh7LGI8x1VVVdKUtMjH0ykn8ohj7NBrb/YETb6ZjFiW3dvfvx6OSkr3Dw/TKP74g4+GJ5dVVY/CkOW53aODk8vzjBSVSuXVq1dxlGxtbI5ubiAFRsV6/fr11sZmGiWEQlnRvvnRNzzHXV96MAzD8zwoynq1JikiJ3Jpmoi8sG4gYlIyPMcARkA8QggwiJWEnb1dQCmkACKk6lro+RCXNC+KLHcch+M4Z2WnaXo9HB2fX7z7/pPZcgEBGQ6HWq0CGeZf/cH3qR/HUWRVKs5sMbo49z2v3mxcz6dulr777pNbB3cXixVFDIGo2+1G7ip07Venx53NQZmlkW2XhHQ3NgVZERnBGc/yMD7Y2b99eJtSyrIsA1mcYcOqqpphVKx6vQ5KACGT0TIhuGo1llPbDeyt/c0oCk5PT+MojxPc391u9LuD7b4iC5evX+x3OlxRFDkuypIR+BICxAvIdfyrq6uiKLzAfXv8xjRNUuLJeOh7jqIoOcbf/PjbL169SpLEXsx/+tMff+ef/WEMKBJFRhAJAbSgMidwDJ8mmOcEXJYcL0OOAxwTpznLchAyplEd9DetSq3d32x0u/VWG1C0s7svKqptu4CA4XhCIDAsk2XZe/fvI4axbXs2m52fny8Xi06z9fDBA0VTHz58eHR0tFqt2r3u3F5dXV1t72x5vi+rynQ8adYbzWZzb3fnyZMni8UqTpLexkA3tcur83UqpgSUIgARm+d5tVrleb7XHVSr1TiO6/UaBWWWJ91uFyGQpvF4PJzOxrVa1bOdn/3sZ6qqLhaz//gX/19RFDudznw+f/fRu51WNw4jSzcIKVf2EiBmtliYplmv18uyLDFuNpt5gZee40dhq96Yjidze5UDYjZqb89OEM/FabqxseHZDgNREsetVjvJM8Sxqq6nabq1tZWkqSiKkEHPnz/nGFYRRFWUZF5otVrz+byqm/Z8wQLo2o6qqv1+/580fpoucvxiPjc0PQrCNE4Gg02OEw72bimyJslqtd48OrjVbLYPj+6omimJiigoe/tH7z550t/Y2d07MmuNyWIVhIm98i+Hw1/9+pO/+eu/ffzw8Z/+6Z9ubGw0qrXID3zPUyW5xIWzsnmOyfOUZVmGZVVDVzR1Pp8nSTJfLXXTdG1ntVjOZlPbXkEIL64uWZ6Lomit/uE4Pi/w5uYmAMSyLFleh2eQqqp5jhFCtVrF8WxFU6fjqSiKsqTKkpokKaJMmmOGYezVynEcnGVR4FFaRkncarU4hvFtNwpCVVURxzIsG0URJaRWraqS3Go2ZVne2t6WJCmN4jSMGApkSZrPJrqk3L51dHh4uL29/Ud/8D1L0ZIgvDw/f/zOY8uyoihybO/169fXN6OVYwPE7uztTidz27Y3Blue51GAjk9OrscTyCBeFIMg1lQDY7y7u93ptafTqapq1Wo1zTPIoOVyWanVgijmRfHx40dFkX3yySfPnj8noMQ4e/71C1XWJsPRwe6e74Us4hr16q2Dw6zAeVFwImeYpiCKsqbu7O+tJ0Y4zer1epZlruvLgihJkiDLu/v7vV4vDSMBMl9++WW713399g3GmJYkTdOzs7OKaR0eHr7//vvz6QxSUK/Wms3m+v5EU1VKqSpKIscnSbK9vV2zKmVZnp+fMwwDIfz9737n1sFh5Aee50VRVBTFfD6Pk2Q0nai6lmFcFAWl8P/6f/m/3To80lWDFqUgCH/yJ3+ysbUVxjGl1DIMBqEoivr9rqnrBBd5nrdaLQApi5h/9+/+3V/+5V/ruo4Qm+e5JAuB54uidH5+4bj2eDxqNBoUgrzAaZoCQAWOlUUpjuN1Ki+Koul0mqaprusEAoCQoihljjXTUA1dEIRuu93r9dZ392tOsqUbqKQQkCCOV567Wq2iwI/DaHQzJqTY3t4WRHE4HkOOabXbhllZrWxQAlpQnmGjMFysFrwoyIq4FiyvVitekper1Wy6yLLs7OwMZ6kkKoIgqIZ+cOvg5nrEMJzv+Iaur83JrXrDXixny0W73d7a3qk3m6ZpzmaTdrtdq1U830EILGfzt69eywIvC5CmQb9hsgxDMKlWa5puEgIIoOzG1qYXuZLGFUVFECR3ZTMM43j2H/7zP/zVbz61Vx63dwhI8eE3fs+eT4DISNXKYmm//fJpx6rCJA6zhLJsnhWIFbwoUUTZyzLAQViULM/xPNerNYuiAAwiLEsBioqi02pD3/c8P5kvOq3WfD6vV+qWZRUUrGx74TuqZRyfn7WbLVlP3333UeT5L2ez/s7Wwl5VdP3BnTsQUgKooelfvXimVq2iyPv9jd2tXc/zdnb2HC/AtLQDR1JEhW+vfIdykJN5kOUMJbopZVkapwkuyas3L3u9Xprn4+mI46EgisPhUFW0OMp1pdKqtxaLxWB707ZtzdQ4ga3WdNcP7HTea3WKvEQUwoLAsjjc353Pl1Ga5GUpSZLveqEfoD66urmhEPb3tr/6/EtE6Ob21unwGrMwSGJBkur1ehLFs9mEgWhyPT7cO5zMzzNaKIY+n04pgRhjIPAze/H+++8BSmmS6/WmIau+76dhZCjqeDg2VM2U1CgIKS3jOCzSROV5L8+zNEGUyKosSOJoNKw3ZU0R0jgVEduoNWVVeRu82r51N8GFrloix/IsRwhwV7ZlVcMoGV/c9Pv9y5ubKArqreb7H387jeJ+r6sStqtVT4/PHj165Pl+f9AzNJWDiJSAQWiz1357/EJQu8PZSFYZWRKq9ep4PDZ1VVPlMJRyMed5vt/tnV1dFXnhuY4kChiXSZQgSOPABwTiLG/2a0kQaLKG0ywtcJgmPMfFfrizs/Pq5ZvFfA4BUFWVskirWopvq5qGMdZVQ5G1sqT2cmVpehzHgGchgRfDa5bjEISGohqaPrq8blZqeZKmWeZHsSxrPJJ4hN3lkqGlwECcJwzHRkHAMoByQt3QBUluNpuv3rxt6c2PHn50enq65knEUfTq1atOt1sURSnKo+lUUGS/yL++OKnX64d37iyX9tn5NcsiUOAkCMKylFRlNJsyDNNqNXFZuGE4nMwf3XvIQPgXP/yPLMt2exuEQsoUosTlIevbjq7zFORZiAlJKhXZ930/Ea6uh4IgWNWKqCoFJdPpqFGr6roas+jtyZtWqxm7vqVpSZGP59Ozq4vd7a3dTt/3fatenazmcZouFovlcnl466gsyyAIkiTJkrSmGwJEMmSxH0VpshbspVmW5qkgi+c3l5fX1wgCjuMQALqsxlr8y5//4mCw1e/2ckIkTpA4FuiqFwd3Hty9vrpRFCVPMwJApVlPSVEAOpnMDg9u/fRv/v7i7Kzd7XY6nZfPn1uGVa/XRvPZ6+O3j+49+vLLL1fOslqvC4JYr1T73d6nv/vHKIpERWQRo2naeL54/4MPNFX59LPfNtutMEkEWS7TTOa5OI4rlaqcZVEcxkmsSXKe54jjHM+FiHAsDxlUIlDSMsgSmZDZZBoXhSzLqqAUaSYLcrNSi/0gijlBklJdHS1mgirrijydzm+998SPg7wsFE1dXdsY488++0yRlHt37tmLRQnKq5tLRlIIQgKFgJLxfCHo5vnFBaX0gycfxnHMSeLCcxaf/GYwGJhNMwgCluca9Va91k6yeGdzw7FtS61oin6vUc/y7MvnX4d+0KlYkRv2+htB6DEApFGiMTxkmL39rdnZ16l9EocrQwTdzgCRkhQgTzJW4NB4Nj44OKhWq4vFajVfuK4LIbx9987pxfmdu4f/+Z//abdZZyD64Q9/eHlzLQvicjT5z/70f7R/dD8sKKawVm8WBPCiwEsiZdikpJJudbpbilrZP7o12NycL512d9DpDoxao9UbIF6c225WlnlRiKLoOE4QBL7vQ5a5uLwUNanTaXme53oBxiXJcBYnrVbr9u3bDMPkeVqWWJbF8Wxq1eosz1WrVc9ZXV9eCqL46uRtURQ///nPmvV6u94cXt8kSVJtNT745rf8KGRZluM4Qsh0Ou71OmdnZ0mSbGxsQAjXgkeWZYsy5zgmCP2Nwea62NVutyezuSiKQeA5q2UURPPxjBRFo1p7+vTpX//Vf6pUKpZlpXFyfnpyeXnJMFyj1hheX8dpYlYqlNJ1+wMgGEURy7JBFK4cm4GIFMXaNVpQ4ofBu48ejW6Ggeepqjqfz13XZVm20Wg49lIUuLUTcjabXVyeBVG4XC4lThA4HiFk1arX19eyLDMQqaoaxzFimXq9vm73pVFcq1SCwBsNrwnNwzAwTVNRFGdls4gxNL2iGSLDHe7u7Wxt4SQ1dH17e7fX6x3ePty7td/eHHz83e99+NFHjUZjHW5xl6u3L1/t7R7wgiCpyvnlpeM4ZYktQ5Nl+dXrZwwiLIuqVYsB1A9cCGEQeIIsBIEnyfLB0WGSJBzHMRxLIdAMvVqtKoqUJAnGuNXqQAh1VcuTVGC549dvIISSJPlRCADAGMdxbJiau1rKsjieTEaTMSEkznLbdrMMA4ogZKIoGmxuJFkWJYkgiaqhG6bZarU4jlutVkmSsAglUZSnmed5sixnSS7LssQLnU4nCDzfdWVRdBxnPh0LLGdq+mgyTtOU5XmCC1010ihlEArD+OjoSODEerV2tH/UbDTm04VhGJV6Y2Yv945u7Rzuv3jx6sc//vFqsTw9PpvPl19//SwKopKSMElb3Y4oy+Px2A2Dg6N90zBmk+nW5uDu3bssJ5RlWa9UCSEQwrzALMd5npfnebPRKHO8tbWJMe602u/cf1DiotNuijwbBd7W1sZiNn314vnu3g7DID/056tlWZCClLv7+3GaBUk6my8HmxuCJLY67fv371cqlTROut2uJEkMw5S4EHned90kjjVNEwQhy7JKpRJF0WK1EgRhZ2s7iqK1YLYoip2dnSSI5vN5mmeNVvPq4tJ33Xa7nRU4DEOMMYLw5cuXEELDMM8uLqMk629sjqfTJEmSJOEYtmKarm0fHh4OBgNVVdMobtUbiqJUq9UoTRADIKHVanU8GcqKkmUJz7BxHLM8J0nS0rH9MIAQbmxvFWW5XC7XbpxWo13kOM+yTqtNinJ9bljf58iynGWZHwSEUoyxIAhhFEEIi6JgWZZlmFa3QxAcLWa8LMmKcnF2qUmaoiuyKs1mC4RQmsaXl5fL5TIKwu3BRh6n4/FQFEXP81iWlUWx3W7v7R8eHN2t1eo8wz5+94mm681ms1qtBkFwcXVZaze7mwOA4HB88+lvP3vx4oXjOM+fv5xMJr7vf/LJJ/1+P0vS7Y3NVrUucny1Yn3w4fsQUsMwXr14HYcJxpjj2CgKIKUnb96Or6+6jYop87qqrJ8jgov1l0W4SM/PT5Mgqxo11dC3drdl3ZjOlxxiOADbnVqraTWrNVMxLav6s5/+dH58+sU//PJ/8C/+1be+94OQ4U9mS05TMSi8JJAMrdHvGGZ1ObM/fPKNb3z7O7cePnz8/gdBGCdZebB/ez5b7h8cZUXJCmIQBI7j6KapGUatWTs9P0myOI3ik1dvRMSyiPvtb367uzHY6LQhhKKiQoqWs/kXz758e3UqGcYvP/nNzWzS6Xd69dbvffhhwYMI5Dezmev6X/7u04vTN+/cuUdK8NnTL3/yi5/yDIvTDCBUUmBVm0vbx2VJAMiybLVaQchYVnUynjmO0+12IaTD0UWn08iybDweW7q1Wqx6ndZ4ckNL8v6TD1eL2dfPnpYUIFFMCK61617gPn78+Gj/QOT4+XzueN7vffvbcRy7rnv39u35eKpwwrsPHuI4xVFytL1X083p9VCAjKYofhAIipwXRNfNwHU2e904jl0/qJhmFHiRbXebDcOwVq4nasru4UGYJZPZNAljy7AwIHPXHuxtrxw7jqISF8+OX848p9qoMwyDSSmp8mI2EVmOAbzAK14UpGVWMdUSZ4ahsSzKkoClhbNa1mqmpPNqVSpgGqSeKHEcCwnNZvMhKPPY93GRsyJv+97W3i5ANEkSx3MVQ6u0GhfXF4iDLAt93xc4XuP5xAvWmnZFFiGiru/plvnyzaulvaQMent2us5Mrxx3NJ5cXl9aVWsynsVRCiEtyjyKYoETOq22rqihH5ma6QcRLwq8yJe0bHZaFBSUIfV6FRVEZritnT3NqqZZcX52VWu1V57rhL5m6KIo3tzcqLJWFCRNcs2wSgriNL+6GTEc32m0Y9cXOOR59vHFKYZUUtRufwMCRuQlgRMJARmlgqaxPB94nqDwZZkrioizDEHqOrYXuJSSTrNhaRoHwbsPH945PJA4brff32x3dFn6zu996/7tw8cPH3R6g+/8wR/FUX52cv7s1ctXJ2+/fPo0ThPEIjfycZm02vV+Z/P4zYlhaE+evCvzyny8SIpMr1WSktp+RGj2j//465UTPPv6dd0ye+06SZPtTkcowPx61Gw2l8tlFiV3j27Tkp5fXHY2Nt775jfCMHbmK3dpK5r+m6dfBKCs1+uTm1GR4/VfoWVZLIA8YpIgxBhLijKaTIIsETUlSmKEEI+QwHMAQUESkyDs1ZtxHGuGLqrS85fPDvd3dVV+c3Yyd1a7+3teEMi61uy0Z7NZkWOe5UI/iuO0Xq/fvXs3SZIgCNrd3mQ51y2z1+5Mroe1igUpGI3HfhjcOTyIPDeKIo7n7cW8ahrT6RghtL2zw0six3GIYwmgSZKoqmzb9qeffZ7g8oc//KHnOqoklWUJABNFsSwKPEJJECqSGiWpoMiQY0tKcJZnSSqJosDzMi/AggiCUJByc7ARBaGiqseX55eLScrQkOKCgbIs8wzrTmfTmxGAsN3rHp+d6oaqiFKzWus1WiJAh7t7l5cXkiQyDPPsxSvHCabDibd0333yzVp70Ov0K7oRx/Hewb5uyKLEnp6fRUlsVLTNzYGlmSIjZVE8GPSa7cb+raP+5k6S5oomq5qoKco7t27d29355Gd/P5wMEcfWKvXIT2+Go+PLi6DAS98djUYKLzJlyZIC4kRkKcuCAuCCYlzmbL1hzcdjQKBhaoCWgetVGh2GYTiGKYvi09/+1nW8TntrMBgcX5w+uvfw7RdfBsBdDTazLPvBH/2z49cvZIqzKESywKuaIMjT0fR/8r3v9zYGF5Pxq+OTmmZAlkuTfD5fdjq9F89e6rrquZ4oK0WezWYLjDPH80VRphTKksS3Wr7rFVnx+NE7o9FIlmXLql/f3ECW2d/fd75wOs3Wy9cnlNIgDuLLQIDMarnsdDslIRXZitzw8vyEn/P33nlIcqyq8mI+lhkGkzLHRZZEnCC4rssipMpiHEeapq0dcv1eL4qi8eiGkKJSrfiBm8Th1mBXEcXq3h4uko2Njdu37/76N//YrDcYTnh1fCqJvMhrYRhKVXG2mItQUCWxUa257fZiNqs1Gook28uVqWqyIC5Xcy/2McYKo67jAYok1+vV8+srVVUlRRY4vmoaPMd1t7vL2QJCuKaaJEmSptnwZrS10WNZBFioV6zNjY3ldHZxdvbg4Tvj6cx23X6/v7W1ZQceh5jL6ytNUReLRbvdWndxD/aOVFVjGKZRr7uum6WxKIqqJpdFvljMmq16miWWoQuyABFdzMeirLSb9apl4Cz37FWapms2vdJsXF9dJXlmVisAgEajURR5rVHPgzRN0zTJwygkhOCySNOcYZjJcMRAtt5oEQL2D3Z9P1x5TlZiUxSjOCzLsqAlAOTs7GRNCmN5BkKIMYYQuq5bUrVasXw/pJTKsjwfT5Ik4RjoJomiqqZpXh+fFVkZhqHn+81ON0/S+XwuiqIuK2maAgB2d3d9143jWBCENE1FnpVlsVqtZklMCkzLQpbFLE92dnZmkyli2ZuLi5pVJYT0Ol2c5dz/H8DCsqyiKCenb7vtJi1x1TA4FtUqlSLPj0/eYow5nl17aNv15sXp6UcffYSzzNC0drvteoEUKTzP37p96DjNfXqom8bxi1eiIHIc/+nv/rFpVg/293/2s5/V6w1SFi+eP7eXjuu6kiLMl4sky7utNiUFLou3ZxfNZvN3n35e5OmdO/copcOnTwVJXHezYQnfvHmzs7dnGMbVcBQEUUEJhMzW5k611fCiWFdlAGC33QaE0LJUVe358xcfPXlf4PjxcNRs1ktaPPngPc/xa7WaosiOY2u82Ol03DDAMu73+0mShEmsaNr19XWn24qC2LU9UZb6/f7JyUmUJEd3jyzLWs5XqqwmfnS4t+v7fq1WK3Ps2Pbm44eX19f+alWr1TAp1zqzerPx4s1r3TJc1ycElISUgB6fnR8e3hLFKccxaZIYmhp6vsAyiGUVRUmSWJKkdf7KggZCaC1Rms8XDIBFqfIsm8YJKQtR5CmlLEQ8x0WhI8tymuMsScsMI8AomgpEyfFsw9KHkzHLcRkkkqpMFnNvZbcaTc92ZEGs1Rt+TlzXl2UxCAJV0Z8+/fpwd0dWlJOTt5ppLdwlLcr79+9rmjYajaIo+uKrp6ZpfvbZp3GaWJY1vL4xTEWUhTDFoR9Mrq6WqsYg4fGjR6vF7NmzZ07oV9vt3d3dsizbzUYaRl998WWr0bwe3pi6EWahHzhVo1XkOYRQlmXfdzuNenuzH16+aNV1hcVFFrCyBhhKaQkQBYCg49evqxXj4d07piDfPTg6OjiaLVZnF1fOynWW9snxJaHcy9cv/+7Hf5uEUZGXzVrrz//0zz588rjEoQjLbz558tE3fu/Bux/85//j/+L9J9/c2d3/V3/yp3FanJxeJll2+/ZtVVUfvvO4Xq/ziIk819JUkeUsTSclQIhlCNIkNYmz2XRhKsbF8elkMuF5XlPFLEsup+O/+vGPf/fb39atymQymoyHuiD785XG8/du3dZV1bDMAoEoyxfDWU21JpPJ5ehq484tN0s/+cVv5tc3w8uLxXxGIEEMAEWma1JRptWacfvWflnkkiQhjvVCj5Ai8HwWMQLHy5JES6ApesW0siS4vjx7+errOIyvzq9fv33TG3QXiwUHQati6jwH00xAvGaYqmWwEitKgshyMCdbvcHw8qqiG4c7+3mez+z5zp2Dhb9yI4/huIJSRhA4gV3Mp0USL8YjjmMmyzFhoCTLz589k0VFlRU/DCiDSogUWTN1y54tCozXXARRFBlC7+7taazQb3ckSU7y7HJ0c3V5Q0rYaDR0XTc13bIqcY4By+Ulvrm4wEnGEqCpimHovusamjZfzjmBgxAsZnNntqBxBgtiamazUtNEGQEICzLoDizdEnmxYlUxLgFkrFoVlwUpSlhSiEuGAMggQVFlWdX0SgIRq6qaZaZpKgiSvXLLknIcp0uKJHCKppmVShonRZo3mlVB5CqW0Wk3IaIQUVWRRI63LBPj3KoYDIsIoHEWP3hwL0miq6urdQ1NlWRZkobDsaLqAIA0TU3DWEwnkiRUDVMVpTyOVEW6Hl3Hcbzd3xAQS3FuqkqeJUWe4iwa9FppFqd5AllG10yS5nXDUlW1Xq+maTLodbIk8lZLmqT9egNCSBAMvbBTb6qyUjV0ivM8CPrtliRwnMjpFUOrGnmRuZ7Ns0gSBEkQWJbNCyzK/HwxGk+uWKbMsjiOg8cPHhiC9Oj+/fcePtruDQ4Gu3cOb2/0Nu7ffyBJ0uMH7+xsbVXqle//0fcfP3zcqNTfe/SeLKtX1yOjWq+0Wmc3N/VGW9Yqz45P/vGrrwuWCbJsOp2naS5IPOSgrKsPHj2kRVnkeGtry6pVoyQmhOiCwAOURHFZlnXLXC3m1Upze+vQsT1VkkWedV17Y7Of5UmRJoNOu1Wrup4tG4og8avFAhA6WU6fHb+s1CxFklRZHt+MWY5vNBoSy5uKVjErjVptNV9AQjVFOz093T/Y0w0NMoDhUJ5lFJDh6Lrb7xiqjvOiN+gjgfHDKMmyJEufvXhebbbygpycnx0cHW5v7S5tVxTFyWTSbrbu3r5TqxhR4LIAcgBpmk4I4RmWg8iZLUiaQ0pnsxnDMazKxwRv7m/naczQIs+iEmdZnOE4lwU5TXOcFTwvCqIcJkmQpIhlvMDjBE5WJcSAQb2R2A6PmK2tLc/zBEFw0zTBRGBYVBSKqOCyDIKo1e7mCDCqxIgsYECEswhnh4f7vruME7+k+XB8tfLn1VZNM7VWo53EcZZlg8HAMkycYpWTyrRgETg7OymK4t6DB/3eBgLw+PWbZ0+//Iu//KvZym61Wlc31/V2T1DUXqveqCqKzNerFYXj+/UmkyXebHx+/LWpMCDPYJ5rioQJZlia5REpyhIXSOI5hgJdliqW0W60OIY1TfPevXuVSqXE5Pc++ti13XcfP9zb3b5/5zaHGMixf/GXP7w4Pfnet751tLurydLxyQmh8OWrY4Q4XdJePXuOy8L3/dD1fNsBBF5cXGxsbEzn8zUNznNcx3EYhsEF8TwvS3EYhizPxXHcbfce3XuAkzj0fFmWAcPu7O9BCBezqcQLi9n86vyqLEvf9S7PTve3dtZsiqIoqpr15tmrJIy29nZPzk6TLBVFfmujX61Z7XaTAFoQoCgKLQnPsKAkHMft7OwghEzNlCRJluUwjBFFUZQokuq5QZ4Vs9ni8vKa4Zm8zBVVb3V6vCAAAA4PD+fz+RpMapnm9ubWeDgiRRkn4Xf+4DvL5dILfACALIuB57MMohDKujFbLniRe3D3TsXUZVlerVZ5WUiymib5g/sPS1yMx9O1rrPZbDIMI8rS0rFn80m323adFYLUMLWyxLjIMM7W3VrHcVrN+mK+0jSD53mO41r1ZqNe39zcfHvyxjAMUBLfDziOZxFiGAYBgBASOH54fdPptipVk5ZE13WGYSzDbNYbg94GyTGkoN1sIQBnwzHG2HEcx3FqViWN4iLHy/kCQZjneZomEMIsw5eX141aPUsS13Xb7bZmVLworjUatUbD8fxmu2UZJiD0q6dPJV5gESJFIXC8qatpmna7bVkQI8+vGnqv3UIIxVmKCYYMGo9HhqarqpyEkb1aybxwdLAv8iyLECno+GbMsqxZsTiBn0wmEAKKYI6zdruZpfFkNK4YZrfbv76+vr65RJDyiLGXc0PVZFmMouji4gKxsFKvzGYzXVXLHOdJGvpuHMcQ0dVi2Wm1q1ZF5oU0jtfLdQ1w9hynLEtdlgPHU0UhiUNCiGboGOOSkr29vbIsG7W653mUUtu20zixLAMnceh7VsVoN+uXp2e1SmU6npyfng06vVa9mSYZw/GmaQqCgDHGGNfqVQBprVY7OjriOWZ7e3v/4KjX3/jzP//z7373u4jj/+Wf/A9v37l3+97dD7/xzcfvPXlw/50PP/zwwcNHh4e3ACGz8WRr0NdUtSzLSs2Ko0BkmWatFvo+IWRra8v3wqKgtVrtwYN7BFBMSG+wGYZx4IWB54/GN8+fP8+ybN3ydRwHQkqKfGNjo9FoxHF69+5djLGmaZIoAgBt2zk7ORt0e57j5lFGMJlMJpDQ0c31bDL2Xde1nUql4jlus9mURHE4HBJCyrKs1Gvz+fzo6EjX9QcPHmiK2u90t7e3RF6ABCwms1q1qinyydvjxXzarFUrhqlIkqootm3X6/X1FXO9WmUo8Jf2zsYmgCTPc8CA86tLyCFG4EVZEhVZlWRZlDDGURSwLMpwuqbDiqIYh4kmK5PhDca5yPOkLKbDUegHsRdEnp+lac2qpWkKKYAUjEajLMO6rgdxlCSZZVmVWjWIowznkGd/87tPoixNsrgkGCJq2/br1y8JIYvpUJNEFnGnJ+dxECZRTAjJ83xzc1OSpJOTkzzPP/roI9M0D/cPPvrgG4PBYDQez1dLRVE2t7e3t3fzNC3zcmkvkiyWZaXEGJakpqs7/RYskkG7BkEBIYQMCygiJVAURZIkxOTUUjQEYLvd5iWRFwXdNLIskySl3er+9V/9ZdUwSIY1Qbo6v/jy66+oyECR/+1vfgtzKrAcxrjdaZpV0zStszen4cw+7G90u+00jxpmBeJSU1QKwenNpZ/FlVbj9OJckMR2u52W2GrU7j58wEoCARRjHMSRJHCvnj791vsfRlE2t72Nja2vvvrK8Vcnp28lji/zEoniaDXfPNgGDJhPZ/cO7rCUUVjJX6y6rbYgC89fPHUXUxEBXGQ30yGlhOW5NC+KkoQR5ng5jTNJkN1V8Otf/CPHiK7trNu6jWZn5fgIsoIoJ1k6mU11o2JWal6cVNvt6/GkBDBJsuFwPJ8vLasqKSoBEDAsIFBnhG6l7rnBL379m8vpVK1WncQfTYbbmwPPc9I0AxDJsr5cuMvxlKP0/PQtz7IbvY0giBAvFITKgrLT3SQFtVerlW3bgRcXeVbgW7duCQwSWTZL4iSLJV0BhJIcN5vNpMgJgn4cEYbmRdZptQXEKqIUe8F8Prcsy7IswCCB4ziGgRQQQnJSyrpqu061Wi0L6iwdQ1EpLizLGo5HjucGQVAUZDqeYYyTJKlWq/1+X9AUUVXCIGAoYCgZdNqHO3siw1nVCkAwzHOGE9IwyoKg325NRuPQdos4e/XiNQFg4iwwKFRVzuOkqlcMQdU4sYhTVRYJBH4QzBerPMtEji/jTOYFQZCms1le4HavvdabFGnWbbbCpc8Q5LvuoNcFJczidHtzu1qtAkiyImu22yzHE0AIJEmeFGW5f3hku37gRzwnGoZGaSlJAkIIEOh5kSAoVq0uisK6/GKaJuJYVuDX6jtZlgkhCKGiKBDPKZomcbwuK3meJ1kqSJKqqmEYttvtJMkQRZDQOIxoSXzX8x233Wy59pIUuN1spnHCQaTycr/dYQG0VP3m6jpLUoHjBUHgOK4gBUVgvlpmOCeARkns+z4FpW3bCKEg8OM4Mk1D05Rq1RJF/vz4LUtIHIfHx2/6vU6Z5cOLmzRIGMjGYRyHse/6ke2nfriaL+bzKUJAkSSWlM5kXKvoVsUo8gwyLIbC0okYWP79j/46ShNZN5auy3BiGmaGUhFVDQlctVqHBDKAsSwrTWOGgYHtCoh/+eLVZ7/7vNNpFUVeFJhlWcOqspwQh2GrWp+NJ4amP3n38WI5J5j02j2OYSAllJJut/v8q+eQwP2DveVqkaZpnuQcwzqr1b07txkEaJbJLHv2+tV8PJYZtl9riAw7uR7yiGnXGgzDSJLEsyykVGa5NI4qzdrStVVVpXlhiMr16TnNC11Vq4ruLFdI5DNEC8QFSerHUUGwIoudVtMPPQLKElCeF2le+kubzcp+tZEEPs8xQRA0Gg1DkVVR4ADbrjRphkGBCSkUTU1yXKnUbHsZBU4eRCcv3xZ5ySEGggLjJEd05i4wKTRFaTXalmY2q40iipoVneRp4AatRpcU1Pcc1VDf/fA9VVVZhLY2NnVD/bsf/0jR1CIvSI4tw8zSOMuysqTOYh46niSYs7mHKSgZplprCJI62Nj0/XB2NbQkGRR5nqUEsByvAiBTwnOcRClEf/Znf/Z73/742ctXl8PRmzdvpuOJxPJ5kg5Ho19/8pt3Hj3s9Xp/+zd/c3F6BiEcbPYpA5989MGDBw8s0wyCYGtzs9vpVyq1m+srnMRxFGUZvro8z5P04uR4Np58/uUXVqNGCFnTQliOw6QM04RhGJZlZ/O5YZqYlBTBdru9cp1Bt2fpxse///sHh4fr7PbSXsV5JiryrVt3Xr55jRC6d+/eZDgqs+L0+Hi1WJq6atv261cvJsMRjxhQkjROCKUM4hRF9RyfECBJSpJllFKG4QRBspfLb370ESCUEEAIWNN8arVavdnMcN7q9HSzUhTF9XDISoIThHrVEkUxSRJJkhiOrdZrcZrqpqmqqm3bRZ4LDMux7M7WNmJAQTAFSFRUPwyLoggCX5LFIAh6nS6gFAJACClpsYbwIQBYlp3P5+fnF4c7e+1aI8syw9IRyyVRev72TBWU5XKZpunW5g4o6XQ0rlarV1eXSZKouqZoahRF0/FkMplEUbRYLCRJWvPwrq+vaVFGUaQoimVZawTgGi4my3IURevmV7/bc20nieJms6koCmSYra2tNY+eBZBlGEEQ4jiu1euiKNq2DQAYX994rpMl6Xrvo+u6JEmU0jTO2vXGGjvc7bYppZCC+WyZ5zmEcLM/iMPIXdklLmhJGALyOBl0uoPBwLZtCoEsy0EQtFqtTqeT5hmFIMnSOM0dLyiKwjTNOI4ns7mu64SUiGHCMLweDnXToGXJMijPc4RQ6PmObduuy/Mix3G9Xg9xLMOxSODystBNQ5ZlyDCu61IKPMfBcRqGYV7g+XJR4sJz3MVioerabDbzwyDLsizLiqJwbWd9rnr79i3P82bFWi6Xy+Uyz3OO4zRVFQWB5ziB44MgWHNJsyyrmGYaJyIvVHQji+LA8wxN6/f7AABN0wCChBCGYQaDQRRFPM/3et0Up91ud83YqVQqHMfN5/Nur8MyTOQHuqSostRoNBRFzqOkYVWrhl6tmFHo53nOQFg1DASBZZqDwaBeqQIAep2OxPFFjjmWZQBVFAWUBCDIi4Ikcnt7G65rAwjv339nOpq2a62t/sCxPQAhwzCSwLu20+t033vvvZOTk16ngyhot9uarrIs3+l0PNvRZLneqGKKcVmkabpYLDDGrmt3u+3pYq7o2u72jud5nMALkuh5HgCAYWCaxoZhMBARQqbjUeD7x69fzefTOAkNQ3PtpSBwV1dXk/G4VqlqinR9fU3KkpRYkUSZF0SWK9I8dD1FkiaTCS6LOE1YniuKYo3vN02TFXhcEkph4Pqqri0dm5AiTSJKSo5hkzQtAS0KglN8c3H56uvnsCxCzwclgRSIPCfwPMMwBSl5lmEgStJIELiaYc1HE0nkLcMIAi/yg5NXx9VKBVEKKJnNJqzACyyf5zjHeDQe4zQjZXl6epoXeGGvbNvZ2dquVCpJml5eX5xdns2Xy8PD/eHV9b1790Y3w3fu3ec4juSYAfDx48dbOztv356MRqOzs4vd3QOO4wqcbe/t9/ob1+PZ7dt3jza3ZUDzLJEUEUJYZAUlhGXZ9cEI/Vf/9r/+f/x3//7xtz6Ki/z49MTSLR6A0dV1hnNRVyerxW+//LzZ7TAC398e7OzuAoqm06lsKF+++CqJgq8+++LNq5O//5u/I1GsqVLBwOFifvbm/Pb2AYRwa3fLbFY831/NFyLDKZIMGQQYFOOs3mxEUVStVFiGqVarCKHTs7Or6+Htd9755W9/+/zNs9OzN0kS1Sr1WrO7cPwoy3lJ/Od/+IMiTv/2h3/1/rvvz6bzm5ubLI+Wzlw0pJVnm5puz1ajySLCNMOgv7ETrMJg6TIFkUVJleU4ioIgCDzfXi6c1bLRasZpIvAiBAghSEERp3Gcprpu+H5AIX3vw/cRw2V5wQtSQYClGFWjGiVJgnNN08u84Bi21qhv7OxOlyuCSewHpiD9/je+SSiIknxra3c+Xy6Ws73dzTjy0jTdP7iNCSogOrhzCwASuqt21bI0WTE1Lw5mo6HEMAXOcZarkgwp3JeDLn0AAQAASURBVOntqIKWFkWn1y9TGtupomhGxZJVxQ8DyzBLXER+UK9Ua9XqcrksKeEVyfX9six3d3YkUeQ4riQYMojn+TSMhpfXLMtO57P7d+8Gnj+3V2GaFEUpC5IsyhwvpHmWk6Ld7DAUeStbZDgeoNAP4ix1w2DtV+E4NooiVZSW0wnNkhJnBDGybgS2Z8ja1fCa5VnDMmhZNvUKUwKIOFk35oupZqpms45Y2K5WLUGsyMZqvJiPJp1+hwjo7fUFKUHkhThPb64vAUIEIcUyJUMjDFm4S8kwJcNcek6SJ7bnZBgrqhEEUZrFs+kYAoLTjGKys7mDAIAMYDk0nU+Wrley7PnNMMrS8/Pzdrvperau66iEuqTVq1VS4CAOMMGKJBVpBgCo1WqiLImiKEnSxsYGIUWlYjIMRAjduXNHkpTpdK7qBsuyoihmSUqLMvKDIskwxggAjDHHcYoiYZwhADiG4VlunfATBGHtts4LjDH2fZ9SOp/PPMfptFqnxydRFLEc50f+zc2VLkuLyRgJzGdffmYoCsU4TiJdV2lZOKuFLIkFTnkOqbKgaNJ0dmNpKkMJQiBJ4sVikSRJmeXLyTyKM0FUfC/mWen68ophkVVR4shezscspaYmhoFrL2cH+zurxTRP0z/8/veTOJY41KgYZRLNx5P5dGWatTCOCSQlyafTMcdw52/P4yCURSkMvYIUBMKt/T1G5EfzMccxURRo1cpoMX/15nVR5BzHFIS0u72CkK2tLUWRXrx+0el1RZ7NkoTmRaNeXzirCCdGxai3G4IhI4U/unsEGJphLMtyvVI1Fa1MMpFlBI6xFBmlmURBxTSQzCWoCIqEk8UM49FymVMKijJxfH+2bFZqqqr+014hjvmSFHGCywKwjCBwg173nYcPNUNvNBosw5A0L+KURUxBCk6XnDSSddWsGATnV2fnLCYMxkkQlGWZ5NnmzrbE8den51mczEZDETGGqIxuxtPp3E+CIA7iyM/zDAM2zIt6o1EU2dX1GaIk9KMXz1+xIvfNjz9MkoSBbOhHs9lsNh+/efXSmdu3D255vh/GUb8/YBiut9U3LG2717MUZTWf/fhnP17F0avjtzenx1VJQiyjGCYsc13hcRZyDMFpggBBd965v7G9bdv2ekXe3Nw8e/bs3XffPTw8PLh1BBiwu79z5/6dEtCbq+uL8/ONjY350s5wniSJUTEfPXp0uH+w0ekvF7PBoAd55sXb1wTBZ69e/uAHP9je3uZZjhBSr9fXVA1VVtrtdrvRpCUROX48Hruuq0pyvVINfH97e/s//f2PMQK37t157/0nYRjatn337v3tnb1PP/30F7/4h9lw7C5WqqKHUeK6bpIkRZFfD6/H0zGFgBQ5D5mD7X0OoDcvXy1nC1VVN7d3MC7zPI+SBDCI53k/cE3TnC8XcRxWKmaWp4apIwbEcdxs1iml8+W83qxzIqeoUr1eX+fHu+12kWPXthFCi8VCk5WyLHlRGI5Hx+dngiQNBoMsjO3VInC9NM6iIMZFsbe3pyjSYrGIwsT3/ZvRKMlSTVdc1xUVOYqixWIGALgeDuvNZr+/MZlNIaRmxSwKIopykuWKrgEAXN+TJKmk1LKqoihO5/M4jkVZiqJoPps2G3WB5VRVNSsmhRQgqGmGyHH2YlFvVHv9/ng8TuOk3+01qzWMMaV0jazhBF43DZ7nHccJXK/A2PU9URSXy6Xveoqm6bo+m80YhlF1Lc0zjHHFstYHZFHkDUOLoojSUpKksiTr2aYoimvcR6fTSdN0Y2OD44STk9M0yUVRtJdLURRX9kIQuOVyuUapLJfLtZUmy3LdtPI8tyyrKIo8z+eLBUDQrFaiNEEcl2RZQcperzfodSRJkiQpieKKZnRbTV1RRV5gEIcQIgCsV12n3aYEQsRqhtkfbBqG5vt+s9VyPU/XdQBQURSWZfE8v7m5ubTtg6PD/d29k5OTKIoEWVo7sJr1+hoj2ul0gijMi4Lh2JevX5mmORgMsiShJTncPyiKgkWoVqvVarUkSdYkxbwodF23XWcdfDq6fXs6nZ5fXqzJ9QghnmVZhEqMEQCdVmttfWpUawxEcRwfHR1hjDc3NzHGEi9ULRNjLMhCEASVipnG8WQyQRySZMGyrCDyVUPNcM4J7GCjv0aTtlqtSr0WJvH+/j4hJI4ThmFEkWMFaBgGy3N+EPACKwiCpmnj6cgP3MV8GUdpliUQQsig8XhsavpsMp1O581ms1arsSxbqVT6/X5W4NliXqvVJpNRFEXrXjEDoMDzHMeRorAM88G9+zjLXddtNBqIApHnXdcuy3JvewdjrGhaGMe1esW2bS/2924dnp6evn7zkpPFqEiWy+X29natUecE/urqav3EhXHE8rzjeVmSrs2raZaJilxr1CVJgpTqquq7XhLF3Va73WylUezajq7reZ5HUSRJksBx6x8/TqLpfFJAatVrS8fNs2KdjLq6usIl+Zf/8l/yInd2eZEkCUXQNHXIQqNiJFl6cXHBssx8MmZZ1G40ZUFqVZuUkNPj41sHhwzD1Ov1o9tHuCx4QeR4UZKU0fA6cGzXXsZxnBdY0TVBEi9vrqbTqaIow+trSeRPj0/q1eruzk4SZ5eXl0kSrR+ozcEg8F1nuQo9/yc//rtOp9PptcPINUQgs0VZZIAUDAQIlogpWA4UZV4UBQKYnL5888Unv4UpPjq8XWs3E4ILQNM44SgEeSGw3On5WRj6EsfeOTzodruHh4c//9kvOIavN5qv3r6yl/Nbtw+8KP6HX/7qs9/9zjSNhC3Fuv7m9MRQ1E6toYoSgSTKY6taIWVZJlnqh8vJjBDS6XY73e715ZWzXO3v7AZBYNatZrf5+uT03/+H/9fNeCar2otnz28uzhEiq9XCWSzfuf/4+macFkWJKCuwXhBqmgkgW61WCQCKomSe850PPvxn3/uDTrO29JwwT4M8vZ7OBcOwowiXKWRARks/ChfzKYNIvWUt7IkgirKivH39Jk9DJ1iwIoCQvnr+YjGbh34gQGY1WwAETdMkOe42WwLLciwDBI7XlI29nZvJaO0ME3gJUoCjZHuwwTLo7OJ0a2dbN6wkyyVZ63Q6lxdn7mpZMfSyLFdeCCR5uFjUavV2uzNaLIx6XbEMXhSiNBnPptVOc7qYu96KZaGfxUGeXA2vcowBgmmeC4IQRdHGxobv+3EcO46DWBaXZZqmGGPPcfv9/mQyef36daViIgQkSbq6uqK4aNTqkqJmuGAY1nU91/c4UeB5PnA9BJAoSEmSJFka4wzybJqmHMeZpslARHJsKv8/nv6ryZZkTdPDPMJDa7G0TLkzt6qqXbu0OqKPmO6BwQCbbpsBNQyEGcz4G0jjz6AZ72gzVGhghgMCPd3TffrokqfE1jt1Lq0iVmjpHs6L1cR9XqxcmRbu8X3v+zwqx3GO4yRZWmBk1OrLjZOGQZmkZsMmDMVC2Km38ihN0wzRoKhKCEjTqh0eHsVBWoSJyHJRFo+cZe9ovwCk1e8mYdRQDXe6sGp2WuQVTREAdlmFRqOBirJIC9uw4yhM08RS9fMXZ9uVm8dJsN00LJPClbvYWIomcFwQR7fTmdmobbabOA59d8vQtC6pqqwu5yuO4cu8WLtrSVeTosSEBFHEMAxPM4vxNK/QbLX0PE8SxN3Dd75cAAA2m41tmizLur43X64FSc4xMiyrQCVCaDab0QBAigJV5fs+hBCy7K5FMpnP1o4bRElWINcLRFn9+k/f1hp1VVUVRWEYZutsuu2WKksCx1IYyaIgsFyaJAwNGQiLAqGKOCsH5UiSJElVXj5/niUpripRVaIoOj09pVguTgpcAI7hgzjR7FqGyh3ERlHlxWqBqKrZrsdpOJuP9g/3/DjJEOIEtgL41dVYq7Uhxy9Xq6wovCgSdT0uUJJnCGFe1hKEltttjpGqyQwgIsczFJMGyfhqhDHykzDIU1aRkiyXRQUC4K7XTatGV4QQUrPtzWLpr9ZZFDI0zfN8EgUCDS9fvG63WoeHh999/U0eJaZdG82nuKqO79wpMR6PxzXLZCEjy/LO0eH7/uvLC0SD4eFRVlVOFgRlklb50ls7Sbjyt3GW8jxfJpk7X1VpwRM696P9VpeqqPF4jHDBcjQklcSwuKwM084JSTFWRIlkWZJEGS4W/qakKEnUESIY0owivv3eB4vF6o+//+12vQrDcOW4XhSXDDVaT9S6qVmWaVqDTleRJc1UEYXLrNo6PgS0qioUXQkCW5alKEo0x1UUqFk2KPHjB290mw1D1sMg0C11vpxdXVx+9fmXLM8BmgiyUGZ5EsQUpgBNyqo83D8CAEwmt4Nhyzb16WjMS2K91e73+0HgZVnEMEhXK0ujSBEDhAjBFEVEkQcQ0CxNKIquGWbTrpmaDiEUFVlR9f39fdd15/O557hJGH377Te7fpfvbgPPJzS1djaiKFIV+fLzr6oKPHh4+td//f8GFOz39j56/4OPPv6g3e0snc2LVy+fPH262WxQUcZBaGp6nmae581ms3q9/hd//ueqrLiuu1gs2q1WGIa7TLGmafVa87PPPnvjjbcoipqMxuvlHICKF4WiKLZ+mBV5o9ve+NusQleTkVFrFhWhWc4wbZYXPN9VVbWqECbI9TxFFASetW17M5tFYSKrSlYWeZlVBGVFGqfJ9e3N9fV1nudpFhdFAagqioPID+IgrEo0m81Cf9uo1ca3I9/3aZpeLBY8ZPI4TZJEN42djT6O490Ff7N1CQVoBnq+m2VJu9tZrFd5nhNCmo0WTVFxGD64e0pQSVBJU4xRq6maYVq1IIg2y0292ZqvN3meq6qqqmqBEQEgLhKWhYCqKAhuxyNJkiqMq6o6Prmz4yKdn5/3+32aZtI4K7JMFkWO4+I43vG4EULNZjMKwjAMN66zd7AfRZGmaVmWEUJYyOiqJsvyDvRaq9VEXqgwVjRVUORavT6bz9vdbp7nWZIShGVZ3rgb1/dqDZuXeMgyJa5ESVE0FTKU57l2w46jhKIoWVZpmg5Cf7VacRy79dzN2tV0My/LvCzrzSagqcVqZdbsIAgkSZpOp81mu8hRt9uFEJYob9WbqqyRqkrTtMiKIiskSVJVGe3ehzQVQkqQJJqhMCIsy04mk90aQ9M0juM03QSALissimJZoSgKKYrCGNdqDc8Ldpg53/d5UXAcB5eloiidTgcAMJqMBUlkWVbXdZ7nCSGaYawdpyxLWRApiioQsht107ayoqiqqtFoZFm2i9U3Gg2aYTabjRf4a9ep1+umadIM5EUhK3JMqkartXYcQNMMwyCEFEVhWZZjmDLPGRpmSbKLuvIcl8VJnCau69qmNZ/OIMvOlwvDMCRJYnnOi8MwjrKiQBWOorjVaCxnyxIhUZba3U4Qhr7vy7IsCEJapNtgCyDwPDeKgnqzlmUZKktJkEuMddPkWGHrusPhUJKkIIorChwfHAoce352ZZr1Wq0xGOzJoqRpGoA0zTIYY0VRbm6uTFMXZenl61e2VcMYz1dLyMHZcrFcr1mGpwA0Tb0iSFGUXaJxb29PELl+rwNp9pNPPhsO9uM45nm+2Whvt76qqjso2P17DwjC49vRe++8myWpbVrtdlsUxTCJWYGXFYWTxCRNDdNUNa0/GBAAdnsRSZJoiqIoiiYgyzJeEtu9LoRUmsYsx6RprOs6QpiXJQIpAugcYZoG+/tDAECaZqSiTMNGCJUYv3z1imHZMi9oAFgWttptVddM09w/Proe3fphKEhiFEVxGm0jb+04cRyrklohwrPcdutUBC3ms9Fo5McJYNh2u91pNF8/f+k6TlmWZZkXRfHmGw9N04Ys/81338ZZutcf6Lpet+pxHBuWhaoSY5wmiaZL6/VqPplCir68vqy16sfHx/1ux9TkfsM2JC7zHF1kCS4EQSjLEhBSZjkPaRZSdBTEDMOZ9Rpm6N/+7g/f/elbUGJb1wRJXrlOSSqe5xumvd/f6/f7YRxdXl8leeZ53tMnTxa30y9+84ezy/NPf/oZx/G2WcO4/P7bP+VhjLKyVqvxioRogOlqrz8o4zSPk0ajIapKmCZXV1fedptnmW1Zqqp++umnrU7vl7/8pcxJKEfPv38Wut7J4R4F0HQ63QZhlhNC881eqwAFzYOcFIRlBNUMs0xQ1IqG48Xi+uaGFSXR0P7w7dcvx9eMJNR1k8lxFsbDg/3p9XUexKgiYZzmecqyMEjCDGFC4Hg8zbLE8zeoKitAD3pDiRWzLGs0aqIoeq4jSDwnsOdX50kSkbKoijwrUkEV63X7wb27jbr98vkzAKp6s0aJzMJ3AQMPjo+evnzB8NzR4R26oueTaafehBR9e3NTM0xD0efzeVEUzXojiZLtxqmqKgzDLEmLJE28wF2u+62OKIpFgXaHR5lmrVrd1ixZkKMoCCKf4YUwSQeHwxIjmqbrts1DVuB4kRd1zaA4JkoTlmVrtg0AsOs1w7JWrkNz3Gg0piiqKhFdEYkXcImKosAUAAx0nA3HsSzPjSZjQgHbtq9urkVR7LfbgFROsFUtA7JMnueiKIq8EHg+z/ONTtuNAi/woygSNWW2XqZFmqapKsqSIEZJnGOUoiKvkGgbfp5UVVW3a+7acVbrPEl7w4Fcs90kWm3WYRBYlrXrXlUlYhhGVTVZUmkIt7639b0oT2+m420YAI4BHJPgMiozDKmKZxRDZSFFYTSfzMsSFyWWFW3tO3EeUxxdVMV66/lBdLB/hyZsGkZHB3uWqQOq0gxD07QiTlnIiLKc5rnjOL67bdXqkKK8OKR5NolDVJQURQGaipNsNJkJkqxoxu4Y0DQNADCeTiHLtLsdPwpN06QoarZYYEI4QbBqtel83h8OeFGYzKaswCq6MplN0zwDND2ZzWaLuSjL263vOp7ESwzDEUKOj48bjUae57tluOt4Ja4UzYAMR2hKkMSqqqoK724DO+ztLrgpiqIgCGWZB0FwcHCwE3XJssyxtMBxHOFMyZIE8cn3Pzy4d6/f6owuzmuGwdLsbDxRRK6maVmSa7LVarRG1ze341Gr131x9jrKUl6WozSJ41jkhaoo9wbDMAgMy4yKLCqyZr+tmLrrehQFF+5GNHXAwK3vc6JwdXtTgMpLomcvXn7xxVdRntI8rKpKU5Q8z5MkuXt8h0JVFEWbjdtptn745vvlYo3KKotSgEDohSLHVyXCeYHKMgpDnuOWyyVFUQzHcgLPcGyaZQDSWYVGi5mXRGvXKTBiBd6LtxnOsyyhCOB5QVb1uedCSTo8PJ6NxiJgQUmCIJrM5gwN66ZVq9UApMM4VC3DrNduJ2OCURbFLEUPur1mvR7HsReFUZbmeZnnuaJKmqaJnKiIKoA0x/OtekNXNUXV06z47e9+naSBYRiqosdZKsoSTrI8SihIx1mq6gqE8Obqej6d8aJgN+rX19dRGG7ctSRJd+7cadWbaVjoutXoN56efx+kASC4bRpUmhmsZIkKVeayxJWgoCAtcDwLgMSwNM5pQFEbb+sEnh8GzWaz3WqBAk1uR0EQXNzecrL40aefeGGwcZyb8ehmNHG2/ny25Fluu906a3d/f/93v//DZrMZ346e//D9ZrXOk/z08OTxg0eQojHGb7z1ZrPd2nqeLMuGbRmmuX94kJfFk6dPW63Wyelpp9u9vLw8OTmRFPmHp89DP7i+uMQlevr06fX1da1W02x9/+Dg0aNH7V53G4SrrXM7vnnw4AHPi3lZVBh/8sknO5O6qGlmvbHeuM1Od3h4ZDXrQRAdHxybtlUUBc8xi9msqqoKEM8P/TAoEI7TPEpiXTeXy3Wel4EfURR1eXZ5ezsOgihN0+VyGWbJxnXWzqYiJAiCKIowxoCmZrNZmWbh1ivitN1qybL86vysvzdstOolRovVkhWFOI5dx2EA6dWbKMsZhhEFGQLIsizNQEHgKYqiaaqsStlQclRWgDTrDU1RozBcLpdxlucYb4OwXq9LkoJLvLvx8TzP83ya55iQ1XpdIFRhkiYZISSNk+l0CiDNCfxqs2ZZNssyCtJVVSmKAgCI4rg/HHCQ0VRV1/WdyIaiqJ1vlhBimmacpvVmE1Rk550RRTGO4yzLRFXhJNHZbqM0ARhgTHiOydL4djzFgKi6adca8/WyooGiyIG3RXmpKSqg6BJhu2EnZeqGfhAlqq6vVqtOp9PpdCjILlebrMgrQBq1uqIo88m0yEoCqp17lqFhUWaKokiSRFFQEIRdSY9hGIwxhKyoiLplEkKCOEqSZPc0DP1QUVSagSzL5nm63To7oxDPC9PpDELIsuxisRiPxzzPp2k6HY9xWbIsy/IcBemT4ztpGIVhyHDcLoPBMAwNAa7Kqqra7TahQIEQocDW93eiYIbjWJ5jeI4TBUVRdh9SFEXHcWia3j1zXdclhCCEdthuhucWi0WUxN1+j4JQEARJknYZG1SU49vRLmGNMcYYV1VFGBqy7N7eXhRFURxTDKzXrDDwRJFP03i5WCRR/OPPfpSlaRRFEEKeF0PPT8Kk1+6cn1/yHGea+moxsw2LZ9j5fF6v18qyXCwWPM8CUEEG9Ae9s7NXuxhPgcooitauUwFakGRFUeI43tsfvvfB+9utz3HC/v7+fDpVNIVmGEmSIMsAABjIFUWZ5tne3t7Z+SuGYYbDoee4DMNIkoAqzHEcKvF0Ot3xAFqNumEYDISL6eLm5obQsDMYrtfOcLhfbzacrXt1dWVo2v5wTxYVqqJ2ZzAqyiTPaJ4tQRVnaZjEjretILVYryDHcqJQoJJmmR0sUxBFzVAFSeRFARWl42zXXtjo9C+vb5MoLUvEMIykyDzPS5Ii8QIhpFGziqIcXd8QCiRJVCTp1nEcx/E8Lw7DdrNZoJLhBVUxyqx0XRfj0tD0MAx5Qa4ommXZRq2mKaplWYLE71ZH0+m03mgghGzbZmnm1asXHM+UGLMs22w2O60WqtDvf//7w+Oj/YMDlmVN09R1fTKaBn5S5CiKorwsvvvh+9VqAcvCEjmD4+LtlgIkzWIIWU6SAaAZyFaAUBSkzU7TajWCJHZcd7FYTG5HDA33+oNas1Fvt1q9bobLqCgQDeI0aXc7CFXT6VxguWGv3+i10gqFXhYF6f/hv/6v/vk/+7P9/t6jh49Ws+XZi5ckK/Iw/uYPXyRRbDRrQZmFSawZer/bMzT98PBQkKXb1fzrJ98dHt35d/+f/+GLr748vzyjaCKJPM2we4dHk/HidjQ5uneakeLp0+8WiwmvSAhQdMWBomrVaw/u3Gka5rd/+GPkON12x6rVECYEQBowuMRPfngu1+wE4OvpeD69pSCFAXYcxwsiTtKyko7CDFRVkmVxksVxGQYZwfR243OymBRlnpe+HyOKLDZrThYX6xXLsqIie1HIiPza25imXlN0U5ABIRjjXqcLATWbTZbLpSAIvV7v+vpG182GbkZrt4xC2zBcd7vxAqvR9uNkG3mqpaycOYCYwGoTOkanzilSkqWGYaxWi8HeUDK0heP2j46WG288W26cLQHUbgKQZRmkqDxNGYYxTfNmPPKjsNls5mXB8lySJEmSyLKMMbYsy/MCw7DKPFdlhWVZz/eDIHBdt6Ipmmcnk4lt2wwLJVVGFRZliWHp29E1x3HOZqPLEsFlWZaqaaRlufG27X5XkiSaplt2/eby6o033pBl2Q8iwLFeEvESt6ub6qrW7/U2y3WRlRjjlbvOiqzIcsMwclytfX/hblJcYkiFacbTXJEUSRotFgsGwrIsq6rSTO389cVmuSrLwtA1gokqKixkeBYmUVAVZeSFHMVamgpQ2W91QEkana7jB/Vmk+NYigKhH0i8YOlWTbcjPyizXFHkRqMehiHkOUzRLMfpus4xDMdxsiy73jbHqKIAS9GqrLCiEBcZTQBHQYZhdvGYdrtZAayqsmaohCYAANfznO3WsqzdJnk2XazWDqkojhXKPAdVxTF0RRAvsHeOD8siIxWK4rhEKC8KUZFrzYYXBl4cDo8OIIQvXryo1+sURUmSLEsKw7CoQFmc2latQDiIosVsVrdrSZYvV6s8iUlZOJ4z2B+ooiBAmgFUt9X2fV+WFI7heJrHJQmcECC6wiCOw6zMojzx40jT5dli+ujxY0mR3cBnBVozJVFheVnCkErKdO0uaY6xm404zzmWrxlm7AeCwF1enrd6PT+OoyTlRdHx3UarburGarKQGC7y/Nls1m13GVIZkkIRgoqCoaHIcHkYQ4ydtYMR+vi9D3bNiTSOXz5/EUYRK/CqacV5kRT5yt0qlnk9GgVx/M7jdwM/XM7mKC9wXlCExgVGmGCGzgAuACgqUhaYAhCVFcvwm+UqT9IsSdM0FXmpQqQsCt/3kzxFBIs8P7oe2bWWFyVxnJ3evYcAFeeZIAutVoOh6Kurq+VimsZx02pYqjUbjU1dgxRgISyyIomSJIrczabIchaw0TbutLrtdjNH2WI2ZWjoh+lw77CqyO31taoIuMpLUlntdgEqwMI0i+v1OoSs63pN214vFq7rnV9cffHVl1lRUAD95Kc/Or++oViuXquJHPvq1av9wwPDtvIctxvDO3v3uu2DPMmXoyuVwRDnsiSYhkXTYgVY14tLwGJWTCsaQZ6e3I4iP3jvvffq9aYoyopmyKK4Wixfv349nc8AAM+fP9d1nWeF/nB/MBg0a/U37j+Qdati2aRMaYaiKDoJk8V60ep2OI7jGS6LosB1Nqv1aDR6/Phxmqau69r12vHpyXQ6/cff/BpCmOf5aDrJivyNtx/xorADLj98843nz5/P5nNEEML4wRsP9+/cObu4ur29Pblz9KOPP+kNBp1O797J3cV4hpPUd5z5dGbb9fv37xNCfvTppw8fPPjFz3+Wpikp0aM33vzJT34iKiqvSKxhFlUFKJhkhe+HN9ejIkeqqgFCFwVK8pzQMIrTOMoYyG7Wru/7aZIRig6SVFCky8vLPM8rQLaBP1nMn7184ft+mRfOah36gcjxAstNJpM0TR+99RbBVeC6FcIHw700Tsajqe/78/l8s9nohmFZ1vn5Ocfzi9VS1JTBcH+73QoCd3BwMBmNMa4arc5itez0umVZ7kbhuChrjXpepLImmzUbsmwUhu+9836Zl4EX2HZ9tVk32y27XovTxDCMKIr2DvYhy2FCJFnNsozn+fHtKApCZ73erNf1el1V1SRNkyzt9nqut6VpWpZljmFEXhBFMQ7CbqttmibDMEEQpHFCM9DzvPund2VZWa1WgII8zydZqplGWZZxHB8dHVGkcl1XURSGonme1wx9tV73hgNB5HYh8Uajoeu6KIq8IGm6WVGAFXhO+KfQYbNeJ4QYhqFrZrvdpmma4ziaBmbNVnVtMhsnSaIoimmaFIQsy+u6OWh3IUV818/iZHx7q4jK7pILqooX2DxPZVnERVlm+XK+mo0nhJAw8hezmSiKhlXb/fB2u8WEyLLsuu5gMAjjKAhDjPHZ2RlCqNFobB039P08zxvN5nK5vL29FTi+AmS3Tak16lmWMQyTFTkGhBCyf3igqup8PkdFoaqqbdsMw+RJent7e3FxUavVfN9P4rjX6202G47nBVGUZNkPgo3jQJY1DIOh6MO9fYqm5/O5oWoix2OMs7LApIqiiGc5gnGe54PBXrfd29XNNE1brVZxED75/ofQj2ia3qVl4jhu2fXD/QNFFrMkMVRDECTIsvtHB6isKEL/4Q9/4AQJAJpmOF7gXr16hQnUDLsoiqLI6/U6zUBMqooCaZoKLHd7exun+enpqSiKdcsu89z3wywr1stlu9mK49S27dO7dxerBQRQ4oXJbFZvNhVFmU0mtVqt1WodHh4KgpBlGcaYZmBFVaZtCLLUbLVuxiOe53/2s1+UCPu+b9dqx3fuYECWmxUGZDAYCByfJYnv+wUqfd/HJQo8nyAMIfTDYLleJVkqakqBEc1AgEGRlb7jAYpieQ4yTJqmn//hcwrQFYBphmr15mS+aHc69994GMdhHIeypu7eViuEaEDhElGoSqKYoSEqUBQlLMvyPA8hbLc7HMfZto0xyQskipKgCIqpC6LIc6IiqduN67mOt3WLooAsk2WZpuuCpKRJTujdqydrG9bxwSHGWNf1erMRRDEFoeOsZ7NZkiTbrc/x4s1k2u/3f/KjH3tOWGHm5OSurmokC0AasAyAkA6CiBBaEhVRFNOyyHAZpkWGCD00W5+982GRpP29IQXpChCKojiOe/vxO5qqE4zfvPdApllDUo729mEFTvb2REFw0nTqeYIi+/72YDAwdSMn+G9/9Y9XN+Pr61tVE//L//3/9kc/+emf/8V/iihCM/D08PidNx8NOz3f9wlNiZoSRVFVIs/dnr8+S5Jk//Dg8M7xeDzFFTi9f2/pLiRTwAzY298HGMCKidwgjyOWg6LIe1vXXzugQAeDvf7+vt5q3E5nsiz/4Ve/ESGVZ5EiC4fDfhaGr7//4Wi4D2lOtxsFpvMSEABpGiZxuJhPv/32W2cbbN0gCZIdm7cg2PEDjECFKVGRN65TEpLmJUtzdau+dNzr2UyrW288euvNe290Gx1F17IiH4+ne3sHhmUrinbx+rrf7jUsU2bh5fkr27TuPXzISLKgG412y7Q0XGatVgNC+Mbbj65vJq4XmkbDWbsizTdkK1x7KC95USxQocjCXqMJ8+J4OExDzzQ1URQKVGRFjjG+ODtXJBlURNd1QRAc13Ncrzfo345HSZJEURRFURhHcZqomuGsN/1OV1HUJE5lTuAgk+GSVyRFUbbbraBIhCKoKEM/gBTZbtY0AQihbeTNV3OGYRqtJgXhZrOZ345NQdYkpSzLwf6eG/iQFW4nU45nltPJ/qDfMIwqRe5mCwi9dpzxalJUuaQISRxxBKKsdF137Tg8K2iKztKQpeFsNPEdjxflpMgBRQVhSHHMdLZotVoUBeI8KQlCFAU4DtCAYsE2DJKiDJIE0PTWczzPxYSKkmI4HJYopwHNMVwaR2WWUjQBNGF5sdcbPHz4hiAp/b2+qIjddsvdrOIgkkWFIrQgCIQCFzfXNcsOt95ueE0z8Pj42HVdCOGbb77Ji2Kj0ZiMx1RFaIrKsgwV5XQ6bbVaQRAEQcBxwny+LHLkB1GSJLhEBGOMcZJEXuhlZUFoShZEQ9VQXuz1B/1ON0tTSRRVRaEAEHh+h50II79et5M4zIt0spxJqty2a3XdGE3HnMD1u935dDrsD0xV3/3HxkkuibrnBFQFKxr6cTIc7gMAgiDY2x+qulrgfL2aozLduMuVszK0mi6aF6/PeYY72juAFTRkQxHlm6uRwquGbOCs2iy3qmgeDPdAVfmuJ3BCEPhpHtebDcf3OVZstdpnr17Gvp+GoSrIg3Yv9ENd19fOptluyari+dsdu3zj+RSkC1QWRcEJQoqxE4aj0e1ms145myhLEcC6bQiq6AXbJM9swxRY7usvvzINo9VoD/t7X3/9dRBHiqEplvH01TPIcGGcQl7AmMiAw17aVAycF7udqqJrJUWiCm+SmOGECmGQVYZmFISsg2C93jz//lVVAVXWkjDhBFExdC/yIQ1UWYzjkGbpXr+v6Josy2EYYoxZyAg8r8ry1vc0Xdc0zTBtRdMVQ3e9bYZKWmLn2w0qqfnC1eo2ohDD0q+ePT9/dV6vN13XTZKIAUCE8Je/+IW72ULIaqY1vrmuKqRr5tuP3vE9TxJ5s2a+eP0KcLxqWnfvnTTqZlVVBDK1Vu+Nt95216tf/+rvQIXiMIqCMItCW+Q7to5xWSAkClqFCA0BZClWoFmeoSBgOJYOtl6SJK9evPzVr36FSCVJEkJoMBhcX1/fv3+foqgnT55wghTHcZIkX3/11X/4D/8BZWV/OOjvHzTbrSTNozwmkOYkWdL1MIq2wZZh2d/89vc38yktcFEUFVnubpzR9U3g+4okN5vNMAybjYamqIokLRaLMAwFjrcsS1Tkwf7ezehWFEWB51+fvTw7fyUKwn5/OBtPbq6u/Y3rLBfLyey9dx4LHP/FF1/QNH3x+uyjDz80FW0+Hl2dXxAK8CInyJKu65IgP3/27MGDN2jIsawoCBKk6CLNBI6VRYFl2dl4lCTJYr16/frl1e3VeDrKysILA8fb3oxuc1RuN47vehUAl9fXs+WCgrQfxn/7H/9uMhpfXV5yPH90etIfDkeTCWCgKMv7/eGz737AZSGK/K46sXBcL03tZiPD5e3tbVmWPC9ShN4slvvDPYbhXr58eXp03DCM1y9e0hRTqzVWq5WmyEkS7Yp53sYROb4osxIXiqIQQiRRkYR/et0pisIPA04UIMfSNL0b9dZrzTTPEK5EWS7LcsesL8pSFMXdODgIAllRLMvCGFMUFcZRgUqapk3T3AH21oulrqj1ej3OUtd1d5NT3/M263XgeWmcTKZTCjKbrVuvNXYdsPlkqklykeXtZkuS5AqQWq3GiZzv+xBSDIQSL4gcX7PsMIz8rS+KoiLLsihGQZjneZZlm82GhhSk2V0fuCgKyzJWq5Vt26IgVRWKgjCOY5ZlFUUBNEUIEUWZE/id1QgAEIZhWZZ5mu7as1VVJVG0XC6TJI7jeLVZswy/XC7LvNA0LQzDJIpqtZogS6IoYox5ns/TrCgKCsJao75j4muatkOH1uv1wPNpAjRV3QXkdwZOhmEMwzg4OFAURVGUsiyzLNMUlef5beAjjGv1ummanChEUaTr+mw2832foWiWhqHnR37gu9vNel2v1Xq93t///d8XRbGzVFeAJHG8Wi4lQdiN6WuW9fyHJzIvUAScn5+bphmEISqxbdvDwX6BKlRh3TTiOH716pWkKKZt+WGg6lqWZYqsqaIax2me5xih5Wxe5mmWJ4aun5ycPHv2fOv6sqzKsszxjKZpruvubqmiKPq+jwnYgRkkXkBFuYtw8Dy/nK8G3UGz2UySBAAAaLosS4ahHz58s9FoTBfzssKDwQAhhEjVGw4QQsNe//Hjx0EUJmm0mzEuZnNNknVNu7q6efTmm5ZqAlzNZrNOp1OzLXezTuIwTdNOp0MIiZIY0DQmlSAI6/WaoaG72ciitFuJ7fR1WZxkWU7T9G77vVxtZtOVqumaauYlMgxr0O0JPHt651jg4Go26bVakALfPfnOC3xV11qdznK94kVhtzhptzphEkOG6Xa7NMciAGTdaPe6ByfHSZ4BSA/7Q0IoXTNC15Ml4eDgoNlsAgB4XtQ0TZSlF0+faZrm+77jOMODwwcPHqACV4i0a42To+PpdBom8XK9WqyWdbu2q1szDPPdD99dX1/HWRylEQsZmgbXtzeHB3t3DwawzCAhEMIkiVVdoSBVonxXC6cpiiKAZg3l7PZa1bWf/eTHdw6Oi6K0LIuiqHun96MomcxmKUFuEqp16+WrV6pmGGbt1fnZi++/kznGCxJKkOr9jpeHi40zmy/qvYZi66ttmBOqOez9m7/+f6qSLLF8GASO43zzzTdvP3q0GE8lhuMgI0vSxx9/TAPQrDfWy9Wg25vPp5vtZut5AuCjVQAr2lmskmC79daqqlYVWE2XRZi9ce8uIZgWuPsPHzx6cO+w2YJJ8pNPPn73g/cxpDbONsvxy4ur1Xa72Gw6veHx3dPT01MGUCwGpq5CSEdRRghRFLnd7yCcy4qgGVoFKlbgWAF6oZsW8Tbw0ihmCRVsvevpeL51oigOPP/y7FyT9eOTU9OqFQhPFsvFZtXud+fr9dpxuq1Wq95Ybda6bSm6kaCS4lk/DWOUVTSRNRVCFlIQ5yVbVM54wnFMo1Gf3VyhLO4dDLS6wTA0QJgBVLvdnm5WmIa6pGdemqWIggzLcjSgKEJUXa83Wl4YcTJv1E3XdTvNFiRUuPVxWdE03ei0eUmURZHg6nY8iossKwtekdIS0ZBVJPXy7HKz2dAUladpvV4nNEnKNMcIVxUH6UGnTbKCRtXOB4YQ8jxv/+Cg1W43anVZFCVR1HU1zzNcoThKC4R11UiShKKoNE8wRooshn7kOR5NM4ZhqaqSpQmkaUWUaECt12uMUBAEmmG22p0sTm3TlhVJFiVQVavVBgOMq1JkRY5mCEIoz6uisnWLhVDgmCxLHccZ9PdIRTnbLQYYVYBQULMM0zYlRUMIsIw4GU1RkTcsk8KIZ2GeZgzDAUADGha4FGXBcdbXF5f9TrcsSy8KO70eXVHj61Feosvb0fXotiiKIssIxsvlctdwmU2mLMtSANA0nWWZLEqh77M0DDzf9/2yLGlAQQghZCiKanU7k+UcUYRioLvdFhhxHMfQMPQDBlA8ZLIorll1GkCOYhmK9t1tv9+165Yo8r1eZ71aLJfzbqu5nC9CPxBFcb1eW4pmqfp2vdE0LYhDyDHz1UKQhbWzYnkGinxWIZYXSlyhvAyDZOq4CaoOj+8uZ8usLBqNBoQwK7I7d08KnEeRr5tqnIaiLPYHh0ma5UXkxxtEsBcEoib/q//FfxGHMSmIbdsIoflspqlKVWJV1TBNIVJNx7e+s6np1v5wfzqeGpoWJqEfhVcXl2WOdF2PogBXJQ1AFmfbzVZRpNV6UZZl066JNDtoNAWGORgMISKJH7qed3rvHs9yZ89fdrqtxXweOltdlNuWdby/9/L5C01WaUAXCAWo8FBeUiCKE0VSAKpIgUTICpjCQVIkaZZlo/Xi/PY6TzKUIhoKWVl5UdLu9jlB2BsOjvs9k+csU+dY2KnXWIouimLpbNYr92Y0MuomI7OKqpIKpCWym42izJ48/Z5mOQK5+Xp1NR6fXVz92c9+tj8cFGnqzhwGw3feehRFQZD6vCbZ9QbCICjKp69fIoJMXZZ4odPpdHsDVhDLPCVl0aw3vvzy6+5gaDfqh8cHt7fXX3/9NccJjuMwLLB0xQ8cTpJ0y16tF6/PXkKBoSDhqFKmsVARlRcgV1EcyTGqCEQ5oTDkKBagkq4oYBhGrVajaXqzde+cnCRFHsaRocifvPfeW2+8UbfswzuHG3/7xTd/irO0qLBm6JoqT8e3VVU9evy2H0dlVTbsWr1mH945vPvwnlmzH7/73sZ1333v8WKx+OLzz13XrQhZLBa//d3vNEPXDWNnSb28uN7fOxyNRkVR/P63vxEg22zV33j4EBelbZiywIdJyHKCYdqtXr8oijSJet32k+dPdNtCFLmdTb758itb1yY3o9cvXiZpfn51LWt6p99LsvTF61dbLzg/P//ii6/m8yVF04giaZITQnRbi7I0juOts8a4DEOfEwVBlnZMBlVXAE1YnmN4Ns9zkeMbjbooCjkqGIbRdb1CaLNcRFEAQLW/vw8hHI/Hg0EfYxSG4e3t9ccffzxfLaMksW2bgRRFUc1mXdf11XIjiXK91izSDOfF4d4w9L3JZPLmm28SQtbuRlalrMwpinr16hXBoNMfbH0v8HyOYYuipBkWY7TZbNIiVVX11atXaZpCmoWAMU392bNnVVXdOToSWC4Igt0FLQ6jreMAAHTT7PZ7W89rtuq8wGZxtPtICJUUAaDCGJE0TXfyo+liGUYJQzEVIjtLEQAAFcV6vSagms/nhmGQigr9SJeVPM04yNIU3DHTCSG6brIcXK1WVVV1Wl2RFy4uLiqABYkHFXE2G57nZVnO04yFzHy5sBv14XA4Ho+TJJEkyXXdo6OjNE0FQYiCoGZZnu+WeTrodClU5WlWVVXNNmVJePrqBSPyrXaDZdm9g2EFyGI5cz1PkuQkSpbzZavVKcsSoULTlTAMJV4KPb/d7oiiGIReo9Go1Wrtdvvy8rKqKgDAbDIRee5w/0AQBM0w9gdD27Yxxsvlstvt+r5/584d3TQ2y1WappPJRBAE0zTzPHe3G1kWeZ43DINlWV1VoyjiOSFJMl3XWRrappVlGUJohySLgkCW5R3ULE1jiiJFkU3HE0lVAABJVkRJBgCo1+s77qBpmhhjz/NEUfSDIMuywWCQJWleFrKqeL47Go0Mw6AZeHt722q1oiTGGO8gaJIiszwPIRulCUKoxAWk6dViuVwvJEV0w22cxZ1We3w7oiiq2Wx63gahDDJUvVnbbp2//Zu/qVs2BcB6vnj44EEcR3EQttutNE3zPFNVpczzMs8BIJ7nAQDyPH/w4AFNwGa57rW7AitIvKSpRpymruvSgGIpsHU28/m002lNJ5PJaKzK6uR2lCXR40dvd3vtzz//fL2cNxs1DsJhv1PmqSqLm9Ual1Wj0VANPY7jPM8zXKaoACzMUZlEcRxGOw3AfDzJ42QxW2y32yiPS4BQWZUFDuOU5kTMcGFe2Kb+9ed/nIzGcRxfXd6gspreTjVFk2W53enVag2W4f04DNOEZrm1u81KVAFKURRnu52v1ghQhmWbpt5rd2ReaDRqR4f7iqQwNEtRlCzLFEMHUbh3cLg3POj1ep98/BkmlRcGgizQNP369evb21vXdcM4+vq7P2mWyQr80Z3jqkIUTWia+f5P385mk7NXr48O9pIoiNJIEITDw/1Hb76RRN5yOrJkVqSJKnIYFTzPURSVx0mZFlQFIE3zDKQqQiM/5ggVBaHj+bwienHw8vzs8vJ8PR9PLs4nF9dlnO73e7ZtfvDpx5P1+uXFa0mVh/t7giBIMnd29sI2LRYweeBaipBEwdOnT7M8Pjk9evHk6ejseutucFVFcTqZzR+8/RYjC6Pl/H/8+7/NyixJksVi+fW332YVEhThi8//EAeuRLPTi0sW0MvlUrc1o26mBJzd3P72D79HVfnRJx9MN7N6r/n5t1/yslSrN6xa43a6gJz48vXVZDIzDOv5ixffP/luOOj/V//l/6436A4GA01U7t19MHzwIIVwG0SybhaQykAFAMXTXM2yEMIAAJZl0zTbOm6eZrqqtTsdWVNlXWN4lsaIZymzYcmmyrNcw7Z6rWa7Zvvr9XIybhjG/dOT64sX9ZrG8IxmGZPJRJXk2Pdmo1ERJZogOeuN7/vNej1JEt/307JgBJ5hOFJWqiC7QUyxgsBwm/lSVHnM4JOTuwQBd+lYhnlwdCBqymazsXSD4ErkWUJVcZrY9dqwv1dEeZWVm+Xq3r3TMIxevXytKpLIMu7aRTm6e3JXEARNUxAuwiiSJGk5mdEIVxTRDE1WZYyxpikcy9ZqNQZymqKv1w4NOcBwZUVVFdhsNtPR2DIMXdU7jUYaREmWUhBKkryYLTRJ3uv1JVGMwrAoE0JwURShH+ZJbukGz3JxnBRFOej1KwoAAHRd3xsMsyKHEDZrdUPVaIpZrjaO4+R5rshqmuXdTs/zvP39/aIodjVxWZYJIf7GlUWJE/jN1gWohICiWGYbBlSFdVm6vb0FoNJkJU/TOAhlQVQEsUKYlYR1EHhxYthWEsUiL4RRBGhalWRSoV2zqV6zAMGaJEIKrJdzjEuaBhUqFEkusryosNWol1mpyNrL84uSgEajoStqu9PMy8ywdEEWas2aZqgbd+26brfdZiF7cXkNIG1oehokZZxXBWYhx7GCqMimbbEsFEU+jsMojQBL8YrghlteEXRdz0o0mkwJRfO8+PrVeZhme3fubH1PUZTD/X1BEAqCVctAqMjTuN/tJUnSbLQlUUmyfLNxZVHK0wyy8OGbDwmoNF2VRdFx11EU0DQdZiFNgyQMUV6ouukGIQYUwsTfeDjNaaqSZJGUaD6Z8yKn6koS+Awg/V43T9J2rZH4garKsibLooSKAuOSE9gPPn6fhsB1Xdu2wjCkAM1RLA2YeqsZxulitiCYyopSMfXPPvvEMpRPPni/VbOi2PNiT5LlLC8cZ1urNZrNJkWT6c2oyDKrXlutlgpPD1p129Yn8xkniN42qDebYRTN5/PFbI6TcjGer5abMErcre9sXWfrZmVRIFwRUOQIIyJJEirKXcUJskwJqhIykm70Ws3DYa8AVMnyrKj2+nuEYjiOIxSAEN7e3sqy2my3N77nhlGz29N0MwqTKEsZXgAUtG3bNM2T4zst2wg2q9fPn61WC8vUECr8KFRNa9DfT9P8+bOXk8mkYdSuL67zrOz2BoQChOC7x0d5lAiCQCBtdprdo+F6vby5ujx7/TqNkzD0JUkCAHAch0v00QcfGqrCslCV5CyKGFLKLEFRyBKEQM6KECFUJgVf0SwiVVFUFaIhgAxF12q1zWoNEG5atSRJLm+uO51OURTf//DtdHZ7eXZumxbH0Aykdrjq+/fvHx4eorKKkkSW5b/6y78ssoyDjCrLpmmuFktRFMMw/D//H/9PKM0vzy9urke1Rj0usla//fDNBw/fekhBRlK0MIklWT45vlO37CQO/ci3ajYF4bNnzxbr1cuXL8MwvLq6iuMYY/yjn/xZu9eN8wwQfOdgv9Nsvffee7qu//znPy+KguW54Z1jDClRFIe9/undO3/2k58+euPh5fnF6Pbmmz99XaZpEseYovVmC3CcH2eCqFpWXRRkRVarkvAMF/lBFMVURTjIUoBBqAKEoAKJgoAxdl1XEATN1KIkxmXO0nA+mdI0ePz22zwDUVFWJZJFKYqirMjny+Xx6cl0MvId90effMzSEJeFqVs0RYVBxLIsxTGOtxVFUVVVgKsoilhRurwdRUGoa+pisaBoetc6MXWDAsDxtlEU9bs9gnCaxdtgW9NNSRDPX59lWbbfGyRByDKwIljRVIqBBSp4nt86riKp0+mU5/kdyF6W5c3KGQ4GPM9HcVBVKIni9XrpOo5lWXmeY1ylaSrygigpFM0wPFdiVGBk1Wuj0agsC4qiKoLshn1+ebFcrrut7nq5WszmsixjjAgAlm0ILF+WJUIoK4udeLPMizAMAaGTPLs4e+1u1gwDsyLbTdXtmmlaOkKobtkYIWezwRWiKOr29hYhBCH0/S3KC01Rm83mDz/8sNPevnz5EiEEWYaCtO95SRg1avXpeFIhfHrnGGPkeC7DwK3jIIQARRWozItC1WRFlUSJE0ROEkSGorfb7Ww202QF4Ori4qLT6Qz7g9DfYkA4UfA8T5ZlRdcYnkuSxDAMSRE9z4UQVlW1mM0tw7y9udnVLKIoqjC+vb5hITOfzweDXhrFkKJxXuRpFgWhyPOEkK3r+76PUMVxHM0ygiTSDCVIfKfTcl23KApSUYZlZUUBKiKKoiCJfhjkZTmbzdI0jaJot6Lsdrur1cpxnLsnp6PRaDQa9fv9LMu2jqvKimma//pf/2uEkCzLW8/Z2bS9YLtxHZpnVFXdwSQGvWGa5llWDIdDCKHrbniezVHZqNcpisym063rRlE0Ht8CSPI8LxC6e/fu7iSu1S1claRCX375ZZTEgKowxgRSkKEJwhCAjeN0eu3BcHh1c8XwTLfb/du//ZsgjqaL+YvXr0uMDMMsyzJN88O9w5ubm++fPY2zXOA4SNHL2VzVNYbnfv2732KMIc3qmkkzjOs6hqkDAHieD7eezPIcZFBZlmUpiiJF0RRFUxSFAUEVBjTleZ6qqgSDwAshpMqqZASeEfnxbBrEydJZsbyY5WWalo1GY+v6zXrD99x2uymp0mazOT2+02rWN5tNWeYVIAzDNFt1VZFUUfizn/w4DYPR1aWta2kaI4TOz18vFrOnT57fOzlt2c0yKbI0vXN8XGZ5FsYA0ISG19fXNE2/fPlS1/V79+7NZjMCwJNnT8uyTKOYIiBPUs9x/ylv4nnf/Om79caFEKKyePnyZeC5usS1TQ0FfsPQeZ51PJdhWBayqMBRFBGaFEVWYpxkGc0oYq3VNGRVomlVlHBelGn2/vvvf/bLn7UP9z780acff/apyAs/+eQzjZcUQbRk9cvff35+dYUq8vrs6r//7/5dHkUVLl9djv763/9Pv/vdH777+puLi4uqArPZst/fG+ztn96/9+6H78VZ/Pkffy/znCpIH73/AQZUvdn4/R9+p4q8QNPOcjE4Odp/cG9wfDy8c+fw/ikUeZZmOnb9sw8+ujw/3yYJ5IWX3z+LF1u5YgUMZ6Pxl19+qSjKeDqdbmaCzvMiI/Dw9bMXf/zdH3/997++Pr8wdPUvfvmLhm09uHu61x8KktI5PuJkJU9wHpQ0BH4URn7M0yIpCEmRIWr+NigRyHK0HC91TpZYcb1cCYrKSnKZlQygFEUZDvuiJmVF/vzFC8uqYUwQqhCmNNMKkpQRed/3TdM8PToOnK2sqSXGWZIILOdHIaKIrCiEkJur68lozApiQbBiSKopU5DK00zipc1yRVEEQDKZjiiKEiX5/PxcBNCSlQJX7V5f5WUOg36vt9msFIkHqCjT5HB/Lwh813MHB/uQYwiu8jSTFHk6n9MAsIAp0gJCGCSxF4W2YVZFWbdrpKpUVeU4Lk6TOI5lUYQUBSqiaRpkmYqmMCCIVKppjGbTMEt024qiqNVqdDqtXfux1WrFcRzGkWFbWZYF7rZVq9Mcy/Acz/M3V5d5kgocx3EchLDbbfM8zwBKFuXRdMyIHMHYWa8N2zIMo0zTt994I4qCAhdJklQl4lmOpWiR5QjGF1eXvf0hhJAQ8sbbjxCpDvcPyjRHCAEAfMc9GhzQmJqOx5qt9g86mEb9frtj1yWGi+NQM9Q0T9bOYrYaCyKsUBF4od2oH945TuOk3WzVOy1CgTROKoQZhllvNuv1OgzDMAzTNI2SRFVVVZFazTpChSQJNKbKpKhb9aqsfC8WOHnYG96/ezcMw263s1mvZEnUFGUwGLTbnVqtDggVBYEqyyzLbjYb13UNw8qzskLE1K3VaoMx0RW1bpvT2VgQuVarRVHU7XTibLcMTcu8EMexaZo70USWZaqh/8+ZQl3Xiyzvdf6pggApGkIYRRFFUbZtn5+fq6o6PNgPklgx9LxCLMsKhC6DOI9TGlB+EpUUQRShWKbV67rbrcJJ/WabAEbSVUaEcRptgkBUtW+++SbLshJUsq7hMgMY/cWf/yeb9XY6nrTbTcVQLm4uJVkY7vWhSIVFAEWq3jIZunJXc1mWK5pabP3WcN8yGxwUdd3eH+5RVVm3LS/PRNPmGDlwglajsVzNX91O5Vr70eP387xMCmTWGzQEPMuIsrgzQZMC4SSROEZWhLzMWcjkUVaW2Pd9SZXKqiyKAmCgCDIEUJAljmdaO74YL89db29/eHH2fLVaxVEqirKpKpvZrN9uY4A5gZUFQeMElWVNSSIAt9o1WRJVXthvN53J9Fd/97dFUTCEmk2n3cEQsLDebCRRLEL2m99/+cWvf60wbLvWWM0XGONuv9Pr9WRZliQpTVMAAELo6urKMk3XcSxVX87m08nEUDXbskRBQFn+6NHjqgKbOLiajGzLYgF8+63HqiSbAm/wUAAUyUtFlHRdhxxbEByh4vEnHwqKmuYFZBlZVejvnvwQxsE3f/rq5uam3W7fuXPn5OREloSL12eiIFh16+bm5tWrVzdX13v7A0iB8/PzNEsePLgrCNxiPA3cbeD7g8Hg8XvvvvnWW5aqf/Teu91Wtzvo//yXP7cs8xc//dm/+E//s06rsdfvPX78dhxHbz9+a+czOjo6eufNN4+HwyKJ//lf/LkqiZdXV51hvzfcK3BVFMWPPv2s3x04y1WFcJblCFWjyfyHH5788fef/z/+zf9d1bQHD+5//+SHTrel6koQeLLAj69ueIYVWKZXb+4P9/KiGM/GcRp99913giAcHh4OBnuffvYTjAkFYa3VzQnJEDasGiYUgDSgKRpCCGFZlhgVs9lkfHNrmRbH84SQzWqtq0aFiONsozDp9XqddjtJkiiKaQpynFBVQDP0MIpwVRGEQUXyNH3x6mVW5K1WK0mSxWpxeHS09T0/DD756KN2s/nDkydvvPlmlmWSLFAU0TSNIKwrOiuwqCo1TaEoEibx6empt3UpiuJELgiCOIxi33Pdzd7e3nQ82W63tVoNY1xiJEii728RQk3b0lWZYRhCAcMwVFnRRDkMgpO7p6ZtLJdLjApZllVVdVbLMPRt2wY0YFmoKNJmvfS2jqzKG3et6EqSJ16wNSxdVCUv9BAqJFEMgiDP0+VymSRphYFt1ZIkWa7X7V7X8zzP84os97fbdrvNcRxkWZFneYYtiiJJIlmWPdcxTVMWpSgIFElK08Tzt7Ise55bFEWexIooiaJYYoSqKkoSTEit2aiqarVaLRaLHSzXc7c73Mou1FEilGcZhDBKkzhNTNvI89zdOCLPP370DqRpioEA0mkUJ2EsCaLv+4vFYr1e26Z1e32Dq+pmMmYYZucqEUXRtK1dAiSKoqqqxuMxx+3CP9AwrDAMa7Xa4fHx2nHSNLVq9na7/frrrwkhNE3vPEF5mmVJGkURwzAHBwdlgWzbRgVud/uyrBY5qjBOozhJElXV4zgmhGCMRVGsqkrk+TSOTd344IMPmvXGZDKxLEsQhDIv/K2Xpul24+R5HsdxrVZLkgRUBFRks9nU63VTNw73DwqEWJbdLYpc182Lwq7XGIYhhARBgBCq12rHh4dFUSiK0my3oji2LGs6nZuG4azWaZLviE9pmlv1GgZY0zTDMFzX5TgOAOB6foFRrdl4+Nabp/furtfrnTE1iULPdUY315PpiOEgoMH19ZUsy/WGXTMtSRL29vbiMNIUBULorNcMqCpcDLo9hoa6rjcatbundzqdTqfXrQBZrTZFmnf6PVEUZ7NZEgV1y4QU2MW6WBqCstrRE8ssFwSuqpBlWWmaIoQMVSvzYnf4hXHCizLHcagoZUON82y73T569LiiwMJZR0nIiPyg17+6uCzKXBR4XVHPX7+GFM0yNAAVKsqmVSMlIrh6+Mb9O0fH9+6cNJvNw8NDACpdVwVVxhRRFKXMcl3XHcfhaLhdb8Iw9MKAYZjr6+t2pxPGkagqhKGzsqgIARVJk2RvONzF/13XTaNYUZTXr193Oh1LN1BZ/uY3v2EFnhCyXsyrONBZyDOwqogXBnlRFKggFCAU2DUzEKkIoTDGNFXh2Wxmt5uCqXz39Pvb6eji4uzLP34+eX2Jg8Rdb9Iy1TRV0RSKpvWaVVD4jccPa7b64TtvGbLw6fvv/+ynP3n8+FG7Y4sc+F/+q3/x7ptvfvrxR3/xF/9scNhfO4vx5cX/7f/yfzVY8dH9B+5mlWUZZNnFepXFye/+4R82s8X3X31zenAYblxQoDfv3r85v76+uL5+dd4wLNsyry7Or66vm83mncE+QzFSzT586636fv/g4d3l1vk3/+3/q9GyMUEAlZ1aYzGZCgw8GPQPez2BphlAunuDH85eI4bp7x+8/eituwdHGq/dP3n445/9/M333395dlWWmDPV0WZCiVyj319t1qzMcizFUESv6SnOCECtWo2lqaooIaY281UYJL3uEFbMqx9eKYJ27/TBrly3Ww+madpoNPr9/mw22zruLoTuhwFCKI2Tg739JI4FWer2epPJhOf5Xr9zfnZmSApPaJkX6g3b87wwDJuddpCGz1486/f77UZzuVzyqozpiq5wGoWCwEGOlSSpLMsS4/5gz/EDP05YnlcUJc8SCFDgOaLA4hKVeVEghEnlum671kiiOInjVqO5gzATQuq1Wt2ygygsUAkhpAjgIKOrGsFVURQ0AYaqoaKEEDabzR3oZr1eKorc7LQFSRREMc/zCmNLN5rN5jYMRFXZ6/Xn4wnHcZIklRVar9c8z4dhGATBcDjcbBaqKhqKupzOGIZJkiQKPJoFiBSz5QJjLIsKTZE0jYuqzCoERT4pc0yqnTvJtu0kTBiKLjHauM7G20KeQzRAFNFNzTAMApiipKIgFnjeC3xMKN/x8rQIo0hV9f29O9PRiqKgpim7YurFxYUgCAzDcaLkBSHHCaIo8jxP07RpW+vFslmr0zQtKfJ0tnK3oSRpk8kkytNN6F7dXHI80+01kzwqAJIMDVUY48quNUuELcterVZRFEmy/PTpU8uyPNdrtVqbzSYrisFgkMTZg3sPUY6KHNE0LalKVmaEEFPTUZbykJYEYTIaFUVRbza2vh/G8XK9arfbw/6eqVtJFECK7JAMHMfV63VCyPX1NcMw5+fnZVmGcbzdbAkiPC+iAl9dXLIQ6qomCEKGSoqBz549Ozk+pipCcOVvA2ez2ev2fde3ag1RFAWO5Sj4+K13ixRJHJ9ncZwmtUY9TVOaYY1aPc6LP379hygPnp9flAT0O92r1690VdNVzeCVfqNLEbrRaqm6YVjmb37zm2bNfHh4GCxmisigIn3/8VuAFO2m1bA1CYBwvWZZIqnCdz98m+dp1zZxHD5/+kN/b3D+6uVodNPv9x89etRp1PMk43hIQVBkZRKlAiflWcGwEKGS5dkSI4pmWEbM4xIVmFBVjAuKE+ut/tHR8WDQi/y1wEOKZoO4aLQ7RYXrg27F0JvABzQNCFWVWJE0VTEux2NOVmgAx1ejNEj3hoeCprhRIPL8yyfPXrx44QfB5dn5l19/5SURJfFe6rEKP926BU21Wo2PPnh/s9l4fvj81fOsSGfLxcp1RvPpYrPu7++lRZ7neRRFo9EIALB2nYoQVuAZhvHCgBASrRwOA0xDL00vbq/fenD/uNPiq1IUecAxqmlIuipIYpomAgP9jVukma6ouET+NqBFloM0cP3tD0+f1JsNAMCr8zNJkh6//Ujg2F63++DuPVVVSVXdXF9vlouf/9lP7t45Hg6HaZruDwd7gx5FUde316vVKo6j0WhMCNmsl89+eAIpWtd1UeR//OPPdE179uRZFmUNu5YEoS4rYeABAGRZ7vT6DMd98cUXz77/odNorperdrN1785Jy66/+OHp4f7+j370o+vr69vRDcMwtXpr6birrYMqLPCsbRkA0pqm8Kwwm8wlSYEcS0GQFulXX//x7NVLmgBFED9894Of/dmfXV6eCwKX58VqtfaDYLA3/PBHPx7ee9DuD+qdLqso169fM7aJKBglKQ0ZVpDsVrt/eHI1WcyWDg05lhPjtDQajdeXV8164/7de4v5dL1etTstilTz6VjgWU1RLy4ubm9v7UY9ydKiKHYtx12MGudFFARZkS43a8uyKoTzKOl1OqgsJ5PZer3cbhxRFGVNXjkbjhNO75zcXl3jCmm2HiWhrKlhEFMVESSRk4Q0TVVV7e8NbyfjB/fvQooSWS7w/GazKUoSBhgAwFD0cDicLZaSroqiyLIsKZGpaDswfV4WfhC0Wq2t6242q1rNwoBcX1+bhl5mOUXA/nAP0jRLwygMTcO4vrpoNBqb1aJZq/M877ruZDbbsbSCINgNXqIkNixzOp406w1FUSazWa1W43hmNpspisRx3HK5RKiSZXW5nO9CIDQNarXaTtq5t7e3mycAAHYJa802AU2ZllVkeZqmtVpNYDlVlLIkNTRDEASapnNUVhSgWCbOC8jyqEACJxiqtVw4BaoATbmui3KUp4hlRIbmDMMcj8e6rkdRRBPQaDR4Udhut4PBQFYVQkiRZnEUrV3H9/3hcJhEMYRQECRN06uKFFlum5Zt28vl0tB1QRAuLy93jacdpcc0zSRJkiTDGJVlGcWxFwaqql5dXUFAMRQNAECo2l3VV6tVr9czDGO+WM3n806z5a43i8Vi1+YNPH/3PVxfX+/wYZqmjcfj1WoVhqEsy3t7eztAP0PTznqNKqwZep6mg15vtVr98pe/vH///mq1EkWx0WgM+3134zTs2nK5bPQ7MSokRfniiy+qqsqTVJVlgRVQlkdRFKdJBSnX2zqOs1qttq43nc0YjjVNcz6f0gxkGKaqgKZpjUbj9va23e1ouhL6zsHBXomL2WIqijwgBFL0n77+FkI2zfO7J6fz6ez7776LwvDu3buGqf67f/vfsyy0G3XXdcPAExhqMhm9ePFMVpXr6+vxzbXIc4NBz3HWsixzHMdx3GwypQmQBQbjssAIChwniGGUhGEMaIhJlZYlpmhRlCtAERZCWao4juL49z78QBTF5WLtrl2B4xVJ2Kzd1WKtSLJtm6vNOkMYU3RvsPf4zbdM3eJ4UbdrHC9SFGWo2snxKQD0eu2MxtMoijTVWK8dhmG+/vprjuOG/UFZlriqRFlae5sgiRvt1ng8nk2mAs8ykKrVavV6nYPMXn+gGwbDsVEUyYKIi9KQVdu0rq6udhMbWRDH47FhGNvtVmC4NE4kSaookBfF0yffyRCDPMJlnmTZjsi0oz+JvKApCgMhpOgyTzmGpilQxUHY73Qtw9gbDBVFafe6K38r2sZotYjD6OXT5zfX1zc3N3u97kfvvCcSELneN19/9/zF68Vq+Q//+I9ffv2VousE0D/9xT/T7Mavf//5erF84/TUkPWf/OinQZ6mAENJ+s3vfs9Q8Li/Xwbxjz78sNNqPnz7LVqVQ4zGi+XDN99q1OreZv3O48embtw7ORUYlgHUej6/OD+jacAKrCBLOCnZilEl2d2sF9c3dF6IPHt+cfX02YsCV2GW+1n2/esXK89BkBRltr6+Pa51mpxy+eTZcH8g2Vqt0ywwOjo62h8eDNpdTdLeffvDn//sP7l39+Gf/Yu/agxPBydv/ezP/7OPPv05pmWrdZxScvPgwb23P5FrXcFs/MVf/tWDd98+uHt0dXuuq0J/rz1f3Hrb9X6/Uzc1kmWSyLdbrb3Dg6fPnz189BaBtOd5uxa4IAigxALDhmEYRIFhWYSQIoprqs4LihuE+/v7x8eH263jui7P8LEXOcv1xx99lOPi6YundrOFEWAYvtUZAI7LMapKtF4t3NBttmuvfviBQ2UWRKREw/299dZzg5DmOJZlp+OxommQZSlIr9frqkTL6axZbziOQwgRBGEHmarbtR3htt/vL2dz27IoQibjcRZEbAUOuv3I2cqC6LvbCuE4jlerlSCJiqJqplFg1Op1N5tNkWWSIG4dd//woCgKQNNZlvnbbcO0uX8au1N5mqmyliQZqyqbaItQRtNgtXE5KMhQipytZRkHR/txkvlRLEkKS0OKAIxQr92p8pIFtKGotm6yhErDqF1v2JrVtGuiKN7c3Mi6xosCXREO0FTFNlvd/nDg+u7e0X6appqoVSkJgsD3PVXRXdeDABYFMiw7StJOqy1yfBBFVqPO0FARpTzPGZ7zt15VVVEUhaEPEOq1WlHgpXEMStyrd1COZuNZkRbdRicJEh6ykKLHNzfdTqvCxWq1ysuM4dkkS+IsrdfrLMswPEczMIwDQRAwxtPRlIWcFwTD4VCTFZZh9ob9qkJphaIyl1UJUiBOk1ajyWCS+SENKEIBRVPteu329naz2aiavF7O4zB47713eJ6dTEa9Xo9jWEEQbm5unj17xrJshUkcx76zJWUBKgJoaulsWFm0ajbLskmS7A/3pjcjhlAP7j0kCHhBpFu2UatPlvODgz27aTEiT3NcEASCIIRxnBapt1nHvpcnebfb365XHE0aDdv1V2t//ej9R7TCn1+eKYL48OTubDYrS0xR8PLq2mo2gzT9/IsvkyQb7PXTvPjNH78Js4pTJIqDFaB4US0QoCioGnZWlFGWAhbu7+/PZ7Nd3rHZbIch8rYhhCzNchmozHpNVLWyIBQUGEEN0iLJC6NmpxxkauadR4/f//QzkYfthq3IclXRgHA8I9VN48Hd466t575PY0qRjRKB8WSWJ/lsPPnjV19jmjk5OUnjrG43MMZJiYuK1HUTJ+Xr1+eHxyeSJA0HA1SWWZzUNJWjQIEqzbDarYata/P5crNx8zhajkaaJMucIEKWgwxLQ1mUFpNpGkY1zcjDuGHZqiAlQdiuN07vnOyCsyzLiqqKKrKaLW8vrxiWPjwalonHg0LiGJHndi0zgeNFhtsNymiAiiwq84xnGVqUpGandXCw9/bbb+8yzrquy6riBX6723n1/IXE8VVV3dzc/Onrb3zPu7m5wRgf7B3qkvb+u+91Op1ef7heOeevz373m99ynFCrN9vtNgBgOp3++//v/+CHwd/+/X/89//j//TOu++XBf7Vr35tGeZiOrl37979Bw/2Dw4RABUNCWTu3r/XaDQcx/nqq68uri+yIg2iUDF0XVeb7ebR6UmJEM3AqqoAoBVBbBpWt9ZAGdqsHI4TBoM9CGEcp3WzHiWJ2WxiSMVxPBndPnv2xPf9OIzmsxkD4WI5//TjzyReSJKk1+6kUdpudzu9/mef/vT9998f7B0cHB3bzfYHH3768O233/3gkx//9BfDwyNZt8x6YzDci+N0s3Z/8pOfXI9uPc+7d+9u07J++O7bPEtPTu88+e77NE1Xq9Xjd991tm5Zlu12y1lvdknVwPdZQNumxXHc2cU5z7PdXvvi4iIvkapb16Pb1xevD+8cR3HKMbwsyhInvX79Oo7jDz/8UOD4OI7TOEnTDHLs/0xALDD6+vtvNdMQBElgOQjhkx+e2XVL1bU8z2maNgzDNM0sy1bORlJkVJSWYU4mk9PjO4qijEajvCgoms6zbOu4kiRpmmaaJkPRCKHDvUNN1oqsTIIEAqZCRFEURdFUVdVNg2FZwzBms5koKbfjUVVVsiwbhoEQSpJkxy+zDMPUDQhhhXCapgzDCZJYlqhAmGEYAACEbKfTMywzL0sAQOBH260/mS8009A1E2O82WzKsqRpOisLyDLz+RwA8OrsNSfwEFDL+SKOY3e7dR3Ptu0SI2e7Raj0HIcQXFUojGPTss7OznRdpwGlKFIcR5qmCYLQbLTbzZbnefPlghOEreMkUbz1PWfriqJYt2uWZe3mNkEQyKrC83xRZIvZFEIoy3KZlXGc5ln5/nsflpjEadbvdse3t2madgf929sbhqEty6jVarPF3LDMer2epqnjbauqIgTzkhinyS4YnpdFvV7/7rvvaJoeDoe73xEhRAi5PL8ghHQ6nTTPlsvlzvS04wXtpFoYY47jdF1vNpvffvOn3SW9yPIgCJrN5rNnzwaDQbvdhhDev/+gyLLVYqnqmmYa6/W6224LglABwHHczejWMIw0TlzXlWV5h70sMRJEMQz9et3eekFZ4MP9gzRJdjLCTz78ABelIEg0oe/dOaEoimY53bSb3R7CpNtqHx0cojzzXef0+M7h/vCdR2+1Wq12t1NvNsyavfW9JMvG0/nR8alu15MsT7IUE9JstoIglBQtCALIsaIi1+v1i4uLfqc77A80RfWCiOYo06wDivWzjOI4JwwJZBhJ4USF46W9vYO79x5Ytfqd0xPdMiuKJgD4XlikWRyEe93+YX/Ya7WH3R5d4c1iIwlyFEUcx3z2yccffvBeWeaiyBuGwUCaBlSrUaNo4nrbe/fuWZZlabqlqQihWsPWLdMPAlGSGIbxXLfMC4KwyPFZFPIMfPudx2Ec8Dx/fHS0nM3Lomg1m2kU77B9lmXV7RqoiGEY8/l85zemGHgzutVNI82ytCx4WbLr9YODg8O9/dlk4jqruqFAXLKQpgElimKapmkQ0RQFqgpjTHAli5LA82VR0JTI1jutb3/4/uXrV99/+21VlLZhNu1ameWqJN85Pt75t3BVMQJ/fnsdk2q8XOaef39/T6AgLrDjhTwvtizr3sE+xOXR3vD09F5alH/3j//Q6raOjo7+2S9/effOSafTyxBu93uYBi/Pzv/w2z9QFeEo+NYbjypAbZzt9e2tF8Wtbqfb7/AiNzg+ZC19m6cFVSBQpHnW6HWJABOcbVZrb+kGm42t6Dwt6KpZs2x3tdFFWePEzEsbZudyPAWSqLYsSuEcFHi5r3DC0Gwtrkc8BX/99//w21//BtOgBNXh0dBZLf1t8Ld/8x/S7Trz1rPJNMuy69H5ejnerifX589Cd/2Ln/5o2O29fPLCEHWSk8uz23qtEyb5q1fnDIC2bmEKXI1GnU6HoihRlm5Gt6ZpKopyfXl1eHjYarfPLs5VVW02m5dn5wzDtNvN0Xz67NXL/eNjXAGOE2rNVqfXn0zntVrjeP9AYFhW4BVFERj25uzK3WwMRdVlJfQ9PwiyPJdUJS8Q5MWDk/s0LwNOoASxAhQhJA58HgJdlViBu7697TSaIs9LslxgZNVsQRJlUbq9vU2C8OTkhOZZRBFN00zDiPwgDEOEsO/7FEWlaRqnab3ZoGi6N+jHaZamOcOyXhitN44fxZfXV5qmpXGiSHKOygKhJIyatXpelpBjVVneMVg8zzNNk+fFpeNWNBQkoUJ5u9miCERFFYeJrEpQYp0o0GuWqOosJ9KE5llYVRXHcWmashw32SxjgLw0Fg1NtI2Js9IsE1U4rxAni6qqOo6Txkmn24rTiOGZJFzpCjtZTCpAHQ4PdEnRTKHAQc0y/K2DAcqL1PM8WZQ4QdhRYvI0PTg+8sIAAHB5eQkBtZNmtLudoihkWdYtkzB0SaqkyAlDowpnuKwg5ccxAqAkFSeJnCTGacLJPCeLFSCiImqGmqaxKAq7GF8FSJyl6/Xatk1B4nOUVxUCgNx/+EDWtSAMRVEs8hwCSmQ427CTJKMgjPNs95caTcZpmgocr0hyGme1WoOm6YoC8/l8MBi0ms2yKHRdbbUam83Gtu3ZdE4qMBqNkiSRFAUTwImcqsoMoETIFkk6nU7Hi9lPfvGz49OTrMgVVa0Aub28YgB1sLcPIfR9HxflTlBVVZVlWfV6PYoix91wDHt9ecUwzLfffus6QYFgmFJPnpwlYVH42csfnjIslDVlOrrkANmsF2UejW+uKLpabVda3QQ0tGr2eDYtEcIECLxYlNlitVAVEyMgCIzIMwBXo9Ho+OiQY5jri8skySy7ZloNrdbUarVmu9ft73WH+/3jOw/ee6853PvFT37+zsPH9+6/MRzsHbWHx61eGcdb16Ugx3CiCNnbs7P1aLweT3zXGV2P54ttoz3YO9ybzW9vr87a9RoGVW9/cLQ33Mxmy/HY91zXc+6/cffq8ozk6auXzybjG1tXRrfXQRTSPGvVa/VWswLQ0i2FZUmWSVCYTebz+TRIw3q7Q3O8qqo103BdNy1yWVX2Dw+a3Y5i6jkuGY6VZLmiwOHJndeXF7ptlTSQTZ2wcLFZ66aRFQXPs5221TDkLPQkjinyNM/z3QpdFIQ8SX3fZ2gICAMAw0CBwpCWZfn6+rqqQL1eHwwGpmkCAGaTaRoloRfYtdpitfQ8T5ak/+a//m9s23a8bRAEgb+lKWDYFqZBs1kviiz0va3jvvve4+3WAQBst86//Mt/sdcfHB4c+9tgvz8s0xzSIM2znBBFNyyrtllu0iRRJfng4GC9XguCNLq5nU2mWZEnZf7Ds6ee76dpGrjbpmFYipL6gaYZimFCgZMN5e79e89ePK8IYDmhQhVD02le8KK0Wq2DILRrDU6SX12ep1UhKHIcx9fnF9fnFxUq8zxP05RmIMfzLM+8eP4KIdTu9FTdAACIvJgn+XK2VEWlUau7rsuybFWhy/OLndCSg8yDe/dlWc6KvDsYHh4duY4Hafa99z8ikNkGfr1et2374uKCImCv1zc0LU8zmqYRqRCpSowsy9IU1fO8Tqejm1acpUmWu66745QKokLRcLPZIFRuNhuW4WkAQz8guCKE/PDDD5Ikndw9jZKYZXnTMGhM1ut1q9edrteswMuqomsaz/AMwxQYJVl6cnLy6sWzNE4IIYqq3k7Gnh+WBNM0w7LsYj6XJCnP8zCIb69uBUEAACRJIsqqpKhrx6EZ6IdxkCZXo1EFQKffKyuS5bmia7uqUZZlFMEAAFXXWJZN0zQMw11chOAKISSrShhHu/SCrhsURZcYi7K6Xq9xidqdzmy+2NmRWJ4jFF3kCEKYpTnL8oam8yyHEUqSRDOMJMveeuutq6urChDDtnZ3zJ3FAgAgSRLPsjRN5xgpplpRVZqm7WZr5y8tUb71XUVX8jw9PDw0DANCuFosd/CQP3z+R1VVEULr9VrT9cVqqet6FISEkF6v5/t+URRVVQV+JIkKDSjbtFabDS+KvV7vxYsXtm3pulagkuHYnbk0jlIAQFUhz/PyPLdtOy+LMI4LVNq2SQjZ4XRM04zjOIoiSZGjKNjRh3YzdJ7lkiQpK8xx3GKx2B1CeVkSiqrX67PZjOM4QRK94J/EUhDCOI7n87nrulVViaK4o6x0Op3bm5vtdsuybK/XAzS9Xq8tw1jPFjfnl/1uz9A0imV+eP7sZnTrR6GkyHme9zpdieN9z5V5YTKaaorebjU266Vpmo1G45uvvqhbNgPZi6vLRqPGMTCKorLEiqhRFXV65zRP8sV0oYiSbVqGpkRB2Ou2RYGzDfPjDz80NP29x+/QNHNwdKgoyqDXFngWY8wKIgM5jmEhQ1gIMMY0TY+uLls1++rijKZp264buh0mqVlv8KIAWe7w5J5s1TSrfnh0KqracO+AE6Tr69swjBVFm01miqRgXC6Xy6JAvh9iCrjBdrN14zSJ04TmGArSfujNlwuWZUFFbm9vCVWFYcjz7BtvPjBs3bJMhuOm06mzXl5fX0uSJEjikydPMMajyUTV9fPrq9FkXKvVri+v4jgGACwWi9vbWy/22/3e0tnMF4s4ji8vLyM/mI7Gmqbxu+xDUbTb7V33+Pz8nOf5fr9fVtgL/Fa3Y5rm3t7e+dWlZuicyJdZQvKEKgsGEFkQFUUpikKVlZ0qAFSVJImqJmNcRlGQ5AnNE3jQH/baHduuEYoK0yRKYt00PM978erVb776/OunT+ezhalo//a//euvfv9Hb74oovD29rbebmcAa7bG0iVL4/tvvumnyfn1ZVzEf/8PfzMd3xiyZCna737z+1//w69/83d/H2027nyeJenTZ883nifKapYVr169+tM3X41urlvN5nq2WU8WEscG/tZLEi+MFJajk1whUjL32Tj90aM3FVbKskKrGYIh5xRpDAd6q1nRcLtyi6xo7A3cMrNaDdfbVlXlOA6pKFygF98/+/0//DaLk81mxUpcmEasKKiGToGqZtnT+frw5D6mKJrjkpyOEpKlhGdkkDFnT64NoyGrxmq9nUznWs268/B0vJi+PH/e2++plrLazAtcaLbpx8lXf/rWMG2OFYIwfH129t47795eXmVRInA8AKDC+PDwsARVVOY0RW0Wy11YxfM8imYa7UaaRKfDPQUK/jZKc7yNw7W3+SfrKYA05DRNg5A+uHOycLZxmjdaHZoGse+ZvHjc7b58/kw31d22XRZEhuFWrs+LCs3AyWzabDYBrgDB8/m81W4XGJU0QJDyfb/f6a5ni3azFcexpCi70EWUJiVGFSAZKhANeFWWDG0bh7VWYxv4682m3W6nadpsNoMgSNO0btdEjkdFOZ3PDNsKk1iRpLplh2FYr9cBTVcUIIRAijYUBRUlJmS12RBAEQg24bakK1mUSVaFfphlmSiweZqqhukGwWw2ZyDc6w8gTVO4knlhOZ9bqq4rKo2qsigsw7y9usYINet1ihCCqzSK683a+dUlJyqoogHCNEJhEk6Xi9XGY1hZkKXZcjYejw/2j3Z8sTzPe71enucszzfs2nKxYAXertc4hkmi+PLmutFubbfbOE6bzVbkhygteYY/3B+eX7zO0+Tu8VHkbdMw0DQlTzNntnBmK5yXIicCQAuC4Ps+wzDb7dZ1XUEWkjzjeZ7jmN2bnKTI/b1hGkd5msm8kKeFunMtUbRimI7vZahs15uB6/muJwvS6fHJ7c1IUOSKpsI4Xq1WNMWUBV6uV41Wc+clD8PQ930agND3dU3d2xvu7e2tVqvxeFyW5c5LIytiFAd5GuOy2GxWdt3SDCNOE8/z6vV6HMdh6Fua6q6WqqT22n26REUShWH4+vXruyeni9n8ajR+7+MPK1IUZWLa9v7+PkOwzrIkT6PQP7l3cvfB/VcvXq4W60ar/fU33xZpMZvM/7u//rfvvP1u4ob+fON7gWEYg1bj/bffUGUpTVNdN53VWhYZhq1c1xU58bMPPozWa9s2a03bdbezxVKQpRzl/5v/1f/6048+/e7ps9VmW6t1wjAJgujBgwcpKk4e3i+LKs9wo98/G90wHDsc9k1Tb/c7Vr8pNkwfl5gXckgXFGl0THe7NA17/+AkSFJCgShIOIbbBv7J/Qcf/+jHrV5PFmRFVN55573h/p6oa0GSMTyXZcX+/v7Gc0uC7z982KzZ+wfD1mCwTRLFUHhFiPJicHRk22aWJyf37jbaHUjRlqKtpnNI0b67rTBu1Op1u9aq1WuGGQdxnuTbzbZCpN/qSay4ni8MVXt58er88kwXGZWHvUYd5wWgqiSLBY4ps3zn+5UkKc3iNA8qqoAC4CVIh1mCqkqSJIaG/X7/6OgOxwo0xai6qRqmHyc7l01VYp4Vuq1uq9FkIfvGG2989c3X55fXi+VqdHOLS5TnZb3efP78eZHnJ/fuPn78znq5uri42Gw2sqoKImea+r1797rd7mAw8IIIsmwQBLqqQUhXVUVRpN5scJI8mUwcx8EYU5BGJVkvNgyhgq2HMXYcp8xTkReCIBhNJ+dX13atJoo8qJAfBiUBo9l8sL/X6Xb39oYcx+GyJISaTVeBH9WarS+/+dqPI8jxcZG5gbtYLv/0p++CIJI05dmLZ5vliqOgrpk8LwKGEmVJVdXPPvvsgw8+mM4mJ0fHd09POYbt9/tvvfWWpmmff/45TdOoKD3PK0m1f+douHfw7NmLk7uniqJ0u935fH64f5CEkeu6zbqdRvH5+WvVNCDLBEFAEXB4eLgr3NuWNZ9MVVnxNo4myv42EARJVeW8zFieuf/wgWaZmAZ7e3s0TU+m05OTE0mSAs8jCA8G/SSNlvOFYWi6rkOGYVm+AjTGuG7XJuMxxtg09SCKOI7jeZ4X2K3nKpoKKAoDwgnCbLawDBsXOM2zvCxEUYQQVoSYNRMyjKrrVr12fn2l21az0/bDMIqiWsOmGSgLYpmlqqowDDOdz3YX4VajGcbRnTt34jieT2eyqqR5pkiSpVumaYZhmCRJEAQcK+wfHLiu2+l0OI7bzRx3RYTVakUDoGl6mmWKrImiuMtK7rrKuy+coqhg6xVZKfECxrgoCkmSXM9jeaEoCpZlVcPkRAlVNMYkDhOe53FRirLa6Q7cbUAAjcpKlNXzywvbtv2tZ2gapGgvCMqy9Dzv9PS01+stFgvbthu12sN797MskySJYWhRFFbrZYl3DBB6h04sEFJVVRCEIssBqBqNBoQ0wzCO40COFQRB5IUiywVByIqcZfgd80Dk+J3qa7vd7tJBO+zMjiwkq/pu/2FYplWzAQA7LxUnCmdnZ5Ik7ZiRh4eH3W53d8MdHuxPF/P7p3cVUbJq9m5kLwjCYrGgabooih0qB1SVxAt1u5ZkGcNxpm60Wq3dNXC+XHS7XUKILMuApnTLlCRpf39fVORvvvsTx3EIIV5g2+32crnkBXZv2N/ZV2zb9oPt7fVlHkdlkeVp0ut1xtPR1vOOTk5OH9wfTSaEovf3DoMg4iXx6dOn8/FE5jgWMjc3NxfXF2dnr9rdbrvTU1Xto48+oihCQ0rXTdfzZ7PZux+8j0g1nS95WTItKwiix4/f/dv/+B9lVX///fePjo5s2x4Oh1mcvH59DhmOULAoCkEQijxv1uuowDwr+NvtYrEADCubZq/Xe/ToUYmqtbOpANZMA7KcZdXsWkOUVZ4XTau2f3D0/MWL333+RUWzGFAMw42nU0Az1+NJRUO71iCElCXmGLZZb6GiWK9WCKGN54uqZtZshmMhhNcXl7qu9zrds7NXl5eXFEVpmpbEsbvelGWZ5fl8uUjT9PGjtxt2zd04EEJeEhVF+eabb1iW7Q0H3UH/k48+VgWWrXKVpZbTqaaoEMKyKDiGLYuizPI4jgkFaIZKi6yscgAqVOW01m1iDi7nS4WT/JU7Or90F6swCFzf2/jb2WxmGfZ67Tx/9fLp06cUZLdekhb4ydPnlxfXeZJrol5heP7y6ld//484QwBVLM2IgjaeLi8uryHH6qb2V//Fv2zs9b0y9bNkOp8JHM9Dxvf91WZj1WyKoniOidPk+7OXAU69ONRVhWeZ09NT2agZre7C3yhN89vrs68uXq29jbua1xXz40cfGpp5eXl58ez5L3/2Z//8L//zh++/02w2F9PZN19/pary8Z3DZrOp8PKDew/1er05GLT39pr9vmSagIFlVdYa9kcffpIm+Y9//qOyymUGFl4QbTdB6FA88Yrt7WoU5tvb29c1VWRQ/tbpydmzF7/6u1/dnF/hHMucNL2Z2Lp1cnRCKPrJixeSJA96g/l8TtM0QbjbbG2322anfXBw8Kc//ck2rYf37s/nc0BTuKp4Ubi4uFAUZbteL8bT44NDCOF8vShwcf/kThZEWZwMh0MvDr/49pu8QrVG7WZ0TdPgnXcfRaFHinzY6+ZFFoRRgjCiaVUxQEWvFmvIsXGazJazNAzuHh6IHOtsNoZlMiK7XC77/f7+/j6AgFCg1myIsiSKoue6SRTXavU333xrOh0XZXbn7slkNvU9TxZFhFCr1Voul47jKJJEUVSWZWVZ8gKXpSldEUs3aI6db1ZxEFqGUSH85MmTZrPJ8pwgSWvHydOcBfSzF8+Nuq1ZFi8KvMCkSfz+e+/macbRDA2o03unFEsvFvN2uw1oNkkySICzWUEWnp2f+2ksyFIFiFWzDctKUZEkiSiKDM89e/7CsC3Iscutk+OCokie51eXt6SiGQpyHFcB4odxvzvYOk6SZ4puTCcrXbcwxgTgPM/zPN8Bp2zbZjg2LfI4ji9en/U63eVy+fr168Vsbmo6z/NFUfACVA3ZzwNGFr0wcDZbXlYhL45mi40XNBot0zTXoTs43mN5Koq8qqp4XrTN2mqx1jR9F/3Udd02zOV81Ww0dodukiQs5BRJvboZGZY92DtwVps4jCM/XM6Xk8kUYSzJ8nyzCqKwNxykaaqIMgvZ589foP+/QDGJswqDs7Mz27aTJKEZSDBKolBS5Nli3ut21ptVFAS2aVUIe55XQQoKHCYVxzBhEEiS1Ot1tluHZWjdUNOymC6Wi42zDgLXXx7d2Z+sV54fVVVFUZSiKALLgapURXEzX/ub4MHJXY6h8jKzaiYuEcew77z9Ls3znG3eOqvOYNBoNCbzWa1hN9p2WkSj8cWnn3486PcFUdoEgVqvvb689r1o6wWvLy62fjoYHnOCVBLgV+R27Wwcv9Xu9PYHV6NLBtAUAs5me3F1SQPK1DRSZKvZbNgfZmn+7NnzH374AVeFyEGD5QVM1RWbo9hBr+cH3sXrCwHyH739VrBaGJJkGTbC7HSyevny+XI5r6rq7PyaV/S0rFzXDcOQZXg/iKI0S0sURRGqcK3eTIpyMl+wHOeuNw9P7u+32t9/9c1quYniFBc5wkVaIpYXLUWjimJyeRt6PsoLQ1UqQJIyb/d7rCQIqoxpMFstNduczOZRkg32ho12Ky8KzTRs2xYkMaNJTkpblt+5c9KvmZ26SQgBgBYgSyMECeEhzXFcRYG4KuOq5EWREAqhihCK3tvbq6rq/sMHq/U6SZIXz1/uFL2GYYmiXLfqk9uR4ziCKEOWcV33/fffbzQal6Mbs1F748HD2WQeBFGr1dFkZTqd7g8PZElN05QQMptPHj9688GDB0+ePwEsPL++sm17fzhAWWZqalHkBSqnsxlkmZOTk+Pj43/5r/7qwYMHLAs3m0278f9j6b+aJMvz9EzsaK2Pa+2hI1JE6izVXVWtRvYIDAkDuEuBhdFII4w3/CS4IUijLWALDoDFAjY7AwwGGNXVXVqkzgwtXOvjR2vNi+BncDc7f/u97/s85bu3DzlefvnmeKmr15P+7q3bLMcVCjLPsKHri4JAUcx7731w795dnMR5kbMsK4+jZrVarZaXq7lpmhAENVpN23FL1aoXh5V6LYdAU9ObtSaO46qqlorFH3300XIyQwCQoijLNkLfCzzXcSzPd3AKubg6X68WaZqenp793d/+/WAwYAiSpRnP9jAMazWaoeNMJ5MHD+7RNDXo9VAY5jjO9zxtpbAU7fqeFwVLZXXnzh1T1xfz+Ua3vb25lWUZgqF37h0ahrG9tdGoVXrXA9d1N7e3TNd8/vw5x1DFYnE6nogMt7u16bq2ZVnVahVF0Xdv3nIcE0WBspjjKAZCAC+LOE2RBAHEaVkuMCSF4gjLskVZxBAo8oOb2SfNsZVaVdO04XQyHI1QFI38IEkzJ/ALpVKhUFjOF1dXV61mMwiCwWBQKhRYmrZ0zVDUPI0pDMuiCEdgGMglUVwsZp7jyqLo2k6cRKLEUxQZx7FlmDRBNpvN0WiEU+R8uRBF8eb+vrm5rRvWDVFnsVhEoX9+enYDI9LW6mwyJTAUhuEgCCzdwBEUQ2GR4/M0e/r0KQLBURThJGnalqKueUGo1GuaoftBcHh4qGqa53ntdtuyLLlYIEmy3W67lo0iMIqixXJZlKUwjhq1+mKxcByrXq+bluWHgRcEBEGMx2Pf9QqSpOs6AACyLMdxXJQLFEEAEFRrNDAEWS+Wuq7DGLxUVs12C0EQ27bL5XKtUUcQJE4TCIEhGL6RE6Rp6vjOjfQVRVFNNUaDQUkugjkQBGEUxcvZ0rU9SRCUlRqFYavVuimqkSRZr9dd3/f9gGVZCIajKBIEAQYhjCTcwMdxHEXR8XgMYygr8IqidDc3rq+vbw7u1/1eoVAAAGDYH7iuW6lUbrrSOI6TJHl+fo6j2M7W9nq9zvO8UqtSFGVZ1g3zslKtxnG8VhTLMAmCGI1GLM8xHNdotmVZJmkaI4k8zx88eJADAI6jAsdFUXT//v3hsB9FEY7jrmMFQVBr1hAcM23j4vIMzFMgTbIkqlWqJEWMRiPPsZbLJc/zDMP8+ONP/+5Xf3t6dMRzLMOxo8msIEosxxAExvF8lgOT6SJK4q3dHRhDNdMSJHmhrGzXKpSKhUIhi5NKpTKbTF3T4AjcsQzLMCPXp0iyUiptbW/s7u5eD64My7i8vATBXFXVN+/eAiAoCAKYA4PBwHUsEABoggRA+L0PP6rX63EcdtrtvZ3dUqmUpimGog/u3z84OIBBCATBWqN+/+ED2zYtw6Rwolwur9drGIIURRkNh8VCmWBp3bYQBAFzYK2pOI6bquY7bq1Wc4MwiCPdMi3XoRnGcZzlcnnRu5bLpXqnpWpaEAQbnU6app7n1ZsN0zRJglivFIogkjA6e3c8uroSCHSjVRcEyQmCHAJRHMuz5Gaay9CsIAg4jmcpACRgloJ5BkH2dP3o4F6UJLprLw0do0mMoRiRD/0ATLLt7gZF0Dt7u6zAExRFYIi2WKzmM74oW4H3//3T/ynwbZHj+/1rRVMq9dJcWX3//HvfNUx1ubOx8eLZc44li6KwGE85krV1YzGZVgry4Z0Dw1zzIgMTGIzjMIKhMBLp5lajvrWxubm9VZRLf/tf/+b89GxnZ+fO/YdbOwf7G92nd+6qC0WWi2JRPL+63NnaznNA7rT+7V/8h//wv/y7+fA6sExH10mS9IOo2WgTBKHoWrVVb9ZqFbl4fTU4fXcWms4/+O3fw0BcYAVTXX/+t397+uKtQPMwTvh5DoJ5o1KtyOWiWOy2NyAAXimWF+alVieGURBCdM3kWfbB/cMoDZfreaNeiRz7+vwMBYFaqXh1caat1SxOdje2lvMFTlMgilAURVFUpVLBYGQ6GK0Wy7v373lBsNbUvYPd6WrmJ2GhVJTkou6YEI48ev9hCmYADHU6GyLJsBBGYwiGgq7txGFYlgvL+cpyHSf0A8/BUSTOQhDJIDBjCUxXVjCQl4slXdcd38Mpkuf52XjSaLdQFDUtK8rSjZ1tqSB7tpNGMYJjjMD3J6PFStnf2UUByHVdBIFEns2SBE4yGsF2u5s0ilMItt3u5kGEgtB8OivIMs9yo9GoVCnneR7YbkEQCZqyfY/n+VF/IBeLDMOQKKau1lEcy5WSruvVYsGzrMVshmEYRbMoDIoS74cehMAIAKIgAoKgIPAYCkehn+f5er3mKNpYrTmCAtIsiiIAgWmaxnF8rWsAhhzs70MgKHCiLEqxHzAEZdiWG7iGuqiUxKLMjyf9lbYaLyYIjtiuhUA5AuUZFLuhE8bx5uY2iqK//OUvK5XKjeUjz3MIAdfrlW9bvuuolmYHDo4iPMtgKGwYhiQWPDdQVypHM2tVNQyNJAkEggSOi6Ow2W7keZ7FSR5lpm65bug6/qP7DxEIiYI4ywDXdmzbrtVqcRDCIMRQVKVcu7i4UlaqLIqqooAgRJIUQzIAjDh+sLmz63kBDKNRnDIsnyVpEsXFcgmAQNf3ytWKoigURUEQDMMIjuMABKZxguM4RhIogcdhuLO1lcZJHMeCIGRZ9vr1awLFKIaOoggGQI5mbr5/0/kERMBbe/tpmqq6trGznaZxGodJHBrq2rKcPMlv7e6enhwjOIZR5OnpaZ7nR2fnYZ6necKypG6ZKEUkSWJYerFRqTZr/cEVgyKZpqm96/7p8XuPH9za2avKxfH1JI2A71+8fvn2mGAw19EiO8AzkECBo3cvwsC9sZ9jGAaDkG3ormW2m40gCtM8Wy4VDMMmixGA5t2NVrtee+/wAZJCURCDIJiEHpKlNImOB/0MzFII7qlLWGIVR+dkninKYrmU52mpVFiZ5nJtMCRVKRd7w6uj82Ocon0/vL6+FFiSgbHVZALD4GQyGF5fxb4nCzyOgFfXZ77nMDjGYaSlKQSOMhx91btcanqMgBCBVZp127YJBBUZLvJ8HMchFHtzdoTSeAqDKQzCCBKGoed5LMsGQfDq1asb1zFBEJ7jiryw2ekqy9X+zm671viHf/BHdUFqVyoCz7Es3aiXoSzZu30glstTdQ2SuB+FCAJ5nu/7fuT6YAIQMIlAJJTjaQxB+lL97G//rl5vbu/tshy3c7AvSCJOEBRFPXn0uNVotjptAIGjNLFd5/j09OXz7/Z3t0VRxkm63qzleVoqygWBZwjsu2++OT8/50VBs0zDNavVKkczX331laZpaZx4nrdS1OfPX9IkdfTmNZgmpmEMBn0/CkaT8Xg8/vLLL//sP/yZKMo72wdJlHq2w7Pk3dt7G612YLvXF5d5mn3wwUfnl9cZCDEMk6fp6emp67p37hx6tuPZFgaBOxtdSRYQHFPXaxKnXNdNgVwQRBzFCYLa3dlp1aoEhoI5QDPMTS+bY9g0TmarGUkTFMfSHBtHkb7WXdspFosQBEVJbHtukqUwij169EjTtJOTo1u39nmeDTwHhsFms75SFjRDdtpNjqHSJLo8v3jy6OlitRzPF41O+83RuyDwtje3YBDUVaV/fVUsFg1NHw0nMAw7jqUZ+mQ25RgWR7Hzy4tqvaZp2rB3bWkajiJJmoIQxAusIAhxEFEEUSwWuxsbN/NXgiBc25mOhlAOdLvt1Wpl6PqdO3cs21Y1wzCM27cPLs8uVVXNsizPcwDMwjCslWuyVHQce7VWGs22JEnX1/3VclkqlQAAYFmWJMkoDkoF+fryXBalKApMUydJnCbIRrUWBaEfuB988EEKpEEQJHEMASBDUhCEgBBQKBQsyzJNsyDJkiQ5vqfqOsdxcRi5rsvzPI7jJIFBEDIbT2RZxjEkjZMkiouybBgGgsB729trbU3TdJIkPC+auo7BiMjzqrKO49g0DIZhYBgMwjBN09lslmdZQRBNVQPSjGEYBIL7/Ws/8t9//31d1XAIgWEYBPPZbIZhCEURrmszLJXmWZynn3/5RZKlPM/f7EtVVd3Y2EjTNAzDRqM1GAxIkgQAgOU4QRAgCAIBoFmt+bbDsexkMgl917etLIkYmlyvFIIgCpLMs6wkSRAEyaI4noxYloVgII3CYrFYKBR0Xa9UKgAI27at67okSdVqVdO0QqGAgFAQBL7vYxhG0+RiMSMILANSiqEhBCRQrFVvGIYBI0iWZWEY3vxShmHcJDFxHIuyFCUhy7JpmkZJstY0mqYZinZdL46TDx4/RWHEcZw4S3GcdBwHRVHHcerVBkuxp8cn9WpVluXZbNbr9SiKWi6XnhcUpWIQRKPRSBR5hqF6vV61WuU4LvD8H//ow1a76QV+oVAAQXCz083TDEXRar2+u7sb+m6BE37rk598+tEnx2+PT09PJVl++OAeDMOiJP+Tf/o//LP/6//lwd27/+d/8j+89+jxdDR8+vA+x3GDYX8xn9fLZUVRQAAgMTyPIwQAWrUaSzNhGKIoulzODV2tVcunZ8eT+WQyG7dajcF4cD24uLi+0Ezt2fPnfhTyBenw0QOBYwxtDWb5WlFazToIZJIkoTimq9p4MHz04D4GQWkQxWF0A/YaD/qB4yIQsFgsYARUNYWjKY5jOp02hECVYimLQgxGQBAsSHK73ZZLRQAAwByIw6jd7JiaGQZBsVgslEswgkgF2TTNMAwlQSyUS0meEQSl63pJLtQr1SRJlsvleDxeLpeGpp0cHRdk2XXd4XD4q7/7+9lgZKs6QxKiwIa2xZIEisIbm5tyuXTjZSRJGoIg1/XDJMZQIs9B3wt9P8hSELp//+HHH3+KI/hkOBF4rlav2rbZaNbqtYqiKP/tv/1VDuVuHF5PByRPy5UCI3LlalUWJRSGkwzyAn81m5AIJBH073z6sz/4vd+vN1pyo5ah2LOXbxCcYDj+3dGJ74cQhNhJnKHo5998pa7XRa7oqNYvfvIphkDTxTRDoHq33dzcXK3NF89e6cr6gyePbh9sLibXcOD8oz/4w08//S2Cl9+en+7s7Zblkq7oURTIPG8vDSyB9/dvFQqFoijtbm5cXp699/6jxPdlTvjwww/Xa/X8/BInmEq1XqlWoyh6/uKHMPXPrs97k9F0vXJjX7N1IE95nrV8WzVUAMhwBCYwZLVaICiEYmCpKJIUGsTRUls32k2KJo5ev83jpNVqrHVlNJ0xLA+A8HWv59j6R++/N+wNLcuTBLnb7b45PUZw7M6dO+/evGUoeqPTtU3Lte2dzR1dNcqlKsMJju+gGAQlGZzmBEkulFW71cBQOEpCjCJBHFcsK0/S0PdyCMxBEIagy7NzguYIkrFU09aNjW43iIPj84tKrVqSJWU+o2m2XK5SFHFxcbG9uUXhFAzDJEmY6rpUKE6XSy8IirJ8a3evf93zvaDb7Uqy3OsPHdfPknwxm6dAujZ1mmOH45FqGQmYR1lqe65rOq1qm+f5H55/H3phuViicNxQNQRCwQTwbQuGwDj0KYqybXOtKtvbWziO93o9x/OL5UqSJLHv6WuVZVkIQgLHlXjO8xzbtpMkgUEQyBJVUzAMsz3H8lxF15IMAHOIgKCKIMA5QOGUxDCJ511cX9qhx/JMGIa988tWtU6jOJgAhmEhGG663sXlZZESypzsWma91dzc3IyiJPR8iRdq5UqSRMfnZ24cIhjmh2G1XITAvCwVYAB0ghCnmdV81aq3wjiZLleu60dRZFuGMp9IDFUQOd/3bx4HtUoJyTIGxWmc0JX1Wl3ZnlkqF9I0JklclkU/8U3PpBmConHT0lAC9fPIcM3ZYiFIIozBuqFyHMOytGkZDMMwPBf5Ho2j9WopzVOMwCiWgmAwy2PXsXlWyFOAoiiGYUzdQmGMxPCrq6vQDwSBG83HhXJhPhlHvhfmKS8XMAwfD0ZBGIuCvJzOCASO0iQBADcMSZq1LAtH0elgGph+s17P0yRwnFq1jODYaD5lGeFg/04cZbpmfPLpT/rDQRbFH3/4kWnYhmFttKsnr196tqetdQyFyrK4mM84mplPlxen58Px1PGCN6enX3/3PIwAFKdwgTUCczrr9Xsnm1sbb1+9/Vf/n3857g1//au//+GHHw62b2lrI43jP/zlH8gsQ+EYT3EP7j+ZzWYCz5Z5btUfMhiWJSmOErJYyML46ury5fm7ib7sbnUd1+IK0s7hbalWxlgSo/E0T9IoNpZriWKbQrElFSWEdHVz1OtDcUhhaBRFeQ6q8+X9/YMyJ7I4ieIYRdOdVmNva5OhWIqiSJ7t7Gws18uj43fnV5c3mgEEQWReToPE9/2NjS6FoRJNYymQe6HnWFmWYCiBYkSYJrplOoZZ4AQeIyPbXc6ngiBgKGHqlqaq+lpJPI+j6Ga7LRTkSqVSrZUNQ3N9h5UEmueiKAKShKYRx1ALDIPmeRR6uqGCaTYdTkAQtB0nAyCcZkiGgXAUADLfcyiGzmEEonkmBYHxbLq9uYVAwGTQ73bb3377reM4zVb98dMnXuDWm42d/QNOFgEYUlVtOh73z88n/T5JkjhGz+dzHMVu7e3ubm1ZrpOAYLPVSXIgzvLnL15pupllWZakHMf5QQRiiOf7pXK1Wq0mUTQaDGuVcqFUjNNEkAuVeqNcKkVRNJlM3rx5c3x8HEWhuloN+te9Xq/X6wFgVizJ09E49P35aukFAUWSo8GQoGg/ilVD/+Lrr3b39y8vL5XVytB0Zans7+x/9PR9OM+Wi9nx8bEX+C/fvrlZCZqmSRAEgMIZkOM4nmcZCMIgCPf7Q8OwogQIwhRBECDLCRzDUQwBAZomj0+PcIJoNFrTyXypabwgNJvN+Xy5ubnZarX80L+6vux0OlmcAADIMGy32+1udd+9e/Pk0YMkiebz+ZMnTzzH/Zv/9tfddpsk6clksr+/s7u9NZ9MgTRnaNo0zauLyzt37sRZrDuGKEq+408XUxgGWxudtaZORuNWqwWh2GA8IXFqe3NrOp0Kknjrzu3laiXJwo3V88b8ee/wUFUUIMsRBGFpGgJAxzZ5TqRoFkPQ64vz3a0d3/cty3Ecr1Qq0TRrODYrCrbjfPDR+2EQUxQliCKG47bnYjgOZtDZ8Ylje6Ig3OzyUyBHCVxVVRiGDctyPLvaqM9mMy8OozSaTqdgnv/kJz+BIGg+n+V5DsOgIAiD0YigKFVZQyDIcVy5XEzT1LIsCIIYhqFpGoaRVqcNwzAIQigEUwSZpimCIKVCcTqZWKZZLhZZmkFRlKbp7b3dmxofBAD3Dh9gKMGLgizLOIbpa1UURcdxVupalmVJKsAQNOj1l8vl1u4OxTJBFJYrxetBP89ziiTPTs5BEGR5oVwuT6dzGCekggxBEAyAKAx2mi3XNGEAAACA47gkSbS1iqIoz7O+GxQKBRSFURTp9XrNRi3PkjgKWIokCIymyfOTYwRBOI6L4xjBsY2NjdAPaIKkKEpV1rZpoSgaBH69XrVt27XN2Pc81w7DMIpClqZ4lgPzFIXAKAzLxVKtUoVhWFEUjuOKxUIYRjwv7u3txXEsSny9XicISjdN27Yev/d0tdZc3xMETtM0TuAFSSRxAsxzSSzAECpLEpDn49EUAAAMR+LA393Y2u5s+b43nU4BGJJLhb/+m//WbXeW8xmY5SiBQxDUKFXU5ZJleZyk8jSjKUJgaAyGD2/tUyTm2jqE5NV6PYij/uBa1zUQyGiGqlQqv/vbvzPp9ViSZAjq+Q8vhsMhz/OL+XI2nhWLxR+++0bTNJogV6vVdDRut9uDwWA5nfE0hWFYHMeL1bJQKvphYLtOtV4vVcqyLA9HIxxHp9MphmHVWv0GexmFiWnatm0blinyYhzH8/nS9/3JfOF5nh14QRQhIPLq+StVVdvtLorgpxeXcrlEUBSUA816vd+/VhRlPp8P+sMszWez+cn52WqteJ5Hk9R8Mp2MxjRBuo6TxgkCwSiMgCC4Xq/X6/XNOJylGQxGcAQNfJ9jhSAIapVSp9lCIJjCqSzLxrPpdD5L86w3GFiWRWBI4LlbW1swDDebTQgCZtORxFOpayuTae/6nKLwgiRHUQSCSBQlfhgQFH5wcJDneRAEFEPrpunHETTWZ28u34VJ/PbojczSzXKRpuknj98TRXkxX+EULshS/3rE0yICE5blsIzsmW6jJH360QcYjFiOu3fn8LzfNx3bcIyzywvH82gc05UVSuD7h4eyLL/33nuFogQA2Y8++qBYLJbL1bVmmL7d2epOJhNDNQ72bqVx9uu//9V8NIo8i8Dg1kbXcNxWc0uQynaang2uTWMduZaA41ngR37wo48/wUiqs7u11JXmRms4nQEIpliG4fuXvcnF2RABIQSCZE4iIXQ+6pnrWUVmgTwxPM/0fQRE4ByoFkosxVqmF0UAR0sUzoFRLjACiZE7e7ccL85hnKH5klwaXQ9sVU993zWNRqMxXcwNx3r49KnlR7YfaapRlAvfPfvB9tyDg4N6rXF2ery7uwsjyOs3b/Z2di9OzzzP88KAxIlWo3F2dpZk6UcffKAq68F1r9tu+657fHRUKJflYjnwvP2d7WKp8t33z6zQazQa1lorCZIoS3bk//D8Wb3ZqDXqnudBELC3t6dpmmu5W9vbluv0ri9bzeawP8IRdDGbg2DearVOTk4Ejs+yLMuyi4sLmiBrpbKmabZpAVneabYsy6rVammatttthqJtz4VJ3PAtgqW/+OILDEFgEEqi1PHczkaXZfnlcrm/uwckKYFigeeDIMiLgu06cZrABFZq1a3Amy0Xtw/viuUizbE3S90vv/wSQZBWu0HRGAAASRJV63VdNzY2tpIwMbS1aZoAAIiyDCFIbzCiCEpZKOZah7KcZ5g4Sy8HvRyGFF1bqmucIMqFIhxnw9MLBiNwHA/z1IlDmueiJHn79q0sy3ma6bqO03itVQNBcLFYbG1tLRaLOIxM0xQEgeO49XItcnwQeNPZ7M7duwAARH4kcBxDUQgEur7vhyEIw4blgDlQrVREXrh39/DJ++9Np7M8z6fzWQbkKQRYvjuaTkiayvN8sVhgGFYqlU5PT8Ekw2DE9zxbNxAAvLW75+ima9kFUUIByDFMHEVDP7A142BnFwIADMOyLNOUtcBzMIgwFF2vVBmSwECAQNDr84uiKAsU42raYjKdT6YkQ/OSaDj6bD4BYWg4HBmaDgE5jmKr1SrPAUmSIAw9u7rgBD6Io+FsQvF0EqeT0ZgnSAIAQzdwHBfBED/0K/Wa4/mO6/q+H7t+6vuVUoFkCT/0IRho1MsonFMEOZlM8jwncaJ/1UdA9OjktNnuCmLBtdz1SjGVRUnCMcjf6laePr1frRcOHxy0u+WnT+6+/+B+5gbr+dpQjIogp55XKZV+9rOftTpdFCeFotzqdBzDwFGs1d3UXa/daSIotF5rDC1IBZnkmOF8WqhVIBw+vjo9ubpACIInmOV4+ubtW5ajMzdgQDQLkgIn3Tu4TcA4hGJelg5VJcLRq9k8BGEz8CrtplipSbUGQpBe4AdJQvBcbz4dLeZFuQxD6NvLC7okqYYaed5epytQTJ7k5XIVSKFKuXFw605ncyMHARDKJVkMPG8+nzuua4UeTOGmbUmSVCjJBIEpk5nMcCgEAwCQ5hnLCXkOxnFqmjrHMQzJhGFsO55clHIoNx0zBrIcgGzTgeK0f3bBUFSagkmScQyBgLEyGc56F2AU964vu902z4lxBIpyCcZgP3Auz84wCMQwLM0AGMVyEIJcy25Ua+PxkOc4kRdEXuj3eovF4vzkdDqZrFbr1XzFUwwMQAiCEhSL4oTt+rqqNRs1mqZJhgYR1PODPM+TOHvy+PFmu/XNbz7/+IOPWpvdOM/OLi9Oj0/OT06zKFRXCoHhcZrEaTKYTK+HI5bnPM+7vu7nSVqv1iAwN12DZHBd1w9u32ZYAYRQkucHkzFGYDRJiCznmtaHH37YaDQ++NFH/cGg1a4bpr7d7YgCv3fnzv7duw/uPfjo/Q80wxJlOc+yyXh4dna2WCwUdS1IIoDAJMUgEAxkIATAjuPQDLtS18Ph0LVs33WTJPnkk09yENja3SkWi24QLlbrYqmW5lm1Wp2NJ9989fWDB48ksXBxdX3dH2zvHhRlOQ79dr323pNH//ZP/y1BEFJBtlyrUCgUZPmbL74sSvKHP/7R2+MjGIYxGI/DuF6trVYrGAYZnlEUJYuSOwe3RqOR63t7e/vHx8d5nt97+ABGkMFomMYxx7CWZQEQdHj/7g2lr9aooyiqaetiscgK/PnZGQrBoij2+z2aYhEEkyQpCsIoim4KKpud7mg4fHj/QZ5lURRVyuV6reY4znw+53jWMAwAyHRdt22bpUhtvb65jDM0B4JgliQYhjXrjbWieWFQbTVs170h2GEEHiWxaZo0TW9ubmIYBuZAu92mCLLX692UPVzX1XVdEIQ4TS/OrySp4LougiAQBPGiYBhalmUEQQVBQBIUx/GLxWJzs2uaJk2TQJ5BEGRZVhAEt27fdjw38F2GIkqlimE5vmO3221FURzHucGFT6aj7d2tp+8/ub6+9mzn0cOH8+VyvlwiCFKvVI+OjkiSrlarWZys1goIgo8ePRoOhxiG1ev1ly9fQhBUKhUoilgul0mSyKLIcdzN2hMAgNDzL88uFXU9nkwwilRV9fDO3XKxdDM05QUpiiLHsg8ODvIcGI5G7XYbRdE0TmRBrJUrnuX0r3vlYhFHsWF/AOZ5US74rqeraqVUWs4XN3GrwHEcyxiqFsex5bgIBBuqdhPAlkql5WqdJDGKolmWcaJQlAtR6OMIWpTkOI6DIBA53nO88XRSrdYxBPUcN0uSW7duGYaWpunm7o7uWEkad1ttQ1fPz89xAisUCmmeoySBUSRGEDCKNJvNwHEDxw0DbzoasgxRLhcNw1AUpVgutdrNm2/YRndre2t3d3fXMIzz80vX8e8c3Hn69DEMptWyVBCF5XRyeXZ+fn5mWPqzH74ZXF7Yqn52fj2czkAwf/zwvq7r0/litlhdDwZhkq419cYSBaM4zXOKuqZpMkvyvb2D635/sVoxDHN+eWG5DgAhKEmYtoXCSEkugAgMIdjB/v7l6VkWxkCcrpdrhqbBLDd1QyjIiqYmaYriGM2xM2WZAilB4aIobu/tenFA8HSpUdVs4+jkuNFolsrVIIyjKLm8uDo7OUujmKYoKAXjMIJhWJAl07JKpRLF0FmS3WihkjwLwpBh2ShNTMculUo3bj8cxW6Y+EmaEgRRrVYhCJouppP55Pbtg1KpxIkCwwk0juVxVJIlDMMwGJPFgq6rz549i5Kw02lRGIZkqaureRihEJiG0a9//WvLsREcybIsTRMwB1AI9t0gyzLXddMcBAAIMq5nG4Xqozt3RJ578+bdeq2ZpgkCWRwFf/JHf9ysNFAACW17Ph6/fvkqjuN7Dx/AJA5hxOnZ5WA8EiReWa8zEDo9u3794tWLL79CguAf/vEfSaIgSNLJ5blhmcp6KTHMdncDBEFO4LM0zPMUhBE3igia2j3YZym6VqlTBB2G4d7d25u7W4HnZWFaazQtz339+vXm9tbJ5fmrd29mkykEQN9///3//L/8++vrSwhMsziSWRrPwsOd7WqlfnXZU2ajeqmA4PhZr79U5giGHD560tw+gHDWCSKKoqIwFDipUWmGUYoQJC2wzU6dExgQyjCCAGHo9du3hqGGvpEDEUwg5XZzaVtOkgRx4LlOFmfPv3tJE2RBKDTKzeO3x5ZlHd65e/rmTeaFG812o1YnGJrimCiKLs8vHty+i0PYv/7Tf/37f/wHmmX86le/+u2f/TxyfMu16p1WBuR5noe+Px2OGIZ2PPv86rrZ6fqBaxkaiuLlchWCINezi3JB5PnlcsmLIkWza0XDcaxer9muNZlOMRijSardbUdRlKcpS3MESqiqepOODsejk/OzTz/+JPD8m8sGg5PDXh+G4UK5pKsakOd5ngMAwLMsR9FVuUghWBLFq9WK5XlVNw1NM3Tdtm0vDOwotJMQxjGCYubKCkbRUqHI0sxsNlvNF1VR9tY6mGa7W9ux6y9n81Kp1NnYGE+nSZYd3DkcT+YIQuAowVDUarFAcWxrZwdGUYKgAACIPH+z3fEcq1IrZFAKwpAgCHEW8yK3VNY4ij29ez/xgslkUqiWG5sdVuZpXgByiKNIMMk6nc5isTg/P2+3WgSOT6fTdrcjlYs34MOD/du6aYa+zzAMSZIExfz6i1+Xy2XHCTzH51g2i6M4DvM8l2U5y4AoCAWOgcGcJtH1er3WDIplABjyszgGsmq1Oh2OLE1vVmqu4yuKgkCwKMq27c4W882d7RyANE0rSDIMQqqqoijearQHvX4UBA/v3wcAQNf1Sqksi5LneXEc8zxvmqbneTgKp1kMQGCj1RpNxkkSlaol07ZyFIZITCgXzMA1PAfFsX7vKg+jqlRmSYplGRzHXNenKEaUi7bnmroGplkWJupsudlslyRxvJhEWZrnOY6hc2VdbdYxCusNr9M8AyBwOJ+GeSoWii+fv2IYplFtJmHCUTSUJYamIDBRLNf9yNdMDSPQ3/vDPxrPF73hIPBcnqUrpTKKon4UrtZanILX14M0yDiSxzHCdYNqpfGTT3/x9METDMW37u7dee/BcDm57F/ptjOZziEYqTaalXrNdj2eF03L0U2DFXg/DKMoQhBU0/StnW2MwOMwjLxA10zX9QVBkmXJdi0AyNI0Xy6Vly9eFwoF2zLms8l4PIYBEMmT3XYzMh0MBDgaq5YljmdKpUIU2DgK+I5t2ybO4cP50I0cEIVK9RKEAASCjq+HzWq7Xm+3N3YoitnpbqqL6Z2DnTBw3rx7jTNUBoCT8YLjONtypos5w7EEQViW1e50dvf2oihKk4SXxDhNDMO46W6ESbxQVpZtc6IEIbCyXj5+/BAEYRIj64UqCaM0hkBZKouFo+NzgqH3DvcXyuzq4ownyUC32tWqSNNQCsIggqNYkkQ0i0epBwMgCiFBEGc5kKYpQRBJkpAUBW1v7w5Hkzdvj+YrRS6VJ/NFp9PZ39+/9+jhtz98f+PuYhimXq8SJA6jyIvjN4ZrW54f+NHO5pZj2bqui8USiOOX/VHoB57lvHv95ujN26uz81qxXK9XDw4O9vb2ZqMJkOWT0fhmXAdhKEnR0+mc47goCB3LbtWb1Wr9xYsXAAwlURz6wcnJUX84wHH04uzctZw/+OUfyeVKAoB3Hzzc399jSKIg8T88+04UeUPVLi8v4ywFEdiznZcvnnW3N0gatyzDNPXT01OcJCAQIQiKwvBqqdho1AVZcqOg0WoSNJFlSZrFYRJCCLhUVuV6GUGQyXAU+QFBEIvV2nGcPEkhBIZxnKBoAIaGk8lkMSNJ+nd+63cdxwkCT+T4PEkRGG40WpZjjyYTDEeVxTJJksFg8LOf/fybb75xHOcnP/vpN19+hcHI00ePJ5NJFEUfffRRUZCyLKN5tt5pURQVB2FJlJfzRZRkk8U8iEKe54E0cyyboWggywqSnGXZdDodDodpmuZZxgmCqqrH745KpRJBELque553+/btJEkm02m1Uqco6vz8XNO0PM99x51MR3t7eygM+76/ubmp6frO/p5pmktF4TiOxInpaMzRzO7urm4anCRW63UUwWezWZ5mgiDohsFwXBiG25tbYJ4HQeBadrlY2tnYfPXiJc9yEAD2Lq8C17thT75+/brb7UqCcFPdQ1EYhKHZbHYztzk+Pr55y2MYdn19PZ5OHMexLMdxHBzHQRA0bWulrkVeyJL82fffQUC+ub17enG5WCsojtm2HQQBBICVUmmlKEEcVSuVq/MzEsNZikYJfDyZRFG0tbX16tWrSqkkiiKKorpuQgjcbLQtx5NFSRZFHEGzLEuShKCpJMtAEKQoCoURZblyHWdvb69arUIg8t13P2AEiRF4FEU8L8ZxHIYhThIMz+U5YFkWCILdblc3LJwiS6XS8elJGEccx4VhqOra4f17BEWdnJ3dwHbmy4UfBtVqlWJo0zT39vbK5eJg0CMIjOX5myxxe2sr8gMMw9I0T9N0sZi12o00TdIsY2mKIAhNX6/Xa13XWZ4jSTIM4ihKQAjiOA7DsEKh0O12tfVK17VaubK7vQkBOYZhOzs7s5USptHde3dM3UQgpNWoCYJwfHy8v38LRfDPPvusUq7dSIVQGMEIPMuA0PM5mlJWC2U5l2V5e3fH910Mg7qbnRTIv/nh29VasS33vcc/Ojo6cW3vv/9H//i3f/pTZTUf9K7+/m//hsQRAEhrtcpv/e7vcJKsasZ7H32kqBpF02mSe54nFWQIgbd2t+bzeXdzY7XWRFEA0lRR1xiGybzIUYwoiv9/QhQEKZqqWgYIglmWERTZqLcMVcMwrNvtiqJ4e3d/NhxXpEJJlEPXc0zLs503r19DOcDg5O29Xd9xQTC/dXv/wf37sshDELRerxeLxe7u7nw+ny1WiqLoqqbMFziCDnr9GylrpVJZLBblcrnT3QRgiKQp27ZJgpBl2XKd+WoJQZAkSZ7vTxbzcrkMQVAcx3me3gi5fvnLX7bb7UGv/9mvflWUZBInbv7/juOkaXx13eNEEQCAwPNuahSephc5xjJVBAUoDE38MEmyG1s1DMMkQUAA6AVBkgF5DiZZjhEkAMMQXi36OLby/ZGyen78dqEufS90vcAOw8Fiavk2K3GlevVq2O9sdXMkH0wGtu/xQqHfH3//zbdgHBME4UdhzlCIwNQ6rbmyfPfunWs71lxZ9Iaqsr7uXx2dnL47PhlcXZrqGsghAAJtz1UNTWSF4fUIhKFitQwAgMTJoRN+/9W3buR1tjuVanFzo02ROAbBu+2N1WQxV9djdfX5d197gb+YjZM4pDlWd90cp1eGNRwOKYparPQ0g2gCXc2HMARoKyWNg7s7OxyOpq5n6VoS+198/WvVXJUbxevB5WQ0ylNAEmSCoFIIyBHAi2LLcbMYqBZrKIpbmt6VyjuN7nS2EEoFJ4pOrq5q251yp+549g/ffncziVINnRX44WA8GAyiKNrZ2oiC8GB/H8OJarW6nMwIEK026lf93u2797Is+/WvPqsWCiIj/Oc/+/NaudLpdJwkfXNy2mzWXcs0Ff1HH/xILBfXriWVihAETccTjmZEQdDVtapo1WKlWJKHo/7Wxvb2xvZ0viBZ7tatfcexXrx6w7B8rVk7v7zASOLDH//o3bt3CILIxQJFUdpSCf2gVC6/eP0ijaPQ8w3LxEji4vJyY3uju7mxWCsXveut3Z00SUbDIU6RcZrolqXq2t7ubr1cS/1Q4oX1SqlWq3AOzIZjCIJwijw7OlaWqyfvPR1MxmtNbbfb5WKpd3W9Xq3ef/rU080sCosy57h6EAd+ElTqdRTHoyjBEIQiCN9356tluV6jOJ5iBNdJWrVNhmY1Q6/VakkYjYcjWRTv3LlTa9RPL883trayLJvPJp1Gk6cZ0zRXmpqmKcMwpqbv7e6maWqa5rt37xiGEQTh6upqo9OJgpAiSE3TOp0OmAOWYxeLRQyH5tMRCiP1agPDqOFgjBMoACaOY3E012l2GvXOmzdvlPWyVqkgIKQZeqlSoylWMwzTdVhZtgLP8b1avc5xnGEYBIaXJFlZLDVDb3c7KZCrpoHTZKFU7A0GEALLxYIbBiRLi7LEi+KLN68JhpZKRctx1rq2s79TKhWWygrFMZog1dXSWmv72zuuYREwutFoYHnO4iSBYjCOTVYzriCUauVCqXh91SNJOs9zx3GiINYMc7FcBmm01BQvjXMQYAiyf3ZJEcTNmEiSJJqmJ5OJxAvVQmneH0WWdffWXcuxQRC8e++eGwTXw8nm5g5Nc5ZlMQzVrLbOjy+LIq+vl4ZhoAje3WihGPTi3Qu5Vvzxzz8p1csXp5ezyYJguCCJ1dXiwd3dj58++O//yT/+J//0H/2zf/ZPD7e3Ll6/PX5xYpn+Ulmv1lq5VtVU1dC0ZrOJYsTZ5UWjVvYc8/LyGoDgNPYXywlN053Oxmw8vzw7t3Sj1WqgKBpEsROEhUpVlsUsi7zQmy8XNCs6drBer4+OjkajGUsLLM1hEJalYBrncRixGKnN1pEdKtMlhWCryeLs5VFLrsoEd3V8wdJclmWqvkZZQpS5WqVULZY80xYESS6VcZqJ45ih6K2Nruva786OgiQulEqO4yRhBALATbiVpqntOJbrcKKgaDqK4gzDHN6+Uy2VOZa+OD/Nonhvc7dVrWMIqpvaRJnNVWU0X2qWjeAoQaDGWvV0J/SjNIwYOMsDi6DwIAnSzEfgHEgBFCUohk/iPA7COI4RDIVRJIhSAEAIkskyFAriwPVsTuBxgqJpsllveJYdOK5t261Wy7btyWRy3RtIYmE8HjMU3W62GIbZ2toSJAmFUBBAMQxDUdT1Hc00vvru+yCKJ7OFaVpJkuRQbtpWmMQIgrTbbZ7jYBCSRHF3d79aLt64HFVVVVVV07TPP//81atXURD6bkATJEnhirIEs1RbqxiGffvt9y9evGBZFobhUrXUnwwgCJpNppVi8cc//vHT999rtNoIghRE+f6DRyCKgiBYKZXTKCZJHACyL778LI4jCALX6zVDC7wsOb4HgmAQRCzNhV6YJrnnBjCMFotFCAQ3Nze3dnaveteTyejR/XuVcllXNZZlvShmRWm2WvX7/eV8VqtUCZoYDHo0SxmmmWYZQdEIggEAcHp6ShPkF1980Ww2Dw4Ovvv6m9//vd8L/CgD8iiJkzwjCEJZrUiSrFQqvcH1TdB3//7D7777DgTBYrk0GE3SNP3www8no5G+Vp++/57ruoNBr9vtkjj+7t07DEZ+53d+ZzaZTKfTzc1NkiSvr68ZhulsdIM4Ojs7Qwk8A/KLq8vO5saN1sc0TY7j6vW64zibm5sZkN++fdAb9tI8LRQlzdCPz05xirxzePfy8jLLsnK5vFwoSZaCCCzKEgzAynKVZxmJ4aIoHB0dxUHYbrdN21qtla2tLY7jzq4uK816qVRSFMV2HblYCMPw6uoKhmEgBVRVvRncwxC6XCziOGZZlhOEG0L65ubmDSHvBmny5s0bwzC2tjY0bW2a5pNHj1zbObs4RzD04YNHg8HQ9939nV1lsYz8gGXZWq3WHw0Flrt391BdKdpabTab9UpV17QgCMrl8nK+gHIgyzKWZefTGcMwsiiFvptlWbPdcl0Xw7AgCCAIukE1eJ631rUcAsfj8S9+8YtWqzUejzmOi8IkThPDMvf29niePz09BQBge3vbtIwgCLqtdhzHN6EuiMDj2TSK4539vThNe4PB4f17lusMxqM7d+7kIHh5fW3a1tbujhf4s8X8Btz29u3bJEnq9fpqtdJ1HcyBUqH41Rdf7u3tiQX56urKdz2CIEzTgGGw0215vqOslxudjbt37tyssTr1Js+yBEpsbe2EoW8YmiiKIAheX15wLHvjdzUsMwVS1TK8OERRdDadtqpNz/FniwWCYTGQx1l6w7VfLpTRcMLS1Pn5ue/7d+8e0hRDMdTRuzfL1TyOY9XQcRxfLxdgnr5+9ZKlKJaiPvrxR5Va+X/61//q//0v/l9/89d//af/47/89//rn3/+9be+6ZIwtlwuURzb3N5J8wzKAZIk33/6nmEYAAq3Ou3Vcl4sSA/uP1yv17VaBcjSxWLRH4yCJF4bZhAEl5eXJEnO58tiqeLYHgiCLE2DIBhnaZpnNzlTuVy2XZeTZD8Odds2bdcwTRwlCpIEgvBwOJ4uFxACi4zgWu6zb76bjif1atVznCjw18qSZWlOFExTNx2zudGJkqQ3HGRxUi9V1OUiCD1B4mmWvaFFybIcZykIgkkYsSybZVmUJoq6JkmSomkcx4E8H40mCAzqazULIggAb1zKKAZHcdButwVJLBQKslzMwURZL2+osb7r0TjK4wiFgjiOkSwThUGexlkG4CQbRVmeg5wgkjSFk1ScA6pmUBzvhykAI9Dk+MxZrCScBKOAQwlzocBBklpepLt5EJuaXiwWa7V6luWhExIw0Tvvmao2n40oGtNNByfoUqkShb5MEliW8IIUJPnm5g5G4EtHI4tiY2Njrer7B1u/9YtPQQi5e+dBSS6EngtnwI/ef8+PvQRI8wTU1+Yf/MkfleolBEF81wbT5OTVG8eyX714XSmUaYotNZs4SxMZUKZpVV1d9C6VtdqsNd89f66MR7LAFiQx99PXz95Yjt1otueztbq2YRhub3STPENZSqrLQlWWqg2ClUlKhCFiOVE4nF3P1p4djMczx3ERELI16+zduziMzMDhCuLjx49ongEo3AzDglSkcMoKAgTHxpe93/nok9C3FW3R3OpgLN1st1ieS/Ps5etXBUl+9+74BpQGI9C7d+/+wR/+0ag/uLi4aLbb89WSZpjd7Z1Wq5XnebVRZzi2224lpg8F8c7ODs5SZhwstLVnWNp40ShVOIb96tvv3CjY7HRXy6Wu69ubWwSGnx+dMBxXqdUm/WHguAiJO6F/A59qdzuu5y2WSxzHZ4v526N34+FIEAS5UtJNw7FthqRSIH93clwqlURe6A8GjuvSDANBkKIonU4nTdMkSUia4jmhWCzeaKMLhUKtXj+9OIdh+O7hbd2xNNusVCq3b99erBU3DouV8mA4JGgKRGA3DnGWJmmaYRjVMkACj2MQAkiRk9MwkTiuUiolSTSZTcM0IWnG0g2OojuNepZH18Oz/bvbcRKMRqNCuVSv133XW87m3c2N2XJx9O5dp9liBf747DRNIoHjAQA4Onr7+PFjVVWP3r5FIBiG4SiKQj+QBZEgiMvLy/u37yZhZJomiqIoisIgqK1XaZomMXB6fg0h8NnFabVc2Gg1CRgfXI02t3eCyJ8vZ+1u4/T42FCNZrvNy5Llu+v1ut1uH719C6bZT378o9V8tl6uYBAqF4rz1TID8tZG9/TyIgeASrXqR+FkMqnVau12+/vvv6/Vant7ezdWjY2tTU4UptMpDMPVatU1jTyOuo3W5flF4PmNWr3dbM2mU900NrY2P/vy86W63trfxVn6rHdRrdd4jhte9ViaaTWa/fNLVzcZit7e3JyNxgxKiLwwGgw7tUa7WrV1nSLIn3z6aZIkYRhu7WyvNMX27FqrXW223MjFSdR2HYJhCZYMssQKPNv3NEXJwjjLQBwnq9Vqs1UdTsZTZZkh2Nqw/uR/84eBb19e9budnWqhhAFAbLvv37t//85BrSafX7zrD6+2drfiLCdIebZyRivv1ekQQZjexaDaqjQ367ptPHr0SFfX7VpjuVzSNO15niBwEs+NB/1yqbiztR2GIYxABEGACPzDu9cBmKEkkwOoF0XNdiOMA5xA8yTHYJKASRLDpaKg2bqumzCKExy3dkwvSzIUdqKgVK/CKCLKcqFeJQvC0jHMyG922o8fPKRZplguETRhmrrIcjLDD66uHcsYrxYpjszWC5zGMRjJ46R/cQ6m2XA0MmxLVdViscxxAkUyOE2ZlrW7s2Oomu5YNMtwDOt5XhRFK3UNIxiGoLEXAFFirvTVdGlHASFyLEsXJBmF4FqlbpvOeDRAgJwkcQhDNcdqNGs8QwJxQEAQgqBhGCMghMJImoNemOQZAMJokuUwSpI0F4YJgMKO5wZhiOEEVJXLFbls6HocxFmSx3EKYyjDMDSOBY7r2g6UA8pqfn52KnCsY5mb7Xbih5Zl4DheqdYdx/v8yy+CIFDV9XqtSIViDiLT2Wy+XDqea7uWquvtje43X3/77/7d//zi2XOWox3HURZLhqILopSmqe+7BIVXqqXZYl6pVIrFYrPRJjB80Lt+9+plWZaKUpHjhDhLIRRZzRf96yuOZzY2NpbLZegHjw/vf/mrXz37/tvlfAZBQLVatlwnA4EkSh3L8d3g+vq6s7mF4mSYJoPJuNvtTqdTCEGCIBI4sVaulUulOIo6nRZJkjAI52lOk0wYhpZtNzvtMAnjNAJhiKSp3mCYgRBCkSCGfP75F//iX/w/0yyhKPL1m1csTXlh8PLl63K5yLJsvd4kCGI0nUAQqCqrwzu3vvjiqzhO/7v/7n//l3/5XzY3u5Vq+fr6mmdYGIYHg0GUJIq6LhWk89Mzz3U9z7+4OLt954BA8fViKQkiDMMESd6+dZeimNDzZVEqlUqT0SgMw5vmL8uycRxzHHfzahAEIUziG6wgy/A3gqS9vT0Yhr/99lscxzc2No6Pj+M4/uCjD3Vd932/2+0iCFIsFOI4LkjyeDzmWCEI4ziOHccZXPcgAIwCP0rCq2F/Z38PQZCrqytelrqbm0EQnJyciKVCpV7rXV61avUsy4qlku/7o9GIYZhyuSxIkhf4Dx499H2/3+9LAn9zi6dp+vHjx7Zt+76PIIjv+4ZhkCTZ6XRGoxHHcYVCwbbtKE10Xf/www9ns1kSZyzN2IaZJJkky6IsjSdDZTlnGGY1X6zX6zzNkijlBP6q34Nh2DYt13U5lr2+uuJ53vW95VoROCYOgzzNZFFK86zebOAUWavVjo6OTNPc39/3fX86nW5sb1EspWkahmEcx3ECP5kvut1upVL57rvv9vb2MAz78vMv9nd2CYKgcGI4HNI0HYahrusPHz7M8zwKw5sxYZqmuq4fHBwYhtG/ur55So+HI9d1JUkCAMA2ra2tLQAATNsqN2rFcimMI5KmdnZ3GY49Pj3du30LwdCluuZEodlsDgaD8WD44x/9KE3Tk5MzWZbyPLu4usxBsFarDUZDFEdEWfA8bzGblwtFjqGePXvmuu7du3dfvnzJ8Uyj0Tg6OhI5XuB5lmXDJLZtmyTJIPQgGN7a2ZZkIc3iIAikgnx+cQxk6d6tA0GUgyCqVqvn5+eFQmF3/06Sgt1ud2trq1KpLpZKfzo9vrheaXp7c+v+4/eqzc7B3Xs//cXvBymoO8FX3zx78eptDsJXg2GOQIvVkiaZ9UoZjQYABKmqgqKobbsQAN/AvDRNA0CoXC4fnbzrjcYkz/NyQZTl0A+iKCkViyiMjIcjhiBBEPTCgGJYlMCdwF+tFcPS4yx2HCfLkmJJTpLENQ0gy1zXRDAQZwgEg3VdTZNoMh7SFIGiMMcxcRwSBJHFia7rKI5hFCGUi4ZtURQBgTmJE1mSVqvVLMviIH798lW1VM5AgKQpEEMm06koijRNUxS1v7ObJynLsuVK5arfW60VVVMogiwWy81mE8oBba1gCAxmOY6Ti8UijUOaIHXLrDXqfhSulIWhLbPYwyAASGIMQSEARiA4CaMcguI0RQlU1deqbacAGEcpCMAQjNleCGF4DiPQ9fUUhPDpfPXg/Q96k/nacR0ge376DsLh0bDv+Y5pqdcX77Y3qndub8sSk4d+GkY/PH+hmY5cLCxWy3qr6YYBJQuf/u5v31CK1rqxvb2bh7G1Ulfzscixfgp5Qba3vwMByVpflWtlVdG+/vybdq2NgAgvM2HmLxaLo6PT+UxZr7UgisuVYkWWkDwbj4fv3r2rl2pgBiqW6SVJtVT1LDuJYm29rpbKAs9fX1/1R1ffPvsmBZNqvfzmzSs/cMuFYrveolDy7dujOM0AEIMgxLM0HEkZllQtTdGU+WoucHSpwHMMzXGs4bgYReMEvVopiR/1Lq/G08nZ2UlgmNvdDblWxQTOiYIYzHGZ+/z5236/TyDoe/cfXp1eCDQfhxHPclmSspyAocTm9haKoq+fP2tW68PxuNJo+r5fK1cEQbg8v0BRFIZR13ZgGG6320Ec6a69e+sgCmKepD988vjbL7/IQKDZ3Tg6P/YC96c/+YlpGK9evOQYvlSQvv36S4JiqtW6rirz6RjEoHKjoimapVvdra5mqHkGNhqNYrE4HA4RBJFEuTcc6Kaxs7lBEeTZ+fnO/l4axe9evqZpmmYZy7IC31MWy9gP0jAiMVxRFIZhbsQCdw5ukSi2XC7Xhr5xa29pqte9y26rjRH426N3jmU/evDw9PT06Ohob2s7cjwSwdaKwjFso1afT6fHp6fVek2SxGH/AkXTnd1Omifn11fFYsm1HXW5atcbSRTKpYJq6jmMJCmozLVGuRW5IQLBmqYBACAV5Kt+TxCkUqGw0WqHnr9cLgmaiZJkral3bt2ulyvHx8e/+MUv7t+/r6pqGIZPnjyRZZkkSVVV33vvvWazHkURwzAbGxtZljmWVanUJrN5GkeiwGEwoqr6rVuHiqKen59Wa0WGocaDIQYTu9t7g0Gv37/WVa0oyadvj5bzxcbGxlJdpyBQazd7wwEAgn4Udre6IscTGC6ynKlqCAAmYTTqj1AI1dbrva3t9UoB0qxRq48GQ1kUaZKkcIIkySSKMQw7vjgBMARlqAQCVNsEELg3HSumGWdpq9M2DAOCIBAELy8vRUGQBBEGoefPXnpewPH85eCK4Kid23sv3r0JsoSVOFbiNduYLxfdzS3N1JaKUi6Xi5J8fnx6e3dfZLjTo2M0SSPH05bKfDr7xS9+Ztvm5Lr/B7/9uxROnJ2dkhyxfbAFo3Acxw8eHlqWtpzMeJoLXcszTZJgcYzOMmA4HH7/4tWb47PrwUSxPabc/OBnv7/Swq+/f/XrL79mRfE3X3/54tVLTuQ4gX/6ow8r9VaeEbEHASAqy7JjWfVqNUpilqNFkb84P12vNZwklOVU4KgkARhGSJJkupjjLC2WyqwkrNarPMkD2w+9ZDqasjhuGbpmqBhFX/YHIIYRDC3IQrkgVwWRAIAyz4NxxBBYgReN1ZrFUSAOoTRP48S1Dd+1NlrNJPDnw/7lxWkC5kbgPnj8qNPeaDbbZyenCIbCKKxq6ygKGY6ZLeau7dXKNRRBCoKYZZlhGDe6yulqESYxRZCuYS0XC3W5QlHUtu1Cpby5twWi8FyZn19du37QqVZ363Ushxr1uuMFGE7u7W4naQhgSJAl1Wo1Df3Q06sVwXUsDMGzIIETEAFRAEYoloEJJErCar2CUDTO8VkMBG7EcFK53nLDuLO1C0VwNlxMEBwb9noUgd+7dw9niLm6fH30Dsaxm94FAMKO63/53bdn15ez5ezJk0d7uwf7t29hNF5u1arV8oMHD3788SftzgaKomkS18oVIM0ODw9xHN/f2jFN+969e41OJ06Sly9faoaZ5lmjVvnl7/1ur3c9X82ns7FlaILAGYaB44RUKEmFguu6SA5kSbJcL2/dvUVg+Hq1zkAgy4GX3z+LvQBGIBRFn798jRO0omokQd+/f7her5I0UtarIPDDwHc9L0mync2tcW8o8gKGoOPxmKFJ29TBLBN4rlaruoGbAPlV/0rV13t7O3cO9lvN+t7Obq1e5QVWVzXTtNM0hQDw8upqPB4HvodhWIbAUoU9u7rMskxT9WKxbAfeZDa9fffOcDgUBNGPIlVVl8v5z3/2k97Vxd7BLWWtvXnzVpZlz3NhFAJBUFGULMt8P7i8vHz69CnDMGdnZw8ePKAJcjGe/sM//hMcx7/86qvdvb1Wt/OXf/GflNni9u3bOEn88MMPP/3pT8vl8vOXL9rt9v7+/mq1Wq/X29vbOIqdn5wiCEJR1A8/PFcV7e6dO6qyjqKo0+mgKLpcLl3XPjg4mEwmDMPUajUURev1KkWSDx48CP2g2WhHWZpkQL3ZmM1mcRzjGJaEUZakjUajXCr1etc0QR7s7oWBd3V+trOzQ9P0cDhkKbrAiyROYBjW7/cPb99BAPDi5JQkyYf371+cnU8no+2Nbhz64+EoSZLtza0kSWiaZhhqNpswDGMYRqPVjOOYIAiaphVFqVQq69Wq3WhiGDZdTKv1WgpCYZxeX1xIonB795ZvObPZ7M6dO/PlYjweHx4enp1d/ObXX7z3wfu8JJ6dn19f9yuVSrPRuDg90wwdI9AojYIgqNVqFEVd93s7Ozs4gZ4dn9yMrZbLJQhD7W4HQpGzk9P3n74Hg9AXX3zR6nZa3Y7tOpPJpFIuV0vlJEtZnsMIHIKgW7dugUCW5/nFxUUQBBzHTUZjlmZc13Vdl6FoAsMlQTw5ObEMg8SJ9XrNs6wsSjs7OyAIrpcrEATTLP70xx8Hnl+tVmEYXSvLQlHK8zyKIpbjcBzPsiQIPN8PJangWnatVGY4nqSpaq1xw0KYzWauY8EIOJ9PcRy/vDwvFmWpULi8vmIYplgsTqZTN/DrzYamr5Xl+r3HT2ReODt6Vy6XNzY2nr98cXBr74MPPviP//4/Igiyv7tnWZYXeJ1OK47DMAzr9ToIgvPprFgswjC8XCqnZ5ez8eTg4IBhuCjOlsoKx8m3xxd/8V/+JgfQolyTxNJkMvvxhx/neT6fz9sb7bOL00K12O1szOeLnZ0d23MJjGQodrVaYTg+HPRarZaiqlGU3Fxp1LUOIdh4NtV0PQVAxw8c32NZDgLgne52liQojMEgSFEUydBpllE0S+AUJ/AgDNE0PZ1OEQjGMIwkyZcvX5IMXSyVehe9OIg/+uBDDMOKxRKQg9989fX33353cnJSLBYdz766uhqORmma4jguycUoCAmMQCAkSVLTdWerpWGZF2fntmHSNL3WNUESJUlK05SkKBTHXMtW12sYAJvN5nKxuNHkPn/50jAMBEFgBIyiaDabKooym80mk8laU9I0juK4UChwHDubzdbLlW3oIkMQcAYBiec5SA4SKGHbdpplcRxlWeaHcZQACIFleR4EQRynpXrLz8Byd9OMUqi0W/IAe7PbYDHkoNvYqBeDwGtudAia8aPYjxNWLFTqmznGo3zBy+FKp33v8cNue+P8+AygUIwnPM/Lgqh32vurv/ivirLUtTUJwwxBdrd3UJLa3zsUaN4w14q+ABA0SCGUpGbTxcX56cnxm2qrwgksAmCbnZ3Njc57HzwJkziHoKvBMM2Ajc5GtVQuloumbRydvAuiwHRczwuMlbbb2YiiCERR188sNyEpkeMkiRVb9RrDUIWSKPD05la70qxiNK4pq1qhGNg2RzMEzVi2D8VZq1zcbNUpApGr5QgGIijPESBwDc9SWBw8vL09HvfjOK6Vyk8fPJopy/PeRatWxwCoW29Wi+U4TjXLPTo7//rZM5hmj6+vEwi4+/j+yzevQRhKotjSLFmWcQL9u7/723K5rK31i7OLP/6jP7nsXV/1LrtbXSDLlvPVncPDHADkglgplqajBQwgcRx6oY+jxOBqACTpwf6uahnD8ajRaOA47vu+5zl3HxxqpnZ5efX+0/dTID8+fre3s1spVl6+eGVYZq1SxTHk7OTk/uE9lmFms1m9Xvc9LwpDZbkqlEs4SQwHAyDNJElI0zSN4tN3R+ViaTqeGJap2aZQKgxmk+Oz0wcPHkAAeHJ07Pgew7EXZ2c4hBzu7CNZBiRxgRVubWy7uknSVJTEdblUkQpXg36MgDTLDIdDEsVuH9xiWfbrr7++tbUjUfzx8XkSQ0WpWK/Vjt6+FXkuyxPTsWmOrdZrDMMoy5WurrM07G63Mih2Pfvwzm1T0ZXZgud5VhRevXvrhsH9Rw+VxVKdL+9s7XYazcvLcxhDt/Z20zQfj8d7B/svX7+6uLxst9sUQw+HY4kXJEkajUYgDN25exfFkauL83K5fP/+4au3r1AIvXf3fu/qOo2jdrdVrBSv+0MCpxqNxvfffkuT6Mef/MjzPMd1WVEQCzJBEECeExi6mM2jIFRmi9ByZIYbD4Y0SXY2NyaTCSPyg9Hw1q1bNx6MSqm0WCx4nm+32xAAWIYBQdB4PH714uV6pWxtbc0X08j1fd2qi1Lv9DJ2fYnh+ueXgWnXyzUwBUI/KIpSpVxeLBYczVTLlW+++SaOEoYTBv3rOIrqpZrECLaq73e6P/noo9j3QjfI48wyNJYll8oKhBGMpgmOmagrK/RFUe5dXIsi/+Dxg8FkOprNW62GYWgv37yuNOqu6y4WCxhCfdefTgaOra8ULc6hcrUyX05sz10q6ySH9g5uNxu1599965hOo1YnUaQg8BLFtiuNKExkuXjv4O5qMMMBtFas4Djuh36UR7zEcBJOcaiuq0mcKWut1+/flAtQFJ1MJoIkZxDIMIznecVisd8ffPbrz9MckGU5z4CCXC6X6iRGZmnKMSxFEXGaG5ZNcTxKEWHkOrY+m0yjMDm7vAIQNMiS69F4bbpiqaa79oujt7wgJyEwn8xhGO0Np4u1DkO4YwdyqUpxQpakh7cOGvWq4zuKsUZwTF3paZAJlAiBmBel+4eHOIU3WnVZlGzLRVE0B4DzszOWZb0wSLMMwzCGpOIsbXU7N7JDBEEkqUBihGNaTx/dN4315XjogyBTlEqVUqtZB6G8NxoLkkwA0EFngyWJPEklllqORzSFCyK7Xq8C3xULMkFTBAYlUQBD1Fp1WJaNYt8wtCCKrSRzEdyDcbxQhK6uz4pFebmYjoc9TVeWywWJo6EfIBDsOE6pVNrfu9XZ2s4gmGGEw8N7pmn/83/+z8PIpxkq9n0MAsEs9/1wtVYN24JQJM9TBIUImspBoN3d+Ju/+q/vPX7S7/e3djZ5UchhSDWtGMgsy7q8PFf1teVanuOeHB9fXV3MF+MoDjVdNSwzzoHRYpXBkGmap6cnDEv4vsszdFES93b2bMsF0nyxWiMEESTprVu3eJarFIrNWr13fskxrG3bw+nk6OJMNUxJLMRRMLy6ElmmWasnSZJnmaEag8GgXq83W51OZ0NbqxRK+q5DksTJ+dn/+hd/3mo2leVqsVidnV2wHGdaVp5mG812FoaO4zCcEMdZjmIn5xd/9md/LghSEIazxdw0zVqtNl2uStXKy5cvP/7445vHuKIoP/7xJ4PRcDqdPnh4T1WWvu9/+pOfXPUGcZaCEPT111+jKFqr1WzDTOM49IPlcmkbZqveALM8CAJFUWiWiZIUwTEv9FaKwnGc5/uGYXS3NvM0pQiSZ4WNzkachIaqVUrlYW/gOz7PsBiM8jy/nC/K5XKSJDAMZ1l2cHAAQdDV9YUocA/u3R8PB47jbG1tLZbLq971zs7OxsbG0dGRZVnbuztRFFmWdbC3b5v6fDYxdP2mbTKZTDzbSfNckqSzi/PpdPrg0UPXdaMoKhQKfhheX1/7vn/nzp2TkxMMw1iGKxaLtu2amn6wt7ter3AcT5LE87zT09PVaoXj+OZWFwDBs4vTdrcFIuBwPNrd3rm1u+e67rNn39+7e69QKH72m19VqiWJF/r9YZrG3a1NmiC//+67w8PDjz/++NXbN63OxsHBbQBGwjgulIoXZ+eqsv75b/0izrO///VnhmFsbW3purpYLHa3thEMe/HqZbVaxTDM8xzHcWRZvry8RBCEE4Qkz77//nuapiEUqTXqIATNxpNyoRhFEQDlFEE8ePAgDqPJaPzzX/yUYZhvv/1WlmWapqMoOj4+bjQaoii+PTpqt9scxymKYtv27u6u7/uLxeLp+++laaooSrPZ3Nzc/Pu//eskjLrdLgAAUJ7tbW/RNHtxcXETrgwGvTyJd7b3Vqv1eDj88YcfoSjqOG6xWOw0W9f9Hk3TzUrNMo2Xr150Oq33n77n2o4gcJ9++qksFz3P43g+zoEMgbb39tM8azabAsdPhqN2u02S5Gw2U1W1KMmdVrtUkG945Z7neZ736MljjKDCMEJwrNqoUxQVpcnPf/5zGIaXy/mtW/soDEVh6Lr+bDJNoxjDUC8Op8vVr3/zeZLkb9++HU/nFM0iGGl5wWqtIgg07vcLhYJlOTTHuoEvSYVvv/kmipJyuawoSwTHLnr9JIMoilJVNc3BBACiKEFRdLVaJ0mWgcDl1ZXjOAcHtzf29jCGyvKcQBGOpGmckFkeAxCGpDzX1XU9iCMQhorFIk4S9Xq90WrP5/Pe9bXnuKVKGSWpRqu9d7Cf5MBoMi4USkmerdbKdb+3mq02Wu2yXEjjbL3WRKnQardzACgUy4qmJlnebDY1TcuzLAgCy7JgGL7ZpkIo4gfB+cUFRVE3fC3LshAEwUn866+/yrI0z3PLcZbK6uTsrHd1Gfo+hiPv3r3TFfX09bswCASGFChc5qgk8tI0lmQRRhHVtAAAAIGEo0hdMze3d5vNpmnqEASxPBcEkWs7HMNQJAkBRmTNVRzDHj594gHARFUe3L130N3aajT+H/+3/3saRLZtx2lI4nBsGkSUUyD+ow9/HESOLDGwGz3Zvw9DWAJB/dkogTKMwFlBZApSCuer5XrcGzQqtb/+y786vPuw1x9fXF9ZtoGSKITDKILXao0MhERZKpRFBAF817M0I3IsKM86W1uX07kSRooTQABUkSXNXJfKoqeuCACcK0vVtgmcsW23Nx2CKJClEZRGz3747rtvviryIhBlOYAMpkuIYORydbZYrJdrieG0xUpZrhAEQQlSKBRb3e35Ur06OVtcDztStcIXfvbT352vDb5Ui3JoPlMolIRA1HRdO/BYnlMWS0PVMAKnGNqLM4CkUxDKc3h42Xv+5TeyJOVZ6ug6mAMvj46cKMJRTFsum/WqZRoogv35X/xnkmblYuHrX39+/+AOQRB//+vfHN6/C0HQ5eWlIAjNZvP89Gyzu8XS7Gwx/fjTT2rNxr/5N//u8Z17d3b35VLxeji4//jRbLl4/fr1Rx9/hBPEQlnTNO1a9mA8GI/H3U4nTwHPdgqSvNHupElSLVc4hr26uDRUbXt7G0EQ17IFQbh9+/abN29833/y6PFisRgMBiIv7O3s9EdDhmHu7O6DcarOFhiCtjY6vdEQQWBZFCAgF3nBdhyCJkr18tnVWQzmjCyWCoWzs7Nb9w8Ljerl+QUGwu12G0EQhMRhAms0GpIkYTQ5XS0Obu07ljkeD4M44gT+Rjy90e6gEKwq6ydPnpTL5W+/+w4AgFKxdnpyPldWBE3N53NTN7qt5gdPng4vrkLLuXv37kpTTd/10zjM08VqHjneRw+f/NV/+s+np+e7e/uaocdZ6jgOQZE3/ICiXPj7zz4rlEu1Rh2AoMvrK57nO622qqpRFMklebaYi3LBtt2t7saPP/oABME0zxiOjZMsTVOKogSWOz0+MQ3j1u6+tlZVXfvo4x9DCPLZZ59BENDtdl+8eOEFwXuPn6xWK9ewHty7X63Wv/rqG5qmNzY2NE1bLpf7+/tbOzuzxYLhuGq91h8OOYEHERiEoMFo+LOf/UwqFq56A44XMZJAcIximIPbd3MQXq2VDz/8MIqi6+vrO7fv1isNXdUC36+VK0AKfP/sxeGDwwzOLwc9QZQKterZ9eVSmbW7LQQn/uIv/8vezm6pVHj+/DkEQU8ev3d6fr5cLwmCWM5XgRPQOBZ4Tr/X+8WnP9/d2nz53Xeqsv74449n04VhuoJY/OqbH+aTablYci03ipJSqdBpNr775qtquViplaM0AqAcw7CHD54AObxSZparAxgE0xSAUgvDevDBBymYB1FWq20AMONHkG/5BV7CIPTi4gJE4DCOoiiSxIKyUi3bhVAojmOpVCVI5sWz59PRKEqSDEAKpSpJM2ESJ2Bu+q5UKhWLpbOL86P+pZ2EgiCMr/sMRoBBioUp6oUbconMABDIWJrMQm81HfmmDWX5ZDJgWKIgCupyFQaeIDKGq2MEiqKwJEkQgS11/dvnrwSxiAGIoxgEghUKEoCCYRL2e73zk3OMpA4fPrE81zCM3a1dluZ4lgMBoCTJURiqhh4kcRAEhUIhylKaYwEAyJMUxlCEwL0stnxbZNnYda8vry3LFjjWVNdgmjbrtTCOUwBgaToOPRoHIlfjKEZgBQACAQRGcAqEEDBOUt8plaUkiZ5/9wONU53ORrVcAX2vwTIimM+PjiEwARmK4Vh+bRo7B/s0z2mGOZlMoiT78ssvMRRV16uLyzNFWeVRYiqqxIumaZIkGUUhDiP9i6ujoyM3DKSCjOJYAoA4QwEANJ8uIBC8e/tuHIYQhJi2lQIgjuOKouzv7nz80ccgDC3mKxRAgAxUFCUHgShK8iRnGSrPU8dzhYJMi6IV+LphCaIsF4uuZxM4CtywmFFU1/U8Se/evW0Y2tHR6zfvXlM0ralGuVgROTEHgUKpWCwXkiwDIUQuloIoXK9XaRpDEBSnmWqaZ5eXz16+9H0/T+McSBmemyvLtWHBOE6xXJakQJYjIEBR1GaniyDI7Tu3uhsdhqFgFMEIQigUMZKJ87xSrb58/c41LI5mOI5zPPcf/PEf+q539OYtSRCWZX351dc/+8XPTy/OOYGFYRgAMgzDnr98DUKQIAjjwRBHCVVVg9gP43A8GRqqVi5WXr16BcHwp59+upjNzPVaUZYffPDBt99+VylVP/nkk++++cZ07I2Njdl0Ictyt9vd3d1eLJe6rm9t7YAg+Pr1683NTY6hvv/2283tTVbgh70+CoPtdns6nZ+cnlbqNRzHj46OCqIAARmCIO/evWvVa3u729pKiTxf5LmNTisKfIrAq/UaBEFXvcsoCp48eRL60Ww240Thj/7kjxAMXiwWn37846PTExABIQhCIMi1nel0iiAIQ3O2675686ZSq1Xr9ePjdwSJ7e/vsyx7fn6+vb1dLJQVRREL8p07d7T16rp3+eDBI5KkbdNq1JqFQoHjmJtRNYog89mk02oVJAmGUQhFUiDd2N7wk8h1fACClJW60d26oZyHcYyAULvRTKI4y7JWqxUlcbNV7/WuWJphWT6KgiAIZrOZphmtTrvd6WQgcNXvNRqN0+OTzz77rNNtiaI4Go3yPP+DP/zDy8tzyzQ3253IdrMoxFB4d2frmy+/cAPr1p2Dk5MTx3MZmpN4qd/vwwDYaTWuLy+jKPr400/PLy9yEKAYmmGY8XBk6nq73bZtOwx9SRZABC6Xy1maViql10fvFstlo9UEQVCW5dPTUwRDb3BMgiD0hj2UwFudzqtXr7I8WSkLHMd9zysXS08fPnj9+rXjugcHB77vJ1FcazQc3xuMR5ZlNRvtwWgYBNFv/dbveF4wuLr+6L33YQCcz6e1WqNardqmubezWyk3vvvuh/VydXh4mKbp2cnJJ598Ui6WYBi+c3CrVqtFUSRJUhJGURTdf/RwOp3+1//6X6vVqqqqBEGYlnV6eooR+D/8x/+I4ZkgjhAMk0tFhmOPzo9zCEQhxHcDz4/9IHr18l3oR4ProSAI23u7CQAGfrxcrkmW9+MkCAJN00I/nMxnTuivdBXGCRDCEBQHcoilaJYi4zgmKSIOgul0KskyJ/AMwzx48EhVtPl0tlgsHNucjsY0iRMo5rteFEW6rsMglGUZiEBu6AMwJEpSvVpZzKaiKGZghkBAqSCfnp4CIHj78K4oijCMKivVcRwQhiACs3y3WiiVpYKhakmSMDzneO5iNvctB0XRxWxOoki9XCoWZVVVeIFdLBYFUWpUa1mcVCulG29XZ2srTDMAAEaDYRiGGIErulFr1DmOubq4pCgKQmDLsjqtWlmkb21349DDcTSOQwiC0hzgxUIURRiKFgqF0Pe7zTaQAYPBQNd1AkNxBA4cB4FSCEDQMAVWqtEfjEzTwDDsr3/zGzcHAALXXTtPw5LEN2p1BIZRBPGj0HAt2/MXS81xozCPXx6/qTbqnu2KjIBAMM1zpu2aK40EUCgH/uqv/qpSqbQ6bTvw/CAAQLhUrFy9O7s8Om022+VqTWK4RqHK8rIfZytFXSjr9ubO9s4eiiIsS19fn9McAzNkCAFr1dIND+M4EEdpEi8IfLvZkkVpNVmIHF8oFUu16tnVNYrT3/zwPIwTAkUqBWE07M3mY8dxoiTWXTtGwJWugAjIs0yj0ZDKxSiLXd+J8tgD4tfXR2fDXqPbCsNA5PhWvcxSWK1StHXVUNdglrq2pSqrUqEMgwjH8LblOYYTZNnKNSES/rM/+zPHchb6enNnM9JNAUZ5kqxUKl4U5yj8/M2raqvmuNZ6vaw16uPFrNRokBw7uh5KHH9r79bh4X3dtlaWynCMLEu+7UVBXCgXDddAEORgfz+N4rPT0/uH9+IwPnr95r0nT0VR/Pa772iaIQjKNM23R29ImuBE7s//01+WyvVqo66q6/l4/P7TJ1fjoRV45UpJ5AVVVVGcJEl2Nl2NppNmu87QRLtV1wyd4TiaonoX52ka1xrVze2N0WgQeW631ZxMpicXl4/ef4JT+GwwqUildqOZJfHzH75zDN2xzND3q5KgLhe3bt0CIOjm3Fwr166urtzQv/foget6y9miKBdIkuz3+xRFNRqNwWAAAACM4o7rB1HoWPbTB4+WyyUAABLHQlkqkKSl6iRNsTxnamYWJ5q5zJD02Q/POVYoV4ordQ6DyL2HjyhBmKkrjuM2mu3JZPKzn/wETJNvvvg8i+JKqawZpu1aWxvtva2NYa+fRvH23g4ApqquHR4eLpfKq1dvHj16tL+//+bNuwwAHz58CKTAxdnl5ub25ubmy5cv291OtVplUaJEs3HgwnlKYcjmRpskUJIiPvzkx6PJzPcDUzdv0K+WoZdKBRRF37x5U63Wdd0UBC7P09B1ZJbVVksYzCEgB9IkSaLB9dXt3d31ciVXSgkErRYKnEFXp+e//zu/u1otZotFs97geR4nccuzdF1jBR6EgU9+8nGcxgCQFnk2tO1bO3v1cs1QdCgBcBC2NB2CUUku8gw/7PcxgiAY+ouvvqVwaj4enb59VSmVHzx4dHx6tFgsDm/dMRS93x/cu//ItP2Ts1NZluv1+ldff0GTaL1cMNXVfD4HYYjAEYYk1opxcdmvdTqsKLx79a7b6Ii84LruaDq56ve+//77Xm9QK5a3Ou3J6KpcEu/cOlitVnf29x1T9T1je2ejPxlhOGlb1nI2/+zXvzFd17RdVTMADFsbBgBAJIYzBJGm6WC+QDmRYuViqUoxAs1yJE6sl4uKLKBgxrDE+0+fcASFAohtu1GWV5utarNR7tQxgU1gcKXpEAADAESRTLXeXKt6nAMJAoEMOVwsFEOrFMoMSuZgxnAsmAM/fPMtkoNX5xfqajkYD0AKxXhmuVb9OCo3q7zIkSi21Whoi8V02JMkCQRB17azJKEJsl4tG5qGwzAJQjgIojDMMhSJYpenZyVJrJbKMJitFsuVovNiyQ5DWuRJmlxr2nCxPL4eXPT6EAIHUQBAAACBged7hpaFNkUjK2Ue+yEKQnmU5UkOQmgOoSiKL6aKtbYxAGcYyvKdpetmNC1USwAGQU4QrtS1YVsQBFEwRiDo/v4tGMN023Wj6HLYPz49mY9HBYFXdXW1XkVJbDp2BiHT5coPA55lyqK02WxNhyNtqUq8oCyWG92t/f1bpmlu72y5YTCcTqI45kRBFISyXJAE2XcD27YFQWBIhqFohmHK1Ypm6A8fPeJY4dNPfxpH0Xw81pSF77u1Vrs/mhEEtbGxFcXpaDpBcExV1SiJcZJwHCeKoihOMYpO01SQRF4U1obKixyGwlnsYxC4mE8IAqc51rQtiReKkqjr+g0nSxSkLElhGI7SBACA2AtW0znHstpaubw8RyHQdW2GokejkWc7tWo5S+OT43dAluME0dnokpxAU2yUZAiBW5b3r//1v0Fh5OjN67/6T/8JgcEUAFeqgdPsDy/f0DxnGFro+x9/+NG/+h//ZbPZLJSKn/361wxL3do/+Pu//TsEw2iORnFke3uT55iry95Wd8t2Hd/34zB68/qdJIggCAIAoGlanoP6Wo2DcHNjo1AoHB2dOKa1tbU1W84VTb1//0GaZou1YrtWDgJBEHQ2ujdPxfF4TLEcy7JxmiMI0mw2kzi2DPP09PRmtbSazzrNVqvVODs7GQwG9+7di6NgOhpvb2/Xm41Xb17DKBrHcRLGDMPAMNzr9ba3t7c3u2/fvq5UKrVy5fj4GEaxvb0dRVk+e/bs7t1DWZa//vprjmYkUSQocrVaFQqFLMvCJKZZfjQaNRoNXddVVcVxcjAYpXHiOS4MgnEYTiYzFEWzLDk+Pao32zt7B7qxhqD8l7/8pWk6iqLc2j8gMfybb77BCLzebA6GPcuyfvqTnzz/4Vn/uvf44aPd3e3RaFAoFG7fvvvy5cter/fJxx9Lovjs++8phvnpTz/967/7awzDbt++/frV2zCIt7e3BUFQFGU6Hd+7d/f68gJBIAAA5rMliqLv3rwSJV4uFBzPHVxdg2lOoMS7N291Xf/oRx+gKBLG4e7uLk3Trh8AEMLyvG4akiQ9fvx4OBwCefrek6fT8WSxWMiyKMnixeU5gWL7+/vv3r4djYb1douVhDAMOY67d+/en/7pn7qu//jhw7dv3/q+zzAMQ9HX19dFWbp9+/aLF888z/n444+PT450Q3McR+CEyA2TKN7Z3NLXqmFYu7u7BE5JknRTQCqViizN/OijDzzbAbKs1WpVKpV6ve77bprFrVbLch2GoXAUM0290aiRGB6HEYkThmEQJLa7u/vm1WvTNBEEY1n+4qrXbLbbzebr5y+azfb/8Z/8nwia2tu/hWMkR7LDwSAH0huclmM6Mie8fvOSIHCOoR3HwQgcZyjXsjmGNWwrzYBqvVGslG9OGTiCCwz7+sXLL774IgOBOAfiLCuUKt1uF0PQNIt3t7YlXkiTaDQdLdczGIRQGIVheLVaABAY5WkOAjRP87LY7nYWqyUEQWESK+o6ybPFamnarlwoQThaKBYNw9A0zTLMm808RVFhGNI0HbiesVaDKHRCFyMJ0zSTJLFtu1SQfd+vN2tREqdpLBbkwweHcRaDCNjd3spB4Ptnz7IkfXj7cDGZglmurBYIDM5ms1evXiEgtL25qS5Vx/EohqZpGshyHEUJmuEkOYlzkmYohuF5dq0uChIPZ/FqOqkWiwRB+GFgOV65XHRcO8kzmmUMdQ1kkTJfzEbjs7Oz7mbnj/7oD6r1irKekxQK8SWR4un7D+5QGLoYzlI77pTrBV6eTRdzRfFBiBLFZrmcug6I5Bu7mySOQXlmh955/9rQLB6jR2fnBYrGYfTW7h4UxP/bP/xj1TRfn50CAICicAplfEFodTuOZdAQrIwnEAQtVkuMxKI0WiqK43hZlgBA9vTp0/F4HATBf/rzvyhw3MM7d54+eABl6TdffiPxkkCztVKRwPGt7W3NMYzQsQJvrqwKtQKAZGGWXF1dMQxjO3qUeCeXx59/+/VkOe/WmkgSVYuSZaoCyxV4EUdg29Dv379vu07ghRzNOZZrrjUKwTrFujVTEtOzNDPJ0kKpOJiOF0uFIChRlIMgwHHcdW0wTRAEMkNnbWoYhiAQioCI6wZxDmQ58PKHH1gC297bmKzm5UbrP/7Ff9vevxfnYLvdigLXs8zUDytciYKp6+vL/Tt7FEt/9pvPbt25PR6PdV2nadowjN6gv7u93e/3syzP8xyBMZETr/sDmuFc3yuUiptbW73hAEWQRrUGpFm71qBxcjWbH+ztEQTRaNSHk1GtUcdpqlSvrtYKhRKzyfQm5KzX65PZHIBygqEQEDo9OiUpttnqXvf7AATu7B8slNUPL54XK1WUIBeLFUHSnCgAAAAAAILhfhhyAu8G3vffPzu8/2h3/9br169t2354/8FkPuuNxoVCKfSD5WJBUdT9hw9PT08hANpodXq9HkmTZxdXFMd3Oi0YhnMAlgoFWS4O+/2CJBcKJU03lstVu9nkaarfG9IsL5XKo/l8rS7v3r395vTk82+/ffrkUZ4EL1++5jmxUW+9ev5KV7Un9x+GrrdazLrdNkET747fiaL49P0P0jybLaaiLKjK+vNff9FodUVZtnVVXc739w6iKHYc5/DwDoYhk8msUChdXl5rqmEYxunp6dOnTwEwMcx1HEWVSoVlWWW9vv/Bo4EyuRyNups7u5t3Xnz9mkW5h3cf6rOlMpvdvreXwemz188bGxsxiLphHGbJRx99NJvNzk6O93f3HMd5+foVxfGHjx4cn51eX1//9JOfzmezd2/fdjc2PvrRjz///EuaYlud9ruT48ls+vOf/5zGqJffPytVilEamWtToPnf/+3fMnXt/PTs8M49hmJ/85vfxABQ73QAAOr1ehsbG6ZpX/UGh4f3EQR58ex5QZJXiyWBYfPZhCZRnACPjt4+fu+p7brv3h5XKjXDMJI05CV6tZ7lQMQxJASkxWLx6urqBvQ/Hs9luYjj+LDff/zgIQrBKI4xHP/06ft5njebdZalv/nmm6++/Y5g2CAKBVp0DWtnZwvB4PFqVat3eEqYjKadre4Pr54RGB76IUpSKQhlWVYulTY7XRSDbduczWYMSSEAWJWKs9FkNpshCJKFaehHEIanEDAZjZLALxQkuSTTJJMmeaFUGM0mbuhLshyF7lpZpmm8vb1NEETv+lrTNMf3aJ4DUSROE14Qtna26/U6heEXx6eFQuHgzu3xfCYVC5EfHb87ieKUYGi+IFAUcWtr70dPPsQBGAUBhsBFlgttPw2SyWJyNepN16vpSvHDwPHs62EvgdLRav73X38Vw8jW3oGirPvX/d3t3SgMcRKXizJGEpsbG5Zm4QBS4sWqXDQ0HUEgME1i34/8IIuzRq2dRnmpUvGjkCGQbruWhiGJEK7pQABEixLCUHEWA0CCIFAchVFk4WgG5RGCAhTHB2F4cfwKDiwey+3VFAIAgGPY3uXVfDrDSALFybOzs9l4xHMMRVEIRvzW7/6OtlajIGAYbjKb5nlOsQwv8bu7u5VSOU2SUqk0HPZJkvIcj8TQ+WySozBBU6VqRdO0IAgoioKADAah+WpOUGSSJKVKOQjDKAo0fR1FgaasB72+bZssyzIUsdFpQRB0cX4eByFPM/+H/90/TgI/CkJVWdfr9SzLFHUNoHCxVmFEPoqiMIh11eB5EYBAy7Z1Q63XKimQ9/rDyWQmCeJyNY+iqFGr5ykQh2EUBK1G3TLMUqkky/JGp1MqVJEcfv7DCxLDHcdhOSbOUkrgdg72QRC0bdN13ThLz87OGIbJskySpJ3NrXqtgZGUtpgHnougaLFapjn2qrf8289+HWUxyVEJmMMoYpp2qVCOgsDzvH6/7/peq9G+uLhqtFuff/Vlv9///V/+7n/5q/+Mk0StVhuNRs9evnj8+PHp5UWSZ48ePbq4uBwOR/fvP6xV6l988UW1Vkvz9PPffNZut8vV8l/91V8GjrvZ3XBdd7VadVrterXyH/7Df9jf3xdlCSPwb7/99tGjR9PZJI3in//WbyuqdnV19fD+vWa7EYb+cql88pOfL5W1abulUonA8NVqxVJsvVRDAWij3ZlO547jUSQzHg7XK2V/55Ys/P9Y+q9Y3dI7PxNbOecv57RzOjnVqVwsksUmO7CbVLekhg2N4QAZMAxjPDM2xh7ZgOEryzOQRmFa0khNjrvZbOZUrHyqTton7X12zvvbX05rfSvntXxx5nbdLeAF3vD//Z4n3Ww2IQS+evX6/s7+0dEJSpAkTp2fXySS6Xw+jxH4KwMUDMMwDEMIDESRJEk4gW1ubV+/ebNUrj5++sz3fSmRerm1E8fx8vIyRVFBEFy+dn12bu7Rg4eZTObatWutThuAkLuvv+4F/mAwmJmZuXnz5vPnz8MwTKUSURAEnpdKSHMz9SgIoiBMJ1P90TCVSfu+PxgNJ4pyeHIMIiCKwTRJFfMF1/NyudzJ0TGF4qlE0nG8VruZyWR4nvd9F0GQlZWVi9Z5IpH4xje+0et1dnZ23nvvXYahdra3a7VaLpfrDfqJVOZr3/iGHwCtVvfWzbsRAB+dnC0tLY1Ho9FkzInc9Zs3XNfXdDNfKKEo2mo1BUFIJBK9Qd/z/Vpj9hWN5MaNW+VSxTTNemO2Uq4BEWhZzjtvvqX+z1vLjbnFpVarU8zllxeXYgiUZXllZc3QzCeP1yulAsMw6+tPURy7ces2TlKO575az+eti5nZ2StXrhwcHAgMe/fOna2tLRzHOYZFYlCdTqMImJmZu3//PoSgV29cX3/02Pf9SqO2d3RI0NQria6iTHmeX7m09mzjhecFd27f7Xb7vu8vLCz85sMPHS+4e+f2ydHh7tY2AECmafaGfQiCSILuDUed3gDHcRiCBt0ejpGyoqq6qRqm6/okSRqG5vu+ZVkERZ+3W6wonJ2fTyaT8Xjs+B4UR5oipyVpIo8ePn7khQEMw14UgjCUKebTuWy5XOQ5pn3RevTokeO5vCjsHx1yAi8khJPTQxyFSRylGRInCNd1U8nMVNf645EkSTiOi6KIYdhwNIIRRJuqpWKxd9Hud7oYhl20WhiGsRTNcZymaePhaDqd7u/v7+/u2pYFRDGJY82z0zCMdct85aWyHLdUKlmGSWEohmGZXA6I4w++/o3Z2dnJZMIyvGFYvhcSBGEaVn84ghFspMgkQ7fb7SiKPMelCMKyHAjGaJZjGMb3/X6/i5FYt9sFgGh+rm6qMklSuVzOd3xNmRIUheBoFDumpYZx5HkOgYC6Mo4iz3KcGAKzhXzkW5GnciRMYwDUOW0398+W6rNvv/lWZWlu83RvqE9mF+vlfJrFUIljf/Q3f9caDLhEctgdQTHCCJJlu1AUhr591jwzfRdkiJYy2T89DYDYdGwhIVy6cYlLCAzD0AQdWA5HULAfQHHUUiY9S3c82zYNEIRphkNxBMMRhqZd07ItyzS0vd1tTZtGQIgTBBQjOIC2To9EjpaVseP6Yei7roujlKYaI3lk2sZoMOZorpQu+I6vWTaMU+VCzdHcSrECw2gEQoph82IqX6isP3leKJVjEA786Od//2MGRzmGHI8GuUzOth3PC1cuX3aguDhXC8CwPxk8fP70pNUsV/KiwMjTEUHhgiBwnMAIiTiGKZgSaB6A4frVq5l61Y8Cz3NVw8REaurHD589CwGvO2xWG/lxr0VC0CcffXzn7htnnQ4liKfti9PmebkxYzmeKIqu7YAxkJAE23U8319dXT09P49h6LU33vj0k88lQXz1rAEC8Bt33xwOBgSG//F3/tBQtbNms1gpL8zN9VoXY3ny7vtf63Za2xsvv//n/0CeTgxN7XW677///sbGhudZ125e/8Uvf40SdDKRHo7640EfR7FGfe7svC1kMols2rZtBII67V6jVhdpFomAzWeb7733XjKZbrfbc/XGfLXeO+/Yhj0zN2ea+mQwkTiBwgmaphNSOgrBwPclSTo5OWIY6trVG82zi5Ozs3fffRdGwK3tTZ4T3nnnvafPXgwGg0KhMFXVVrczM7eQLxYGg8Gr6/BoNNI07dq1a91ut9/v57O5OHQRFMIIGoRxiiAffvmVlMhAMJ7P5y1T7bab5UppNBrs7OykUilRTEAQtLW71ZhrgGB8dn7yve9/P53OfnnvfqVSKZfLw+Hwwf1Hr926i6Pkw/uPBJap12YO9/a1qfrm62+MJ/3Do92/+Iu/gCDo8ePHPC++duu1509eHB0dVavlw4M9xzDtqRU70cuNrfOLJojDmECBHBngsOY4t167+2T9BcdJkRs+XX8i8Gzz4sy3HHUiEwQhZVIP1h8LkghBkGPZkR8ZhsGLwqNnT9zA50XhyePH0/FEHk5gACwUMmEY7mwfGKqNwRDHM+pU9/zYtu1sNptNpycjmabpRCqZL5Z6g4HjOMpkappmKpXKFvLnnVaz2cwkUiSKIDGYSCVBCMFhKJvIEAhNUwIOYQzNkSxLcdzd23dIFHu5c9iYWw18QJ/q2UxpMlaHw7Fpmn4QkDSVSSQLyVwUhDCKrl252h+PI8/nCMIyrDAC7NDvjQYzMzM8yz5/tvHGW29PNWU4GgFx/HJjC4bRRCYzkmUEx6djBY4gN/JO201VMyiGgUgMF+mxKgsC7zgWQ9M8Rm493/jq2frI1JzAhzEc49h0tRyCUK5YcByrUMhVq9UoAnrDnh9HGEoWcoXZRoVliOFw6Hleu33x6We/H08VNw5pkQcROI7jyA8c3YyjKAIhOwgS6QxNselEunPR6Xa7BE012y2CIIAgrOWLUBRHQWAGng8Bnu+Hnj9R5Eqj5oMxLfIAAGAIQhFkNV+kQSjNJkiM9qzg9tr15s7e2d4eDkMogRQb5amuxBFIkwKOkCRDa44VQABAYhEcWo4JwmgMo4SUUv0Ip2hRZDESDkJb4BkSRaAoFliBIhmCYmzbjoEIhYE4tHEMpBkiiiLbdm3T4lnGi8MYBjlRgmEYhMIodqPYLZYyUDaVXltbQ3HMD4PRcFIoFWmWiuNY16aea4MBAIMQQRCTqcJLoiiKfhi4rjsdylgE53OZVC7bGQ1023n77TcJEn/0ZL3Zbpmm6Uf+4eFhoVDQNG1ve2c0GmmmcePOa5lsNo5DhsAJBO12OuVyWTNU0zRrtRqKooah9Qbd9WdP/SCIwejSpdXhcPjwweNarVYqVnRd73R6CIK4ro3jOMcKHMdJothvdSqlUqNWNwyD53kYRGiKCjyf4bgYAjXbZFh2PB6Xy2V5ogwGI5JhTcclCOJo74BnWZqlg8BPSdJcvR6EPiew2lQt5QuXr1yTUkkcxwxTp0jC91zXNk1NT6SkKI7BMHItO5vNcgJvWCYQhaqukSwTwZBqmk9fbDx98hwCwYQkdLvtXD4NAFGn26JpmhP40VRJZrNHR8csy71yYM7M1CeTyav5QbfVzmRyumP/5vcfNup1TdE2NjbuvvGmruvNZhND0DiMHj14qCpT1/dAGPzyqy/uvnl3aWXxv/9X/6pcKC4tLOzsbBEEEUdRpVw+PDwsl4s0Rewf7M7MzCEIhiCIrussR9EUoeum7wVxHEdxDIIgQRDvvPNOu91ut9uCIKytrf3d3/3dVFEbjRlFmRwdHaXTGRwnVFXFKTKKIlme5vP5fm/Y6XWlZOL85LzTbl++fJnh+JcvXxaLxUqpvLOzM1XVt956S5nKijLJpJOe5/tBlM3my4Uiz9JxHHf7fVXXdF13LdP3PMdzRVG0bRvHcQgE93f3PD/MF0vycEgR+NLisuOFn3/2WSaTfvPNN09OTkzT/va3v9Xtdk3bmptduH79+sHBDi/Qly9f/t1vPpxOte9+98/2D/fk6WRlefn2zTsHRyeuF1y+fNlxvOfPnt28eTOO4y+/+iKREBcX58/Pz0EQxHGcoihdNzOZXCqVSiQkCIKazeb16zchGD89P0tlMsVSYX9/17attbW1wWC0/ujJP/yLvzw5PDk5OXv/vfcKpYJtW7qq/sEH3zo7Ozs5OfnTP/1T3/fPz89rtVoURb7v7x4c/sEffKff77fb3b/483/kWC4MQksLi7qmNZvNdCK5trra7/d3travXLqUSaWn02mn00mIkqIoEAQ5nhcC8XA8Xl5cQiCYo5koigIwBlGEZ1kCw6AIGA9HqWwmCALf9cIgcB0fAZBup89SrOO6WztbU1XGSeL6zdv94cBxXCmZ3t/fv3bzRqVSkmWZ50XPC16+fDmdygzDtC46F+3+3OyCokyiKABBgKbpF5s7UjLV7XZhEEoIkmO7duB5oXf39t3IC0VeGI1GH3/6eTKZHA0n+XzeNM1MJjO3MN/uD0zfdQM/AmLbtgPXi/xgMh7dv3//qHlmRiGIYz4EuQAAkyTNcKPRyHbMo6NDZaywLNsf9kAQxHFy2J80T0/TKalUKjXqs5ZpsCyL08zpRas7HEiSFEMgTdPpdPqV6yrwfAiCLMtCIZijGRhFKIZGMQzHcZZmDM0MXE+b6izLRnHsh0EQRxiBG7ZludYrRP5oOIYBsN/r4Qh6cX7a63XiOPRc9+TwRFcNw7AOT8+OT84s0wFBkKbJGASnmprOZjK5rMQLMARVKiWO4wicAiAYRhDD0CiSlFIJhmVHoxEMAQmOQsCgMVMDQVBWxgSBUSSaTyen6jj0vVyuwNBs7Eeu6wEQIiRTqmkORkPfdTzHAuNQm8rQyrXVjYOXXz1bv/fVl73TJuLGSSHhOJ6mGQRKIzCcTqRxBC0XismM5MVep992bJOCCQJAMRD1PYfh2HKlOFstvn7t6rVrN9qdwUcffmxpeiqVwEjMsm3Tdw/OzizHHXe7BAwFbgBF4HQ8olBcmU5xmoEw1A+CZrOJEWQAgMlMGsExISE923iyfHnljXe+trt34nmB57oCzdqWJYp8vVr2LBf0wDQvvvn6aw8eP9g72L1541pCEgAoJihcSoqixLMsvba2AuEwwZKCyIdRIIlJGMFNL/ABCIiiwLHPOycx6oOBBbvWTLmoj1UOpSRSNGRNlVXf9ZcXFgWWGfW6k16fZUjD0iACIEg4n06mUkmWo/P5vJTNiqmMZpooBEFRBIHo8VHzy0/u9y76jdn69sEWL9Fg7Jeyqd2XWzOLSx999vmlpTWR5v/9v//31XptZ29/b/9wde0yBCGjwbhaKrfa7cF4VC7kMRTOZDI7e7teHGE4AcYRFIWGZuI4KQic7/tvvffu/ccPe8PBlStXDo6P/DCcjMY8y3m2s7+zK4giK/AIBKY4LnZ9lqQG8phgSN935fEIiMNUMmmoxlf37idSybEy/sUvf5ZIJYV08uDs5IsvP3///fc5jlMURXcsMZe86Hd7oyEEQQRBzC7MQyjSuui+/vobduBGYDxTb1y7fG1zY9v1gxs3r503T0eDviQIcQweHZ+mkynHMsE4zmTSYRiXShUYAuIo2N/du337NcdxZHm8srzE0bSqqmNZuXrtsqpMHN3+kz/8U4qg733+WSbBXbu8vLu7m06n33///dFo/PTpRr0xl8rkfvrLXwgSW62WP//88zgI775+m2WpwI9mZuZxBDdNXUwIDEerE30ynERAKKR4mmVwFKuVK/JYQRBsaWmJItBSITseyUAMLa8s6MZ0Mp5WKw0Sw/d2tm5ev/b2e+8+fr5hBf4ffOePev1O+/zszds3kcB7+Xj95qXr8zOLP/7RT4uFynf+6E+ev9w8PT9aXVtsNGq/+tWvbly+ura4/OjhQ47j7tx9bXt3x3ZdUZLWlleerT9Zml+qlis/+tGPEBS9ce36y40NCETeeeutUj738Ksvy7XqH3/3Tx7c+8Ix9JvXb7wqcM426gRBnJ6fjZXJ++9/7fHDdRgESZzo9Drdfv/ajes8z9/7/Is4CK/fuPrJJ59gJHHjxo3To0MQirO5NEmSjx4/WFmZIyn43oP7IAzTJNo8P46AWJDEIA5Oj/dz+Uy32x0Ox5fWrswvzb7c3agUSxiCf/bpvTCMOI6bTEaixCcSCctyp4qxurr6/PlTHMMSiUR33J8YarfVOz04kmU5jiMMR2iaHozHmmHevHLN0NXjs1MABMdTpT8ZvcKGp5KJw8ODT7/4PEJhFCNAGNIcByIInGUImoqjaOPZc9O2EQzdOzyCIGT10pVms5ngElAIjtptDAIDz3+FlkykkjhD5crlxbnFV10HNwq63X4uV0jyomvZoR9oqgqCIIJjNC+QLDczM3N8dtpsnRuW6fpxqVgJHBuDQJIkIwQxfD+K40IqBYehPtXz+eLKykq725JSyVy5gFEwTACKqVNCgpVSrJTmxLSYyAZ2CPhxq3tmOipJE45lW1Mtdv3I9iIvzGRyBEFEjusbRlLgh/0egZCj4TSTLyEoioE+hXiea4BIFIEhw2IQ6Du2nk1nMAxTlWnoBoEPwBCJ4LRm+qYb2G6AI6TSnziaAYcxFIHRN7/1wc3X7q5dvioPR71Wp9cdaJZN0SyKoigEg2HkOs7x8bFtGgxFoCBAIljo+5ZlDftdQRCy+bzlOI/uPzg8PBwOh4ZtZzIZU9OHnd6w01teXqZpOpFI1Ov10WCAQDAEIoOJHMex6zs4Rfd6g9ALbcvCcfzatWuCKC6vraq61u/3J6rCicLq1cs4TWMY1qjPgiCckhLTyRQCYHkwck0rDP3NzRdSUlQUpZDNaIo8kQd+6HEc16jW/MA1ddWxjGwqOZ1OR4NhpVAsZDLvvPkWQ1IEQSwsLTKCiFMUTBLNfkdVVcfQA88PHDtynJliSVXk33/425Pj4/Ggn5A4zzEe3P+i1Tx5+uzRi41nuztbZ0cntmkBEWhZFgAAhqbHcRxEIQBDvd5AEhMvtnYBFDNsCyUw13cOjg/EdNIKgn6/Px4Mr924PlJkP4wLhcLBwdF4OKlXquenZ41G49rlK7/77W9hALxx66Zpm71+X7cMCABZkmrUaq9oRIfHB6o+fXWOVlWVIIgoDi6vrQx6namsVKtVGAJOT08URSkUCgSK9Drt1dVlVVUd08qkkvVKZTIZAxHwve/+mTZVk8nkysqSaZquH2Qymdu3b0aBz3Oca9mzs7OpVIoXOJ7jMqkUEEYvNp4VK+VXaECGZuM4DsPw4qxVn50RBGEymTSqNQQCX0kjs9ksSZJXL1/hGMax7Nu3b+/t7WnTaTYp5dKZxw/vp1KpDz74YGdrU9WUWqORTKefPHnC83ytVvn5T3+WlsTvfPAtjhP2dvcL+RwKwaZp5nI5QRJ1XQegmGGYTL7Q7/dn6tXJeCiPh4ZhHO7vpxNJGARevHgRxJGi6aEfDvvDhaVFO/CePXtRLJZZlh1PhrIs57KFfr/72WefXblylWHYra3NarXM89zZ2VkYBSsrK+12e39/f2FhQRTFo4O9a5cvJRIiGEeWZszVG+32hW3bK4tLru1sPHueTiUYkkIguNNqp5MpUzdMXS8XigAAvPLfAhBIk0wcRgLHD3p9FILL5aooihft5uLKIo6TL168cBwLIzF5qjx/trG2tMIQ+POn6+ViPiEKU1kuF0vVYoHGiU8++eT1119HEERWJ5VKJYqirc2XIAhevryWyCb/7b//d1//5jdEkd/Z2Xrva+80m+eKIs/Ozvq+f3x8jKIoKwkBEDVPjlfmFyMgMF0Lw7DRaNTvd6ulsjJRer1+t9/xQ38yURiGm5+fZxleluXj42OGY18NvarVBowipm2U8nnLNLrD0fXbd0AQBGNgYW5uKsvzM7NHR0e8ICAY/vnn90iMHA77vMCSJEniRFpKkij28OHDzrCPsXQMg0EQsAzHsDxKEpVqtd3qhp7/7W99J5FIBFFcr9cJnAIjEIbRQX9E4aSq6qqs0gSZTCSiONYtM18oZbPZKAjrtVocx9lsFkGQUX8wHAygGCAIgiAI07H9IAAgUNf1o9OTRCJBsRzFMrwouL6n63oQehzH6br+annv7+xq8gSGUVmWdU27tLo2nIx7o6EX+H4YoxhGMTQEgAgE0yyD0xRKkpbrrF65QrL/s7e50+kyDMNQlDIZt9sXpqULIsczrK5OXduOgphnhTgMa4VihqcCYwrHoanrLEdalmm7LoSjmumks4U4CAPXI0kKx4gohMI4jmLItDzXCarlGhAjlulAse/RGKFrpqJqYjIxOz9TrFVMx45gMARjDEFZmgIikMIpAkLU7vDb73zj0uLy3NI8wRPV2QpNk+12Vzet3mRy2m6btkFReKt1NhkPHc1gcdoybBTG3rzzxuXl1dnGTKFQsCMfIDExl8EYZjSatC9apXzhzTtvJDhJm6owDG9vbzuOYzkuSYsffXbv959+ZLnaRJEfP1n3wwAAICBEwBB6/bW7nmcZruGB8UWrx/DCsyfPMRzhEgIjcplMLvQjFEDgGKQgBAqDyAsalSoSeGqvXUpwpUxibnEOxPHTZv/SzbdT86uTCK7Nz4tJnpdIx1Ym7eZHv/jps4cPA8e2dI3jmc8///S3v/kVDoL6ZBIEHoxC6mTs21bsBxzD4iSRSmcphgnjGOewAA7TmcTOzt7KlVuqHQpC+uy8GaMwxpEhHNfm6p1ht1gvt3o93XY4Sfrthx+VCiWB5h599XB+dj4tSR9/9Pv33n67UMj/V//Vf0lz9LXXbr7c3aZpWlXV9fV1AADee+89EAR//etfv/HGXVXVXdddW1ujCfLLL+4xFH3r1i0EBh8/fPDOG68XC5Wf/OwXqUKKE6mPf/fhYn3ma2+/Mx72Dw63VlYXYJx48vzFbKMGBC6F4spEXp6ZpzEKCALT0AxVu3Xjdves2Tk9XVtYYGli4/mLpCC+/857R3u7DE1IAo+iOBDDLM0wLAXDkOPYe7sHq6uX6vWZ+/fvz8xWHVu3LOuT33905dJljiJ/9bOfFrMZjiIfPvhK4NlGrRr47pP1R9evXy0W8xRNoDjCiyIIxqLIp1Oiq6sEAh+eXjBCKpPKgFGgKHIYh5cuLY/GfXUqv/fu273uwHX9tZXlfDr9+MEzQ7O+/2d/+uL544k8+uCDD6aqGUdwMpn4X/yTv3z89MlwOPz+979/cnKiTMd3bl0vF4pbm5uSlG7U5549fWEYxsLCwmQyIgksnZLq1YrnuBTOYCBaTGcyAs+TtDyUDcNq90e8KIAoMrW1GAMoDpcEVtfkJMelWQFyA5HlQt8PwxBDUEmSYBhGUZTAcIogKYII/UBXDZaiHcdZXV0laAIhUZJlgigsV6sAGiMkTDA0QZFgFCMAWCuXIt+bDCevv/ZG8+TU0QwkAm5cufrxp59EEDi7OPfk+ZMEL3zza+93u+2LXoeVuIXVxadP13Vd5zjuk08+qdbKhXLh0cP7uXT+jdfeQRG6P5EpjuVZ6cG9LxcWG7aj/fzXv6RZPp/Jf/7pFxwrpHPZ3374u2w+l88XHz581Gw2/diHUWw8lnVdz+Vy4/HY0K1XkEEMBSEI6nQHpu2dnp+5rgsBoKHplmGAIIAT2OnpiShK8kRJiMLNG1cjz4XcINasx198tb2/Z3gOgKEsL1Ao4ZsuiVMkRnIkK3H8wsxc96JlqEY+m8VwHCNwbapTKJlIiifnp1Iy0271Ou124DqW48AIls/noRgIXe/s8Pji4qLb7YJgDMShxPI8w/Z6PS8MCIJwXff09DSbz127fpMgSZogQRA0HJtPJQAEDoD45OQkk0wBYSQKwtzcnOXYhVIpXyqeH510L1qiJMAoCgDEuK+MhkM4DgggcjT5+HR/YsjFuYqPQU+29vKFukjzCYZDSGakm5Zvm46u6YrrO8pUA1EsjsF0MhPadikhcQii9tscgkoEkZF4U5WziZTjuDGMKqYbYwyCMzAERKHnuK7t+TCIwACGYEQAgF4Uy6qOoAT4quO7ub1lGBaK4ImUlC9kc6lkPpNOpKTX3njNsgzXcSQxSeCUMp7kc7mdrS0pmdjc3e4OB6OJ4vt+KV9YWVq2PXc8HrMsOx6PYwRCCJwXBcMwlMmEIshapaqrxvHJ2UnzolKtZrNZAIIVReV5/oOvfyMKgYcPHxYr5ZPTcwAAgiAAAGhpZQ2E4Eqlqut6EAGO7y0sLsnTqW5aHMcFXrixsVEolyIAXLt8pVAqQyCczeYlIXF0dDQ3O78wt9g8baIIPlObEXlel6eiKCEYrjlWbzh4+PDB9vZL17U5Ubh7+y6BU7lSGcTRB08ey1Pl6dP1Z0/XLU2pZLP5dCIpiQJP4ziaK2QTkpDkuRTPkziBo8ji/ALPsDRJMQzVqNUpiuBYgWaZGIo5gQ5B4LN7X/6Lf/7flnOlXm+QTmcdL2BZXlVVBAa7/e7yysLZ2RkMI7VGvVAqCRxPUxSCoS82N37/+98XCgUvDJ5tvPijP/w2TdOj0eDG7RsbLzcHg0Eqmclm89vbuzzP37h+/aPffRjH8dra5RdPXzx78jSZTmEEvrO3OxyN/uiPvnNwcNDv97/2ta/dv/8FL9ALszO+6z148KhUrYiS0LloZpKpWq02GgxNTd/b361Wy5bjtDrt0+b5TK0WBcHx4X4mlwvi6OzsZDIaXr9y1TKdp0/XU6mE53mO4/iul5ISmqb5vjuRRzEQvv3m67/42c939/cuX758fnqmT1UIAj744INnTx47hvn1d98d9ntBELzxxhsiz/V6HQSEEATZ2dkJguDw8FBT1Xw+jxHUi82Ny5cvMSz90Ucf3bh5U0okX77cNDX96tXL0+n05z/9RaVUnp+d+fyTTwu5HI5ivW439INvfO392frs06dPc7lcHIePHz+ulmulUgnC0cdPnty5c0uSpPsPH5SrpVc0YNPS4zgWePGVjYRhGAiCVFX1PCeTSbXb7ZOTk3K5XC1XPvnwd65pFLMZZTKhGC6RSrK8cHB0WKqWVtcWT4720ikxl0o8f/IUjoHZ6gyJE5ZhggCQSCXX19dpml5eXDo+PDJV7fbNW6Hvn5ycLC0tVavVjz/+GEdRSZIePfjKtxzQZqmRAADcGElEQVSeZQeDQb/fTyTF2dnZ9fWnUQhcuXL5q6++RHAshhHP84bDPkniNEngKPJK+Y3j6NbWpqZppmMPRgPX9zKZTKvVogjC87xmp+36vmU5QRB1Ot2j/aPxeGLajqrbhm45lru3s0sQRDKT9iMQBFCGYYbDYTabzWbzR4fHHCv4vm855uzcnDyZkgSbyWR+/bvfgjBUKBV/99HvIQiCYXh/d3tmZk7XzZ39/dn5ufFgWK9UX5GzisWirushCBXKlfFw8NlHv/cdt5jNffTRR+12G0YRgmHcVwTFKNYsW0qm1lbWBJYrZ/MIBE1GY3kyUWT1lTBraWk5CEKW45bWVmVFlVLpyVg5PT0Pg1gSEo++um9MtUatTpHkwsICgeFwDJAIJgiC5Tqv6PyGafI8/8d/8B2RF8SERPOCbjswgoEgCMNwJpclSbpcLru2UyqVms1mGAShHzUvzrqtNokTQRCAMIKiKM+w+Uw2l0zDMcBRNBgDuVyOF7luvyNJUhyEp6enrygXFEUlsmmcphAMo2haFBJTTYVhOJPJxHHIsJQylW3TSPA8BkZAGAxaLdd0CJSAEGw4kVGSYcUkiGIkSQSB54UBRtGe63uum0wmfd8nWQ7AyO5kYgYB1JflZqc7nshBHOmG8fD+V854fG12Hoeg0WjgejZBkiiOjyYTkqQN0xwa6tBU3/7ae1euXYchfDiQbX1KohBLUGAMtE7PMQTxQ5BOSKeDVmvUnRp6Jpft9/vpbIbgWNNxLE31Ddsx7OlYGXY6B3v7HMeVKrXecIBgaC6TX5lfvn71mudYgshGsY8jhKW7MIIqhppIp8M4lEQOx+Cr168bptvrj7/4/H6jWkMhdKpqxyfNmeqsPFAwEE5KyWQ60+4PKJKfnVl+ubWDsowcR7HIRjj17tc/4Djhxfp6s3ngOfLG4y84Cq42ys9evrA0s1asIjDmhQEQh3HoRXEYBD4AxmEYgiAIATASQbEfvephA1Ds+Y6uKSgCJ7NJgiBBF8JA0vI8jIC//50PvvjlLx49vH90eBJ7gDW1ADsgYJSh8aOjg2I2S2OEOtXDIFbGynAw7o7HCEvdfuPuRJFjBKAYcjIaN09Oc5ns08frCI5k87kgCD7//PPVS2thGO5t76SkRByEz548LRWK8/OLcRzu7O8kMlkvij/85NMgCgkMb583b9++bRiG73r9fr882+iPR7ZuwmF0crA/nYyH41F/NHzv/a+dNc+fbG/iAluuVz+79wWB4/V6/f76oyu3biiWOjs/MxmNVWU6v7wYwXEQxY5lz9QrvmdHQIxg+PLSajaVHQ363/3uH2Ikdnp+UivXctksiqKKpvA8b1kGjoDVYp4kSct0ojjI57Ikjs3M1E3bVXXzytpqpVjY3t2xXWdheeXnv/qVapqvvXl3POlBiI+TWCKderb+bLYx++1v/ZE8msIAPFNvKKMRTRIkhqMopk+nYByhMFYslgvFMkEQAkO3mxcX3W6z07UNkyVJQeBCIH7FQHZdu1jMl8vlIAiWludBKGo2W8Vi1ff9L7/8MpvOLc4vHezvdtvnl9aWosB7ur6+sDAHQOB5u2VZ1vLigjIZP11//O2vv396sNfrdj/44JumaT569CgpSplkKpfPX7Tb6VyWoqjPPv6kXCgSGP6zn/z0yrVrjdn6080XMQTm8/lOq905a85X6o6qRpZ78+qNTqttqJo6ld/92vsYSfz+4w+X1pYIgd/Y2xWyKT1wQwjoDwcpUeqcNW1VBaOY4diN3W0pm0UIwjD06XiUEVIIiMraNJXPjpSpYVq5bHnQG+MYcnBwAMRwtdroDfoURdVLM61mbyRrFMd1B4PJRC6WC71BT9VNjpOiIEwmE2vX1kzX6PdGqmooskqzDCtxCIlCIDKdao1GYzqdkhg6GY3z5QqEomNF9gO3P+ql0+kHXz1Yml9stdtBHERhWCuW33/73ftfPVQci04lQByLQoCAcXkyNWOgcflyvlrt9QZHL/dICDk7OwNhSOKlwAtRGAs8f3d3xwv9o9MTx3MBBFZNCyUpTpDWVi4tzy2szi9q8qTVPMtkUpqmMSRFIRhLUpbv8pLIMIznuIlUUkomOIIatDrPNzea3TYrChhJsAyljEftVpckaBzHJ8pYVuVMNut7IQzAIk2TGOoFfgxAGIytLiyleDrFMfpEWVlYmyg6xUk0nyAJNiNkED+6eWmZwSFF16zQLxWzKZErliogipMEB8bI3MxsNpOYDHpB6MY4ggmcD4Iky6Sz6Wwq6dmephjD8bRQqCzMr+i6abquauhTU0cpIoTh0VQPQJBmGAZHTV3pDbsYSxVqNZxlIRCMr125VC0WNWU61TXTtmRZPjk80jVtd2ubY1gYghqNxpUrV4I48oFo5cqqkBAGna7E8gRBvoo0yOMhTVGpZDKXySZECYbhXneA43gY+gRBnB6f7OztdocDkiQz6VS/PzQMAwEhSZJeGWqevXieLxYyqbTnuNVKpXl+bk41S9VVVRkOhyROFPMFyzBTyeT1WzcNy2x3W17gIhhi2tZ3//hPv/2tPygU87V6GcfQS6trrh2wNPP3P/4RQaC6rgIQmMhkddv+5rc+QFE0kUoiGGFYZhzHYBTfvHrFMfT97c21hbk/+dYHh9svfdvKZzOTycgL/CAMEQyNIRCFEZLAAs8PwyCKIse1EBBA4hgIA4YmYRhmGAbDMCgGAttFIFSQUkEMuVGQymU//uwj1Zi+ooOdnp6uLi0d7O6QKHp+cg5EYDqZuv/VV6Vy4fGTdRCEl1ZWm502RhK0ILzc3en1+xzHqaqaSiR73W4QBNlUOoqi47PTubm5e59/8YrjqE51KIJmKo3T07PJZCKK4vLqyun52USZEhStqHoURd1+b2PrZRzHge+xLDscjYajCc/z5VIJhCIYgSzLuXrt1oP7Dwmauf3a7W63u/702TvvvXt+cba19fIf/Pk/3N/fpyhyPBnJyjiRSMAwaNt2HAZvv/124Pl7+zvVapnjuFar5ft+tVL53W9/UyjkeJ4H4pAmSNezz5unOEnOzs4+fHg/lUqxLNtqdWzbpgnSce2nT58uLCzZtru9vd3r9W7fvh2G4SsMPUZS8lTxPRuG4uvXrwMgxAus7zhxHNu2OxiMFhYWAAAIPG9+fnF/f7/T6TA0nU6nf/ubD23bXZhbPD486nXb6XSqWq89e/LcsZxarXZ6ejwZy4V8UdfVOA4PDw+XlpZ+89tfRFHw1ptvd9pdDMO+853vdDrdbrebTqez2cyTJ0/GymT10tr+/n63256fnQOA6OTkpF6tJUTpJz/5Cctxd+/e/fDDDzXDmJuf2d/f96PQcRwYhuM4fkX9DcNQEIRUKvHo0SPXdTOZTG/QH8kjGEXq9fqTR48zyVSxWHzw8GEuX0BhhCTJe199mUyn1lYuPXq0brtupV777e8/JEkSQSBBEP7+pz9ZXV3lef7o6Ijm2NffevMHP/gBBEFvvv7mxsbLVqfdaDQ6nc76+vqNGzd8P/zZz39+943XlpeXdndesgz1CopimU5SSkEgguNEMpP2PM80DZZjSJJUdUNKps+a5y+ePoMh4KLVOm9dlMpV3bI6nU42m1EUeTAaZ3KFjZdbmmGwFC3L8szMTGcwjIC4NxiwvGjYFo6RUQQoijLo91Op1HSq/fjHP9nc3SZ41gqCIAJUw7T9gOS4VKEQw/BkqnpRzItCr9cbjQYgGL+SI9q6EQVBEHimbaWzaV4UpGTCsC1WEnhRiKPoeP9g2O2UCgXP83q9XuD5nU5nKE8sz2UpOvQDhmFQHJMkaSorW1tb8mhcK5ZnanUvCNO57PLqiqprmXS6Wql4jssyjGVZKIKAcZyUEplcluU5zdD9MB7Kk/Vnz4eDURzHUjLFS+LqpTXbtVVleuf2bUmSOI45PTp2bfvK2qWkKIWeP+z1QQC4df1WNpPJpJPjfgcB4nIxn02lp9NpJpPJ5XIkjiJwDIBxQkqVKlWSYlXNjCIAhmEEgQvlAk7jARADCM4nErrjUgw96HZSkghBkK7rIAgahgV5thE5dpKgLs0uaJqRqdRimj5st0aDMYvTgBMjMdxttprN89Pzo06vvbW182L9+fHLLbndcW2D5ZnhZHx6cg7EUBQCLEePh10SglIch4FouVBOsCxL0wNlcnJ+VsykeJKcnV/w49BwDRgFR9OpmMlUZqr373+5u/GSBGHXtbPZdGBbuUSKZ6VGfTaXz9IkNl+rEzD64W9/EwLxjVu3V9YubW5vlks5GkVQKLr/4DPDmS7MNxAgxiDYVLVKNf+9733n0tpSFAXHF+ddZXzRuQBDP8eIM5mSxDAP7n321eefhK6TTmYcw3v68NnHv/rd4OSskcmo6pBgsBCKQgjyAMDzo8D1Q8sHogCCAISAMQolSBSG4tB3PdMGY0BTjcCPCZzGIDgh8BGBowIHIqhqm7DERiyB4JhhGM2Lo6PjnZW1mTjyKIzGQGI6kdOJxHg8LhbzuqUHcVAqlXrDwdPNF5XZBsdxgedLUhKCEFVWs+mcoZk4RubyxV5/KAmSJCQUXbM8Jw7i9kUbJwhFnYZ+MOkPGYYhcBIBkVQqHYBxebaWyRf2D0+hGACCcDSaJBKpwXDy+NmzdC6JIIBju5sb24lUrt/v95qtcr7w/T/53heff1Usl+qzM4+/vM9i9KjXdyy7XK0omvzJ7z96/eZtFAb/5m/+J8uyquWaZZj9boehiDgOXzx9xjBst9sVJd517Z2drfn5eUlMbm1tDUbDW3duf/HVl/3hYHl1BcfxbrfLcdylS5deqbERBFlYWGi3u47jkDiSkHgviHqDSUqUPNP+7e8+UnXjypVLceS9ePFkfn6WILCPPvpkcXGZwKnP790rFSuSlDg+OdncfLG0vKBpxscff3rlypVKpUSQyPHR3tXLNzwr2NvZrlXKFMW0mu27d+/GQDSSx+1eZ2F5XjO13d39IAA0zTg9PT07beq6gcEwQVMwicMEodsWhMIiy8SBAwNxMiG1Ltqu41GcwAhSbzRMZtKOa2mGwSelqaHHcczS9GQ4oml6ZnH+8fOnQ3nsBb5tmuP+oF6tRVG0e3yYq1c8IBqbhhmGhu8/295kRD6MIEO3UBz1fb/d7IiMNGn3GBiFfMCcGubUcG2H5oWdo4OJoTq+Nx6Mh53BazfuTIajB48eLq6u9KZyZzS8efUm5AOnh0fZbFrMCDt7W6oyeuPOTWU6RNCY5wUwBieTkaZPSZJEUfSseYLh0MJ8Y29vp9ntWmFAkiQMAfpUZRmmN+gjJM7xoqGbFIaPBkPdtBKZXLc/ymRyLMd4ru35IYoRBEVL6UwUxccnp5XZ2ePz5uu3b/3F976HIvgP/uZHXz1ZRxg6BCHTsgiUIEga5nlUEFKJTJJPESzrg7Fim7Qk5PN5z3FxHI+iyFA1IPARFFpeni8Wi6qqWpaFEvhgPMEounXeNNXpZDQMPDebTYdhmElIuXwmVS3qgRu7LhIEjuMQNLW0sFjI55ud9sqlNYFiAMdHAKjX6+2dHFVnGo1Go33WhKLY0U0EiKejyercYhSE9x49OO93c8UCThKm74cIMjHdnqKf97vr20+7Sl+U6BRLvXy6fjG4GDsGk0znC9Vhu29OtGF/UinVoTA+Pz5IS9z1K0t/+offJhA08GOG5hcq8/svtketVkYUI8eRh4PTVlO1bC+KpWR6NBgmBbaUl/TpKI5DAIIn8hQlaCaVDDGUYRhH1R3Vmg4m9kSD3QhKJ1OR57fOTo8PD2iCbjfbM/XGO++8lUgmVVU1DDPyQ1lVgsCbm5trNBoojMijsSRJr5jXtuv4flifnaNZxnadXr+/uLya4DmJ4SI/iKKoVCohCPTnf/7njuM0j0+V8QRBYcO2MrlsGAf1er1SLg+HQxCKaZLkOObh4wfbu1vTiYzAYClfiOPYM00CRWxdizx/dna2XChiCKpMxt1uu9Ptfvr559vb22EYOI59fHwsy7LnuEAUoyj69OlTgefiKEIRjOO4dDqtadrz9SfmVKNxgsKJubm5py+em5peKdeEhNTtdlOcmBXFpMSDMIiicBD5QRDgOM4wDIqiDMMAAOB5nue4ruvGcUziBEkQCIJgGAGjmGmaJEngFAkgcAwjPCeCIBhDsBP4AABJoqgq01/9+qOf/OTHsjL6+rvv7W29zOfz7V7bMHRJknb3dwqFgmOYx/sHlXr18OT4+bNns7X6l19+2R0M7r7x5osXG+12O51Mtdvt09NTURRPTk6Oz04T6RRBEI8ePYIgRJCk5883uu3eVJ7quq7rJorinUF/d38vjMFSqXRycqqq6tLS8nA0bnd78wsLW1svT06Pbt99jWTYw+MTimYlQcRR7Mn6i3K56ochisLyaDIZTqqlaiKR2NnbA0Hw5s2bD7+6P5WVWzduXFxcGIaBwjACgycnJ6PRaG1tDcfxbDbbbDYhCMrlcvJo7DjO7Pycbpl7B0cLS4uj0cjzHAAATMeejGVFUbvdLgAAd+++0Wp1vrz3VTFf8l1n4/mLbDKxODt3dHDou97q8oogCD/4wQ+q9drq6uqjxw/8MLj7xmtffHlPt8xqtcpx3MnpBYIQ12/eGAx6SSnx+utvPHz8yDRN29RnqpWtrS0AAPK5bKt5Puz3BEHY2tqKogiC4F6vpyhKGIZRFDEM47puGIaiKAqCsLm52Wq1sqVSq98dKhMUx3zfffH0iaworutyLDsajRqzc61Ouz8YcgIfQ+DL3ZdSMomThG4azVarVK0cHh7u7+9fv359Mpm0Wq1yuUwxzNbWFoJhtUb9/oMHEITcvvWarKsHx0ev333TNOxOp1MslvqD7s7OTgzCYRhjKKqpaqMxOxpNZhuN09PTRCZdmZ09PDgOPf/GjRuu6+7u79XrM5bp7O7u57KFK1euffrRpzeuXrt66fKjR4/CKFq5tHZ0cLjx7PnK0jLDMJ9++umVa1cX5+ebp2cgDKE4NhgMEARJCKLneSTHXLl+7dnGM5omr1y58vz58/mlRSmZ7A1GtZnZRCLRvrggKJLlBNNyUAw/v7jACByGwdFoRJD0aKywUsJ0PZqmv/snf/Ty5ct/82/+zX/8j/+J5TgAhUMIwHEcJUg3jAAExRh6ae1So1yHY5gi6GyuoKqq67qKMplqim6avcHAD1yGYTAYebm52ete+J4VBEG1WvU87xWcGcfxQqGgTCfHZ6cEQXRbbcswERS1PdcPA4aiIRDsXLQeP3o0HgzTqdT25kvXtNSJHLoehqCpVObatRu9Xg+CoFdeeATBGrMzDx8+JEm8UmvgJEHTdL/fD4IghmDTtBRFcTw3AOJWrxWCMQoClqJMlDGEIjRNQxBiaDqJ4ThKyLKcTmYIHO9324d7uy+fPwOiSFGU3nB0cXFRKhQnkxGOoyAYB3GQzqZIjjEc1wsjVdcAANCUYfN4z7MtGIQEQQiBGIDhRCbjBTEAYXEIqONpq9lmaBrqdfoXFy2YwJLpFMewUBCVBUHt96b6OMRASuR2Dw8CIMgXc4VCCQYRkWVSiSRCUrrvj1SlO+gzHGvb9nQqV2aqIYbun56yDC+wDATDQRj6YQDA0O8//C3HUMlMujE3e3p6fOXKJdtzYRi9OD6+OD6eqdXrtRmKo9ujgZRMBFFoelav39/f3z452E9LCdALbl6/DgGxoam2pe29fEFhaC6foQVO89zeWEFgnMYZEEIsx/M8DwCATrt//8HjX//8N4OLHgEhFEL0egMgBlfWVsPIT/BipVwGMWTl6tVv/+Efp7MZThQuuh0vDtu9rqHpOAxJogBDMQpDCAL5UWgGru64YQyQAMZidAgAlhdomuH7PgAAERDapkFxlOyaU8cgMISnKZ6hGZyOLI+EiMAPXS+EYiSXTXphvL6+fu/3v64V0wSLxzhULBfCKEgkk45phKZ1eXaRwgmR4zN88jvf/Pbs3LyUy/zyww8TmWxKSu7t7ibExDe/+UEymcQIXExIh0dHhmP/2fe/N5kqURDMzMxgGMYxLMfwiUTqxcZLzw9FKXlydLS/vz+/tAgg6G9/8yFNs9dv3tzc3Lx140a5WPr883uabsIoFkTARbs1mUz8MOwPxhNF/uSzTyv12nQ6PTw8NkyrUivjFP6q6ESSpGUZhXJenk5s08hnsqIoEgQRx7GpaRdn5/lsLoyj/miIomipXHj27Jmm61Iqc9Ht5nKZo6MD3bAYllN1Q5blXCbFM+z9h0+S6fzS4mK/04UAmCZJMPDg0F9YWlR1w7MsS1dnZmZOzpq7h4em46qG7vieahrn7QvTtrr9nueHuXwxDOMoihRlEscRihGD0biSLZlTbTzuCwmBpWhTt9KpJMcxruuenZ7TNDszM//o0ROSpHmJf/HiCYqCKytLsjI+PT2dmZl3nWDn5dbMzEy2WHj4ZB2nyGs3b4AYsr2/K4piKpn85JNPSIbmOG5jYyORTX/9W3+w/vSJqqo+EKEUsb2zs7S6sry8/Pd/9+NcJvtP/sk/+enPf66o08s3rrQ67YPjo3fefjt0gv/xr/5dNpVeWFh4eO+RbwXvv//+x59+PByMGzNzMIbfu/+gVK6gBP73v/w5JQgxjEAI2hoMQAReWV72LLfT7nKStLm370PA5UuXpoMRCaDj3shynZ39vbOLs5nZxvHxqe9FvJBUZHPcHuuyjtHk/skhEMUkig0HY9PyEqm8bYd7B0eaaZi6PhwOWI6Wp8rJeXM8UXRd7XVarVar0+7FIOR4Xi6Xe4XfKperu8cnYiY1GPY812YoVtPt8+GoMNNYnp/767/6q6+++urL+w9S6SwIwhAAQjEwGg4BAABxtDo3IwoJwI/hCMIgRJdVRzPmqnUojAmaSuYysqZXanU78izXUCZyWkzYujpVxgAAmao126izDMWKkuG4URyIopDJZSEE5nkejOJcOsNzHMyQZuQ3KuW5SqXf71Msc/vKtYyY2N/dmZ+dKedzoe2Ghnd+dNZo1CAcVnSNT6XGqtbsd7OVrB/7HMmSEGkb5srCPBIBc5U6jeMMgS/NLyR4oVxvXL9503N8QzOvrl2FgqiQSLA46sdhiIAcR40G/d39g3KpLokpVTGOjo56/U42l2IYipLYiTnNFgvnnZbp+2xCZJJcdbaSy2diECZ4gWBYAoEseUTEoKOqSYFNSeJgPPriq3UXQAw3xEmOYcUYhE7PmhCB4rIs0yy1sLywtLKcSSc//PDDB48enXU7zV5Hcy1W4mmajqLINE3DMBiKTmeSg9HQjQKao3EcDcMQQRAMhWkCF1NpmuFc1x0MBhiKqqqqmlYMgUAY5ZLJ1++8tru7S1Ckblgwgvf7QyiObFMNXI+m6QAADduCUVwUEhEATXWNoWkQiEaDoaqqO3t7pm2NRiPDMFAM63a76VRWVVWaYymGHo9liqAdx5GVcalUOj+/wEmCYTjDdkRRrJfLgWOHYeiFwen5uarrD588afX6/fEExbDeoGtZVjqX7fZ7fhTbruMFfhyHmjYFohgEAN/3/SDAcNyPQhTD4hiwLJdmxSAGKJoFIciw7AiISZryo9CLwgCMTceGYRBF0VQqlUhmQQANYgBGMIxipqpJ84JqGNt7OxcXzU63NdNoHB8cIiDQPD0FAICiqLPmebVQpgiq3x90+4OJNn32fOPNt95xbO/zr76cmZ8DYfCHP/yBrk5n6rXRYJjK5jLF/L/76/9YKBYpjlt/+gRCYIZjddM4OT/JFXMkTT1+/PjGtauz9drj9XVO4KvVqqpMHz58WK03nj19rkwNgqQN065UKpIkBVEkSGIQRxtbLyEIunT1yvbOjhv46XT65cbm9vZ2v9/nOXGqGZqmTSaK7boszzQvzvv9HkURjmO9fLl56dKlhCQd7B4wFEsQ1GA8Mg17cWU5AiE+kTw7v7i4uFhbWzs+PWl3ezEELi0t7e7u7u5tr66ufvHFFzRBAgCAoBiGEq3TcwyBD/YPpURye3vbMS2CIEajiecGjcZsp9Pb3TvI5LIMxz179mx2drZer+/u7m7tbJcqZdu2t7dfzi8tZtK5Tz77NAzj6zdv7B/sNtvt5eVl3TD7wwFJ0jhOKuOJpmgffP0DVVZVVb9+/bqmaV9++aUoiqIodrtdy3Lq9Zl2u9vr9d54683D07PD07NqvSZIiV6vh6FEuVo9PTmXZfntd949a7U3draWV1d83x8N+rZtp3PZ45MzTdMuX75s2/aPf/zjQqmoqNPdnf1XAsUXzzdpkpqfnWu2m8fHh5fXLnmOe3Z2triyjGHY8dkpiCF33nx9qIzdwL9152Zv2BuNRrwovkph0wQ9Ho5cx1EUhReF8UTxbC8OgCAIoigiSArG8Fql0jw7TyQSiVRy/+iIIIiZeuP5k6eCKFbrtfX19fFgks/lbNOCQKRWa3C81DxvXb96lSHI8WDcaMxiMIZh2PXr1/zIB6H49TdeOz4+TiaTIAiYhlapVAaDAYwg5XJ5MhlfXVv5R//g+7Vy6frlS3/wzW/85T/+h0dHh3wiWazXdV0HAACCURQnYxihOZ4ReZwkaYoCAcB1XRgAKII0NN00zdF4AMJQIpnECRTG4EajhhA4ShERHEsJIZtKR0E4VZQo9Nvti36/LySk/mAgTxUvCMREguEF1/UJBM2nMoZumaZ5dnxiG+bMzAwIghfnZ4FjLy8uDgaDo6MDZTKhEQyNY3k0BoAIiENJ5MWEBMJQBAKqqk5l2XddXden0ymOIbauwXFULZYAPwxsX2S4e59/OZYnpmm2mmfppHSwv6soSq1WabcvbNchCEKZTl9sbqiq2u12vSCCEQxFcVXXLNcp1asgjg9V+9HOWUcPVC9245iRBDtwGElIZNOT8VQQJMdyMAQVWOqiecoQRCYlrj9+PJ1qbhAGACSJacsMIBJG69VarlCYqJP9o10PiLBkksxkxWyRT2WswEFIlKMpz3Z8L1Q1o9VpIxBcKmZ5lpxMJgAAgCDY73SBIMQAhCOZZDLlRh6EIiSCLS+uLF2+5MQABEFAGD25f58jqNFEOb5o4ZTA8hLN0fls1tLU9nkLI8hybcZQrUKhUqnW0+ksAaM3Ll9N5bNiLuNBMc1zHMOLXNIHwYGmxjGIwljseSLPSZI0mUwQIL5z8waCoawoWJZVbzRWr10ZTmXDnAoiPVVHmm1AKDpQprn5OQdG3nz3nf6g+7d/9zejSf9wbxcAAJQkZEODUdR1XQTBBDEJwShFMRhGQBBMURRK4BGCgRg5NWycZlzAd+IAoQgYo9wQCGOYJCnX8dw4VG3bj0PDdX0E1kLfB8GhqsiW48OIalpOHEcE9dWzzd/98nePP/8SD4C1xnw5kwtCX4s9gMIn4/GoM+DT6QebL8aqVq/PhLabz2Zn1lZNKD4fdZYvL/XbTW04rBYqg+7g4PBo+dLli0F77+jw8vVbfCK9d3JkRW6xUuz2O+p0/P67b50c7ga+i2GYruujfieZEBqzcyNFcyI4AKChPOEF9uXGC1vXcBzvD4dhGNy+fcu23UcP1+dXFhsLM+PxuFGvF/J5giCSyWSr1RrLSqXW2Hi59dkXX955625v3L/3xWdJQbxy5covfvELx7HAGOi1BjhMdtqDsTzRNI2iqIODg0ZjVuSlF8+fJ7OZTDGLI/Dh3m6lXkllU8fHh+Vi/uhwZ36u3u2Pjs6aopQCILjbH+zsH3z9W9+yPG/90eOra6t+4K6vr1fKtXy28PzpC1Mzv/mtP/jiy68erz8slQu+7+7t7dAs9fqbb7zceK5oCpNI4Cz7YvNlsVQpliqb29sBCFYbM/tHh9PptFKpwSD04tnzbDpTLpSfPn2OIXi5WNF0k+UEL4gEQdjb2xOFBEXRDx48BBF0ceXSJ599EYZxIpm+aLdGo8mNG7dKpcr6+jqK4H4QtbodEIxna1XfMFTNYIREZzAcTxQUxwRJJFg6AgGJ413dKpeqBEU6jnXROsVoHCHRvcNtGIPyldJX9x+EQLiwMHdwcLC9uyslRdNS97e3Vhbms9n0s+dPSBSZrdZ/9Dd/m06mrq1dmg6GPMuTOH3/4fp5u5utlNqjQWswCOLYc9xO6wzHYFVVTNt0Az+IvVI573qOoqogRgQgkBOTgOepysT23J2dvTiI7Klhqlq/NyIxajToa/IEgmCcJCbKYDjqkDgW+j5PM4Net14tBYGXlsTz01MIjF1bbx3ufPGrn7yxsvDJz37i+r4NRWbktwY9AIpDII4glJbStJQCCSKRTPu+57qu7/vNzlkAeJahMwyTyKYJhg7CGMOISqVULGbHk1G737HhcBo43eEIgmAIDOPIl2UZx/FGvWobOoigEQTppqUZZhDGQRBtPHnmm3ZBysxVGo7vZIrZyWQkciyGwgxNQFHgOZaYTiYzSY6lfdsCgQiOAYmmSQC0VZWnKddyoyA2NDWKApoXYhgiCGw06DAMY5omFMUMhtmTaUZMUDwNkQjLMqIo5CtVy3cVecyxtDxVIIKo1CsADBi2mS0WODHhB+BFayAKCUHgBuOhqtsjEzhS0R/8fnPkUTEuxiQGEqCUFAzdgjEeQniaS0AIHIde6KoCjWZE+jvvvaUMuvsHB4ph2w6QFPPQq2Cvqmu94WAsT/rjUa5UpBjW1E2GoiEIKpVKvXZvOp7gGHZpbQXDsAgIfd9/FW0WWEGbqvVqbTySB4MBhWG2ZY5GEwTBJoqcTqdfnbUpipmfX7x25fr77773J9/+o7feeBPHcdv3KIYeyONur8fyfBzH+lQt5ov7u/svt3emqnpycsayLADFiiqrhu54biKR4DiOZOggChOJBEvRtmWBIMhKAkLiIAgPesPd3W0SR0VeOjs5p3CCYZgPP/zo4uxieWExm0yZhvHmm29+5zvfEZOJZrN5cXGxsry4sryoKAqOk5PJJJ1Om6YJQRCCIJZliKKIIEg+myvk8oVsLpvN0jSdz+cX5ubTqVQuk62UitlEimPZWrmSlhKlbLFRm1meWygXS+l0mqZphuZq1cb8/DzFsGEc+VEc+LHnRrbjwQiCotjJ0cXm1kvb8arV6mg0unr9aqfbDmy3Vi7FcQwjSKFQOjo6Ojk6fePu69PpNJFNFyrVR0+fWZ7P8uLP/v5nAsc35mb7wx5B4Ldu3ez2+zt7e1evXp2p1bc2t8u5Ujlb/PzjzyAQiWMwVygOBgOSxH3fb120dc3I5oqOGzaqjfFwMFevKZPR1uYmQRCOZdz7/FMEQer1+ovNjcFgEEdBPp/XDeu82To+PVlcXHR9/+nzZ3Oz8zdu3Xzw6GEilRRFUVXV/aMjXhK73b4gCNpU/dUvfnnlyhUAAFoXnRgAZuYWzs7O+t1uvVLlOM6xPdf1OU7o9XoIBndaTY5jSqXS7v5eGMelWn0wkW3XK1VqKIZt7e8LUqJcLt+79znP843Z2bGsDEbjV97Rp0+f8hIvSYIbuJwo6JatadrZ2ZmqTA1Nq9frk6nS6/UoiukORyNF831/f38/jmMYhgkM13U9CAKWZWVZHo0miq6rpnl0cqqo2vziQgjGnd4gn88zJGNbLk2zfhjGQWwbpud5um6+2NzQTWOqqU9fbCyvrhSLxXv37kURIMsyBEGnx2e5XG40GT/f2szn84IgnRyfObZnqBpJEr4fzM8vjsdjZTp9pbjpj4ZeEFy02zCG+n4IQRBJkkEQ9IY9QRQTonB2ePwqePPqL3hR8AJ/Io8kSep2OyiKFgoFiqIAKA6ACMfJbCZ/fHqSTqcBIEJQEIbhmblZzTDa3T4nCLVGo9ltgyhGcyyKoo7vSYmE43soihfyeZEXM+lcuVpttToiL+A4DoLglStXOI57ub3NcZyiTEAwbnbbzzdf0ATx7huvQ1H4w7/+wQ9/+NeFXOZ/+7/6z/6nH/w1SaIQgjiuhSAIJYoQQfgg2BkMSrX62qXLHMMSKBaHkaoqIApFUAyhUGOmBsNwPp9P8AnbMDOpxMHeru8FFEWbngNiCBCDcQigEIzjKEvR48Gw32mPRiPHcRbml3iW06bqcDh+NfzgeR4CQct0oija391DIXjQ77daF+PxUDX07qC3u7+j6NrJyZGpG6HnMwwDxLE8mQgMDQaRqRv5fP6V8UYUxVQ2Y1hmIp0KgXCiKGJaioDYdz1NmS4uLAuC0G93eq323v5Oq93u9nokRcE4ploaSVPjyWQwGpq2BSCwF4W5QjGOY9s0XMu+deu1TL5WqC2v3vr6f/jRR1+8OIaZNIgxMIxQFMXzIowTEQrHMGQYFo6gnmPAgR872s1LS2gMbr/c2dk/BAkSOjk70Qyt1xv0uiOCoGiS1GQFB0HYdWPTQgNw3BvncgVJTJIA6BkGLTABBJimyVAsEAKe4zq6ORgMitWalEqWCylXnWSkdEJKsaK0/nx9f293tlbLl4p//5OfHR6dnp2c7z5/AVhWHDuXr15qDUaq5cxcWtYDKwg8AsO77Q5L00AUBlHIS+Lm9s5UnpiaDodx4ASWYw4mvUQiUatUoTD+g29986233sgVsk7kTS3Dj8Feb0ghCBpGKUa0ZWN/Y8ecmijMUITYPW1zKJFguaOd3RdfPoAt5+zwlKW56XA0HU/eeOMdluE9yw5cD0dR1/WjKIARSNOmQByGvuuaZhT6gWMnOA4OQzQMGRhhEZRHcSKKa8m0iOJZXmAwQiKZHCuWxEQulVucmbt9+fKlpcWlheVbt24Vi0UYiAkIS7MiEIQYgnohgLLUi8PD/+c//+f/n3/1Lz777JOTre1SIonAQDaT7HUuQCDiKDItiZcurf7VX/1bAoYm3Z6qGpnKDJ0rHrS7ly5dTnCCYaosR8BhOOx2XvntemcXF3tHq/MraITJXeXS4pUohGMAabf6thsiGDmWZRwlcpm8YViu65IYvtiY7bY7DEVfvXLNd7x0MlWrlH3XByMABuFut7u2ttZqtw9OTmeXl1GSWn/+tFDKVxvVg+PD0WgkiokoAlwnNCwnBqAYgHPZQrfb5wT+23/4nUePHqiqKghCrzd4ufsyAiMIAnRVOzs4aZ+3Go3ZIALa3ZYsT27evL65+eK03c6Uy8fNs939vUy5YIfhsxcbIQCm0+mtrU3V0urzs6qqHh2dICRZqFVUVYURMJFOKNNpDIG6YcUAQlIcghC9TrderVmG/vzJejaVXF1eOTg4IAgqlcoYukUTZL1USUqJZrOZSCTS2czJ2Wm33VlZWbFC34nDmbk5AIKavY5iGZdvXn/07Emv11uYW5zKyuHuXqVQ5Bhe04xEKrmyttYbDAiKKZfL9+7dGw8nb7/xdrfbn6gGQpA8TamTYbVRXrmy+rOf/tJz4qWlVYKgIBRyfWc8Hm1vb4MwtLy6dn5yqipTKZGJQci1bY6hCZIKQgACQIrAZucWLjpdAILz+bw6nh4fHCezOUYSDMeGcDiRkrYPdjLlHM1TsednRFGRZVPX5PGEJsh8sXJy1ioXizxNDYf9yWRSKFYOT8/bnb5uW9lyEWFwOw6avc55sykIQhQFIRComnZweMwnpa3DQ5phWV6wNA0KomF/jOGUH4ceEOiWTnJUdXHmz//xP/zp3/1IQPHOaVNXzY/vffV4Z7+nmxFBEjTlmhYegbETyqYb4CRE4K+//qbEip7hFFLZaqEcBAHNMlIubYUejKEQBAWux9NM96xpylNlKEMAhONkuVSlERLyAYZkKJJBIJRneMdyOU4Iw3hpaaWYK0ZeiIKQwHLFYtHxXC8Kz7ttnKUjGAw9n8JwHEZIDKdpGiYww7GT2XSlUY/BaDQZ4hQmJCRFnfpx6Ie+PJn0ez3TNs6ap4lU8nvf+16305pOpwCCwiQJwoCQ4o3I18PAjWPHC/b3933XW6zPsRhJQDDP0aWZGSMKYwwEYGhzZ3ft6o1UKg3AULZUkDIJAAby+TyJYmgMjbrjKwuroWqSMMln54+G8ZdPmrbLAhFN4rRhyiFg24ClBy7OcijOIjARR4FAY64yuTY7e2l2oTMevmyeQq+9+UauUByPx/1Od2ZmZmVpmSPpUiZ3aXmpfdaUOAECANf3KYYej8embSm6NlZkEqegKK5XypLI66aGEXihUMjlcuPhKCklSJxYf/okAMO7b979y7/8RwuzM48ePVpeWZ1a1kRVWZrpt1u+oQeOQzF8fW7RD+OzsxMMhZMJUZmMQRAkSTKbzUIwTFKUZTlgFENhPOx3DVNLpVKWbiAgpMjjX/3il+12W1EUiiCjKEIgOIrCGASkZEKWZQRBUAguZrOZbOLkeD8OAgKBYTC2DO3k5MTQzFs3b1IYmpZEU1WUcS8MHBTFHTsIY8j1w6mmq5pmWEa702lddAaD4XiinDdbw1Ff06fdbtvQ1X6/f3JyakyNcX94fHTQvmgNh8PJeGxMlO5Fq3XeHAwGp8dH416v2+2iMFavVpKi5LruVNFiEAIQJABAiCC4hEiwBMXQ7e746fqTr+59+aMf/ejo5DgpiRLH6JoyHPRkZTg32zje3zVVPZPJnJycACDcaMzK6tQNHJ5nZVnudDoojOXzBX2q5dMZCsN7rV6nPQAApN8fAgDU6nQlKcFSDAhDhm6xLHtychKFPsewzbOTnZdbxWKxNxhubm5GUaSoU54XYQCEYTghSvMz859++qlpmm+99dZoON7Z2ZldmB8MBvv7+7OzszhJu36AEaTluOVyLY7gi4v28fHp5atXDo6PNrc36zM1EIxPj48Zil6cX0AQzDAM27Y1TSuXy3t7e51el6RZeaq2up1kJktzbK8/pBmuWp85PD0LQQAnySiKDNP0Y6A/nrie9yodkcqmYxAcy5MwDOdnZhVZVmS1Vqt5jvvFZ5/n83mKoj765ONUJjs/Pz8YDH7929/MLy54nvf06dP5+Xkpldza2X6xubG6urq7v/f4yZNsPh9E4ef37iVTKUEUj06OQRjKl4qe5x0dHd25c4dl+ZcbG7ONmXKxtPHipaIol65c7vb75xcXs/NzvV6v2+1eWr2M4/ij9afXrt+cm5s7Pj4NIv/Oa7d6vU67eXH1xnUYhj/+6NMwihOJ1P7unqyMr1y/MhiNt7d3r169OpkoO9u7qWQ6lUg+evSoWCxSFKVpWhiGvcGwUCzvHx6jKC4Jomc68njiur6sThlB3D8+SedzmqaxLGvbdrvbTqQkAIopBpc1OQYA3TAZjjs9v+j3+8l0+tnGCwiCLl26JMvTra2tQqEAANHGxsbXv/7NTqfTal28dvt2FEX3H92neBrGYM2yOTG5f3CoKEqtXLn/5Ve3b97CMKxUKQ+HQwACl5YWvvbuu//4H/8vj4/PpVTKg1DN9TwEcuPIdNxEIknRLEKSiUyOE5JLC8ue4/I8z7HCycnJcDxKJNO5fLHf71eKJRRGNjY2Qj+gCXJtdRVHsV6vp6rqeDwe9PtwDCAAGARBu3OBIIhpaCzPkSTueY5tm6qqbG6+MAyj0WjwPAuCYDKZJBn6vN2SkslcLscwDAxCqjI1HAsjiYkit9tt27Z9379y8/pEnT58/AgAAMdzO70ey7IEQXhhwInC3t7e1uZLHMct3aAIstfpMgyztbUlT9VEJnt0euL7PoHjAiuMRpOzs3Nd17PZ7HA84kQJw7DBYABB0LA/wFDKNJzBYAAAkT5VfM/LJHP5TAGFkcePH09G4/2DYzqRx/n8r794vr7fieicGuAOQDkAOp7ajJjqq9bYCCgx64FkZ6jgJK2qUwwB7rx2U5kOoc2t7a2d7UwmMzc3c3Z4Kg8mb752d9QfTGStNjc3VWXHNS5a514UsumEEQSG7diON+qPCBi1dHlldf6dD74+mMqeY/EE5dqBrtm261y+vPbd7/7x7Fzj9PT0/qOH3/nWB+lCFpe4iCKqc7MrS8s5nssL/Gy9YVseCiLFTAECYt81cRzNZrMsy08mSiIpXrTOfT8EQTiVEC+trVy5en1heSWOAsc0XnV5ZUXvdYbto3MexWx1QtFYiOKfPH58MWqTHC5mOE7AA0f+s+9+Q5DIjZ0Xju+wokQKohMD+9tbHEUsr8zDsL//8hGDxjAMK4bjRrDh+hGIeFFkhUGMonYY2kGsWLYLxLrrjpSpHXiKaY51nRElzXHGyhQlyBACpobZGfSPz89My+qPhu1e13LMdrc1Go/3Dw7k4YihyXK94aGwGUdq4NiR63i2qkxDPzANi+Xp/ljuDEb9qfLbjz79/PPPf/gf/5OtT0WewpDY883ZmToQBboqSxydpgnQc3zAb48GZ2fNbLrA8dJUNz03SiXS6kRtzM0yPAcS8Ne+/Q3F1A3bun7l6vnpGYah6WQynUoMhr352YbIMgc725IkFGulo1arMjsnphMogTt+ZLo+SuBAHMuj8cV589bNuzhKffHJp4V0upAt/Ornv8pkMtVqdXNrC8Ow6VTrdHr12uyz5xv94XB2boEkqZebW4IkwigyHA4RBIFBgKeZw+0DdaziGM2wohMF65vParP1YqV4etYslWeiGH329IVjWqViPgyjnb09GAYhCJgM+klRCgBwMJGDIHIcj6bparV8vL93vL9XazQcL9AUkyd4SRBC10vx7Htvvb67+5JPSvl6ff/07OSsrVnu3NraYfMMx9HbN28cnx5NDb1QrwvJ1O7hweLiYj6fj0EAJanG7BwUxeP+gGKZk/MzjmZszQDDaDIcTUbjMAxHo9FoMk7mMoqunbdbGE1SNN0fDjiOy6bS8nhiGSZBUkdHxzgIpwXB9N2HT5+BEShxPMdxTugLqYRmWkAYF7KFKIqaFxckzdYas+12VxTFUrVycHAARnE2nXn64rnlOhiBW5YTR0C31w9CgKQYBIZL+fy1lUs8ybiWGwMQxjD98QQMAN/2d08Os5V8GPsjpY9zWLaSaXbOaYY6Pr+IEJTkBIwiytUyRRGarGTEhEhy1kjRB+O8JBEITJEkGAGKoviBNzPTyORSqqEFEMSnMiGGmr4LguCta9fW79/befHiFz/75dOnT/9f/4//5hvf/MbvPv4ogEAAJXU39gEYoWkAQjCSgHFctqyhbcMMOzuzuDyzmGYlAkQdxwMwJF2tFGdm6jOz3XaPJ1gaIfPJ9OLMHIZh562L07MzKZ0qlAtSUsplUrZpIBAo8hyMgLzIiRJvu5Zm6xNNFkSOpogg8K5evVyr1S4uLnrDAUEQo15/3B+kMsmpMY0h2PO80WAIw7DhOBNVoyg6ISbGw5Hl2NuH+53RoFguy7LcaDQKhYJhGJZta5Y51VQYAXvddjqdbtTroWZJKEmAWCFbImEyCqJbt26lM1I2lSwW84wkJYtlguE0Tb+0uqZPJvJYoQkywdHdiyYEovlMMbBdiWGTPAd64clJ07S9REpSramPwyBNCIKAUbRUa1Aziz/fOPrsRJmieSNKcdKs4WCKj4dsqmuAOshMQVqJ4JBAERx2pqOVcg4CYWRpaYXjuIX5eQiIx8PR73/9IYYRiq77cRSEIUmSs/MzMAqFMGj5rizLvueRODEajUzbzeWLcRxjGOb77tMXT8+bTctxFxfnBZETeXZ3Z9/zPMMyJ9q0PxqCMKwZ+ouNjefPn6uyEnn+eDiJ49jWjGqpqE2nve5geW2VZpnz0zMwBibylGY4HCMxjNBMQ9WnLMsq4wlH0oHrdbtdCEYLhcJUVsrZ/KjbT+fSmqEyHLu8tqoZ+mA8GIxGL3d3NFv//P4X2/t7nCBwkiimJIplQDBOZ1MbG8//r//1/+WHP/zhV/e+QFEYp3A/DEb9AY7gYRgHQaTrOgyDnuP7fqgbBgSjYRgalj4Y9TVDDYBId6xsPqc7VmvY7ctjeaqEMWDYlm3bXuCruqbbuhO6qqUBKGi4ZhhHOEmUKzWSYcMoBiAwBGIIgjAYg2HUMh0ExXGCwFAchIGDg7Pj0/PPvrr3y9/96vjsOJfLjMdjGIQQCPRtc9LruqbuBS5GEoViudVsubZXLVY5hlEU1QeiVq/bV4YkRz148pAR2V6v02pf1GfqIAju7+5SBI4gyEieyLJcKhUc3/HicPXS2tHJse8HlmNPdQ2EoIuLi1arhRI4J/DHJ2e269y5c2fj5UtOFO7cvhuGsaIoQRC0u93llRXXCx48fjS/uIyT9PHZme04N2/e7Ha7QkLCSML2XJbnVFUNPb9eqU6n6kieZIuFcqXW6nbiGKw1Zk7PzxAMz+TyvV4PjOJXGWdBkCYjWdU1iqIEnidwnKbpcrk8nU73d3dnZ2byudzHv/89iqIYhj199uz8op3OZnd3d588eXLlyjUEQcbjcaMxs7i4aJt2s9ksFMte4D9af0xzLElTw9EohqFMLrv+9KnjOK7r2q6rqKqqTAWOB0FQEISprFiWFUURhhGvvvhhmEymNE1LJpNxHMuTaRAE6WRqOp2iOCkIAsvzDM/ZnqtpmqZpiUQqkc7EIRRHoGHoBI0zDJVMSqIoDgbDRqOxurp6dHJsO26xXG1ddNrt9u3bt1EUVWVlbm4OQuDhcEjSdCqVGgxGrCiRLKvpOoKiBwf78lQuVSqqqjIMIwmCoek0Ta+urnb6vdAPFucWJ1NVVlSCIARBqOTLvuVlMjlDNxVFcSxzfm7mxcP10PEKuaKqTB3HQSB40O1wHEezbHc4zBRyJEm2mmeLszO+bawuzq+tLtE0fXFx8fjRk/Pz8/FEgRGCwEmB5yhBwFjWDALD80IYpljOcj0Up3UnRCmeS2QuX7sJQdBUlkmCDmMwiKIwBlKpjKYZ2XR2pt7IpdKj4XA8GLuu/wpZUyiXBoPB5sttHCe6/b7v++lEWlM0GEL9MJ7IciqTC8KQ4VhOENafPpmZn2MF3jDNw5NjCidYhiqVSjOzs5ZlvbrMsSybTqd10/DDsFKvISiq6zqCIEEQTCYTTuAHoyGCYefn54ZhvCq4LCwsZPN5FMMWl5dardZoNNINFUXRKAJEMREGQej5x8fHFxfti0673evDKGq7DoLiMQBlkqmEIMIgJHK8PlUW5mYIHJ+MJhgEe7ZrmiYAgSiGq5puWUYmmwrjqFSpDoZDRVdL84s9zZZDZLetdAzwRIk9phhyFTRVL6zewbKNcy2K+BKUrI1DzEUwgiBxBIdEKflyc9vQzCgIMRxBUCgIQl2zCtVSDIGO58Zx3O62OIlzw2g0nCw1GvVCIVtIrVy9DGLUebM/7U+uLi/LhnbabZZmKjiFTpUJFMbnxxdgBLY6F5VabTRV9w8OwcCLbKvbakVAvHN4QjBsKpXa2dzA4PBgZ+v2zTurl68AGLa1sz1bq79+87bjByCKcRynqspE1aaG2e22p8pkPBqVisViuaSb2vn5aSqdEEWxUm84MZgulIfdPui4cATRNItTLCWlHAg/bA8iCFleu5TIZWRdBXxTEsif/+bn9x4/lNIZlKBJLulFIC9yBA6lBQEHYc92fNdjCMIzTTgMSRimMTT0bEtXJY7hORpFIce3NUO1PVtIcBAKgBiIkoRuGBSGO7YNgxDLM7bjOKEfQaDhOVPPDhDE0HQKwWiCloQkglFRCMUR7IUBhCIQAoJgHIR+5Ps4ArM8gTI4gGKK4fzyN7/96U9/+fjRs2fPnqWS0vLSnOfqt25fqzcaumn7foBCKAOh7eNjOIowAlVM3Y1DgnsFUPVd255bmMMpvD8dxQhweW112O/Zro0QWISAJ+1zGEMJghh2e4HjEjgFgQhF4LqmlsplIZlQDf283VpYnu/0uvfX15P5nGU7/dHw5ORk0B9VyrVUKrW1tfWK7jscDmMEIkhyMplcXFwUy+VerydJUhiGD9aflOu1+uzM0elxJpMulUraVH+5sVXMlNpnLUPTK5USCIOGY5M0a7u+5ziNSsW3XBBEcqXK/cfrxkhO0ywYBrI8nowVMZE6PznHEfyNO7en8nggD9/6+nuFSrk3GnPpTKHWsCxn1B9xOMmgGOj5mUSikMmCMXB6fiEkU14YWbYbBAEEgpwo0AIXxyDH8AAA7O/vz8zNwzDaOm9JvMTzYr8/VHWjXK2CMLR/eDg/Px8EQbfdKeVLaT6pDsdQDHAMNx7L27s7fCph+u7JyUmpXM5Xyxf9PugBcIB4fqToOieww367fXHG0pRmmQACjwfjvZ19UZBc1/U8HyfJer02GPZf4ZoHw65pGqlk2nFcmqISiQQMw1NTByn8qHNGiAzGEKed03Q2MR32lUGP4xkIBs7Ozq6uXi0ki9vP9iIPzqTLF60uEMUEAMS2A3lhIVuYjCZhGGYS4uxspdNtub5nhzHNS8N+50/++I8CMOqOR0et1mm7a9ruP/izP9169NUP/vV/95/9+Z8VEsI/+7/9sx/93Y+dIDLcECJw2TSCCI4AtC9PQwzVQhvnKYohTceGSXYiW+nSrJguXb9ym0CIo+M9IS1MfXt2ZXmqabZunWzvKe1++/TcMW3H9aeqCpF4KpctlIo4jh8dHeE4ubS00htMsuXqH//p913TsVXTdNyLTtcJwk63bxm2KKROWp255RVlqh0fn47kydLSUmBZk/4QIXCYwPK5DIEiCYH3PO+i29ZtC0GQ/b0DiqIIhs5mszRJ5TJZVZnGIGD7rmk7hmGCEYQh+GAwGivTEEFOW20ERXGCgCnKgmNcYDGC0Iyp79nlQrlam51oppjOvHb75mytyvCMG0ab2zuu6yeEBMcJOEWKSXF+tv7eW2/l80UARqRM9tnuDsnRDE93+51CLv/u3bfBGEJIXEhnHMeDYkSVDTGVcXFmXzY29XDLBO8fDT99cTQG6ZnXvt4y6d0RICzcVrGEA7BAzEIwDKdSKUPVLs4uyuUyx/NBHPE8a2g6TREcx1m2y0vi9s6OJk9mqjVHtQ539x3ftXzbj8EIALut9uHevhcGqVx+MlVc32NZulavvHqQ0lVtPBmura5yLJtPJJZmZhqNBs3ypXrVch0SQ2kcnygKhMAsy9ZqtV6vp2gqw7H94WBudj6dTB+dHCM4ZnuuH0QkQejyFIghDCfnZhc0TesPuq5rX3TaCIoatjMaTUgK1zTNtu2z84vecISg+NWbdwrVKogiGEkgJI4RaBC6X3z2saIoyWQSjAGR5wPXsywr9D0wDi3LsCwLiGMSxwPXE3mRJmjP8YEYMg2Lphld113XtT3XiyI/Bs5b7e5gGECAF4WqbUYwaDseAAA0y5i2DaAwRVGuZxMExrC8YZkgCIIgiKCoKCVz2QJJ0giGEiRtmCaMoSEEBFEEQRCGYabrBTHghhGEoBBCKFOtPxjef/jsk08++9u//dv/4T/8h+Pz5sLCEkNRc7MLksjbupZPJgxLMR1jcXEeAAB5PPFdF0EQFEU9z5uqOorgnX6v0+sKkpjJ5frDAUbgb73zjus506li6hoIgnEUCYLgeZ7tOmNFBkAYRGCGY7f3d5LZFM3xCILhFBmDQL5YXlxebnc7iqzOLixOdU1VVYZjIyAcTsY0TeM4rhlGFIPtdnc61VZXVx8/eXJ0dsLy3GA8aLVaPC8kEklZVgiCdBwHgiAEgdrtdjabi6IYgqCzs7N0Oh2G4cbGxt27r+uaqaq6wHKlUoXm2E6nc/XylclovLOzs7S0RFHU5uYL0zQRBBmPJoZuQiDM0GwQR/3RUDfUIPBc100kErVGQ9N03w9fJShoglRVFcOwV3Q9hqIuX758fHpqWOb8/Hy73e50Otl8DsOw89NTGEIbjVlN1y3bTiRTR0dH3W5XFEXfD8MwTKVSGEm8eLEhCEI6nX65tXnWukhlsu12F4XQCAQc35tMJlEU8Tw7Gg0MUwcAoFQsoxDSbLcgFHEcb9Dru66bSIh+FF502vV6ncDRvcODfD6vKMr+/q7re+lMri+PJ7qK0WR30Oc4IYqAZDI96A1r5UrropNJpXVdl2U58NybV69P5QmB4YlEwvc9IIo4hqpXymAUEwTR6XQ6vS5JURTHtnrt1cur6XT67/7ub7PpzNxM45/+b/7X796965t2gpMO9w/kqfsv/+V/LysaiAKpbAYjCQBBHd9DMQKEMNuPpFQWpWic42CGhCkKp9hMtnjp5p25pZXlS5d5USiVSlevXu0NB14UKoYGwahpmrbpTKfayckJyzDpdHpucYkgCNu2z87OZEURJF43DYIg5ubn19bWvvzyy2w6k0okGYZJpVKlSrler3MC3+l0JEkKguCVxTcMQ0s3xv2BZZhTU+8OB71eDwZAXdOCIGg0GgLPIxBE03S714VRpNfrvZKdxWEUhUA6nS4USjCMIhgaxpFm6DAMxyAAwBAMw+1eV8ykYgjcPzpWdGV+fp5lWcuyBqPx5avXbNt+sv4IxxDbtvPFwuzsLEVRM/XZ5tm5YZmHx0fb2y+3t19GURTFoOl6OEU25hrD8eD09HQ0GftRqKpqQkriKDpTb0AwQLPUy92933z2edtwz6fuJMK7Vtg1gjNZ9zAu5jIDF14/G5pU+lh2L8wAsm2TpPAoinCcBGDivNMZy/0wcqAo7DVbvutVq1UUIT0nvLlyifBDdTzNJHPXb9zc3t3pDtqt9nm1Wry0ukzhFM+LopCq1Rowio1Go3whrU7H3/2jP15ozGZ54drS0qDbIUnS88JWp9MbdLd3t+RBb6beIDgBIEjLs2dmqq5lzs3PS4WsZttZKV3OlTTbjFBUSqUBANh8+lzXdZyhhrKys7NH02ytVsvkc4Ztua6fZAUaJ/0olgr5TCGfSqVyyTRDkP12azwc5TNZ0zQ5mrl587ofhRNtyuCkb1iBYThTlScICoQCx4Vh2ImAEIYRBAo8nyQZz4/DGAJRPADQEEQCEAFQUneCGCFihAwgxAdhK4wtP/RBCEBhRuBDGLTD0HBMgsKDMDZNM8myoWkCXoBAaOB6hm1hDKV6tu/7gsDhDOUDERBEtut4URjEsR9Fhu34cWzaLgTBCIKapgVCKICgrMCdX3S6I82JkX/9b//TP/3f/R9//KMf//AH/+ns7BiInWSaNUwllxV7F6fldCYrZRzdZ2leN+zxROYFUZ9qQAyBJKm4zvHpSb1ay6ZTh7s7YBzBEKCo02Qy6Zk2jRNeFFqBxwn8qz56CAJCKuED0XQi0wR5cXFBMSxNMyfHZ7btkjS1t7dHkngmk7IsA8cxFEfz+TwMwwcHB47rLcwvB350cdFaXFpiRX7nYF9KpqrVaq/dMVQNQVFZUUajkalbYAjk05nd3V0/iCAY41jeNA0wiu7cuDHq9fhkwgPAwWgqy1McR1eXF4/2diEgohhOnmqWptMoDgSha1opXsRBuNfrBWC8f3FGpyQfhDqD4VmvSwm8PFYIGM1KSbk/HPVHoR9JrLi/tXfWPLddZzIav9zYmFuYxxjq5fZ2tVpNZNJTTauVK/P1mX63t7m5WSgUYgg0PSdZyCXymeNWM53JMBy7ublJY8Trt+8ow3G337392ms0wRzuHa9duaxZumnqAADUao10On90dl6qVQPPvWieR1EEYzjLc57nYTBSyGS77YvBeASTuOW5ymiIQWC1WhlNxpZtNxoNWZZN07R8L5HJGbpNIKSr+xhMdXpjRkhapo/D6GAwNAy9WM/jFPjO69duXlo82Nv2Aj9XKim62h509/a3pQTvOI6im7rlSpnkleur8/M1TZ/0xsOxrFi68uP/8d+zkU+4zo//0w//z/+n/8IMQFIU/7u/+v/9l//s/y2m02ZgARhAcTgMA45tWIEXxEAQRGEY+xGAUhyAMTDBpTP5hYWFy2vLq8tzMApONaVcqipjtdvudrvdSr0GwFCEQIzAF6sV1TQmEwUIIxonQtdjWRbGUM0yIyg+uThVNeVwdweKI9s2U5lkFAWlYp7EUM93oijAaVQZ9U1NDUJPnk5IDA+DgCLIIAhOL5oTbXp+frq58bzT7oERVC8WRZZJJtLVap2TEiRBkyiBgEihUAJhhCAoBEIj208JSYQgx7pOUkwYxnEEAgCEEGSpXGUZhmGY2YV5jCRc21EUZffwgKDIYX/gGObK6qooSSzF2qYznijjsUyi2PL8guu6FMOACKyo0zCOM5nMq6L7F198kZCk69evy4ry4PkTnGFM3bGnBkuiCBqGgIPQSHlmrlybMQ3LCSKukI85RtGt9eebuMAS6fQEwB2xkL71pixkIBAENU2jCNKyLBzHPd/PV0oPHj8YDvt37twZDAbn5+cLi8uXV9f0qfJ/+N//0//iP//P/+v/5v/u++Hrb769uDgfhr7IiSzD1+v11157rVQqlUqlSqWyt7cHAIAoigxNDjrd0WAc+hEEQbt7exEQchxHUpQgcI5tzs826vWZMAZ6g95wPJyZqXueR3EsI/K7e3ssy77+xpsoivEMCwFgBMEoSTmua7tOq9N2LDefLVA4NT87B4GxoWm1Wu3y5csXrRYtCJlc1tCV85PDwHczycRbb74OREG/2/vy3r1Pfv9hgudQBKApgmc5hqSiwGVY0vddLwhIhozACEFxFCMACLEdz3JsAAIJkqQY1nG8GAIphsNJCsdxxwtolqEZNlsoEhTJ83wYRwTPJ7OZKIo8z8NIAoLRMAxeMY9gGKY5luFoPwqjKOJFDkVhmiYpishVSxCC+QCAcqwfAwTN0QxHkFQcxDAAYzAWx3EURZZjExSHYngAQBTH8AJ3cdE9OTl5sfHiX/7bf/3//Rf/rawpDx8+oDD0z/7w2+NOmyNQS1FSoiAKwmg4EHhO5HnLsQmCwDB8Z2d3NJrI8jSKAFFIsLxoWRYIQ+etizCOgiiK4tjznFe3DRCCTMNeXVlKihJHc9OJLI/kTCbj2N5wOKzX6wzDeJ7neraqa3EchlGAomixVGEYrt8fztZnGo2Gqk0jAIxBIIhi07Ys14khgOE5lMBsx200ZiAAOj0+vbJ2icBx0zRRAhdSiQePH3366aezs7NBEBweHaXzOS8Iv3rw0HKcbD7neF7//0/Tfz9ptq33fdgKe+0c3xz67dw9PenMnHzuvecGgJcIBAiAEJhE0oQYJNsUZUq2ypJc5bKsX6Qii6QsqlgmRZoskmLRBEDQvATuBW489+Q5YfL09HR6u/vNeeewgn8Y+z/Yu3attfbzrM/z+U4nmqGnWfGSYqaUUlosl4ut7U1KC69cXi5X+wcHEGPTtl68eLFczl3HiYIQClAul3u9HgFoa31DlmXTNE3TvHXr1tOnT4UQe3t7z54/H03GEKPhcHh0clyr1fb398fjsSzLmqaFYThbLCzHnizn/cHgtddee4lDKLJsGMZ8ucSYXLt27ejFi4LlmqZqigwB8sOos7G59FelUqnsOS8LlCiKVF1bLuePHj/eXN/Y7KyHYUh5Ua01JKIcHx+bpgEE711dbWx0JEUuEpaGebXSxFCOgrjIqGW7WU6jKOJCjCYjCtnJyQsExT/7p/9kMhru7+/PFoujs+MoT3VLv3X3znK5rNVqhmlallUul3uXVz/zja8Pri6fPHt279693/7t3z0/7/6Vv/yf/nf/7f+9272I05wCKKkaFUhSDIaAHyc55XGcKliydJMQRVL1IEk4Qs32xq1XXrtx687P/8Ivb2/vLpfL6Xg0HPZNU18Ffv9q8NV3vupatiorAIDNne3t/f1Guz1dLDGR+sPheDw+enHshwGl1PO8NE0JIZqiCsaTKCYQDfujOE4ViQQr//NP703HEwljUzFUWfGXyzAMFUVJ03Q6nU7ns5cd5pel5OdffrFYLVer1UcffDgdjE7Oz5aBTylljO3u7iqyfH5yrspaURSz6UIIQSmdLxe1en17e1tRlEqlkuU5pTQIAsG4oGxrbc1Szaurfp7Tvb092zQHvT6l9NPPv1j4AcZ4Op5Mx5N6ve77fqPV3Nzetl3HtJxKrcGK4qJ7VqtX6vV6tdbQDater08mEyjh+Wrxyp0bG5trH336yWAwYAIevTgbjCY3bt2pNzuv3Hk1yVIAIUBCUtD5VVdg1NnaHS0jpdbqRjlqtxov5zs21zvjYe8X/9jPS0TZv37jzht313c2au2mYpr9wRUD7Pjs/H/4W3/zH/6Tf/Df/nf/t+/9wQ+jKCNA+uZXv3H47OTycmBbRu/q/NaNfcPQHj14+M1v/MzR6dlZ9/zeRx/Vq5WT0zOzVLbKZaLLRMGWpW9tbAMOZaL2r3pr5Yqtqkt/NV8uoITffOv11+68+vjxY7deSQWdDOeGrNMsRRg0ttZzDIqiUCQsOJjPF73z3r33PlgMBuFyUa9XkyR59viRQYgfhJfDfhT7r9496Kw3jo+fPn/yKIvj8/OL+58+qBq2DgQmlGPqxwlHGBnqKo9Ux+CQAp47lhFEWZgWnHNNJ4apEgkmsS9BbhlK2XMUghUJWipxNCUJVoZCJM7zIF7NF2EYTuezJEs1hdi6AQFWDL1AIBUUqSqSSRBHaZrqmGgIxYkfpgFCwHVdjHGrs97Y2aWSLOl2lLBgGQiBhYBZlPGCEoQ554SQNE0xJpACRukq9hWDSDJSDDVFMMHS4fH57/7ud//JP/pH//lf/2uPH9773nd+R0PMxNzTpBt724ZGaJ5hJoLZMgriWr2ZUag7ZS6kJGWrlZ/meZTHhm2UvLJr281ms1yrPH36hEhoNV8BKgRjQbAaXfUt1TQ0vVqpVCuVarnS612uVosoCk3b1ExtFfiKohRF8eTJE0mSTNM8OTkZ9vu1SjUOQghQuVztD0dIxtv722meQowsy3r08IlM1FdvvzqbTi1DNXRlFawG0+nN117d2t8/6Xar1UqtWomymAL29lvvnJ1fXownxLQwxgVjnd2dF1cXmmXuXT94fPYcmbKuq1gAXPA3Xnlt2hv3zq+qlntj/xpAYLKY3b7ziuOVhuPxW++8nef56fFJq9EikhxGcff84s6N28HCPzx8bhjmwe7esD+ACik1ajN/yRjzbOfk2XORFQdbOwohsiy7nicp8un5meu6Qog8z6ezJUZkOOyfn5821+rlSql/dbG1tTWaTl6cHEdh3G52+sNRkhV377xCEFxOZtVypbne5hj4fsgKXnEdjMCLkzOOsOEYYeh3Ko1wvhRCTKdTGucsKdIoXS6X/9X/6b9Mw2C5mhY0KdVLRFdUz6w26pLAJjG/8va7R8dnkqyqqpqm8f/2f/9Xa/X6d773+1FWvPbGm+//+MefvP/e0ePH//X/8b/8f/zd//mjDz794rPPAABYIZSQaqfGdZ2YZgE4F1kSzohETUNZ+iFlACNVIyZLBKQIC1wp1zTPa27trG9sASZ+6ed/oVopvf7Wm7/2a78GAOpd9inlYRgmaRT6/rX9/bVmiwnOBJ8t5ge3b66vr0tEaXfWBEYbu9txnlmeO1vM67WaBKCC1GaltZz5jUprs7M5uOhjKNtWaT73/UVIOE5XsaDQ0B2WA8DwcrkslculRkWzzd3NLQPLqqy8cvcu0VTV1BVFURQlSpMwSUuOOxuNwyC+fnDztVuvXNve9RzXcRyApPbauq6bO1u7V+dXwSq8uOqtgtAwjMD3/cmMR+nJo6c6lPxV5LkVqeB5uNra7hiu2dncLTgSEBc0cwxdlvDVuP/45BAr5Pj4tN8bWbpZK5U0Ik1GQ8u15r7f7GwMBiMI4c/93LcNW17Go8cvHjAo6s3N4WDxF/7D/yjy4+9/7/uYyKtVoHAYz6bDwfn9B5/4wWxwdb4a9BeD0XA4dcsNJAT82Z/92WvX9uqNarNR8Wezo6Mj1bIazfa//+7vX7t5o1Txsiw5OjpKaWG6nuHYumHsbu+cPj8Ow/B73/vewcHB1WXv8NkRYPzhgy/Pj18UjP70w4+CMHwptdja2upsboynE4yJ67o3bt/qbKw3Go3ZbPHStowh+vY3f+ab7369UvZee/0uIeQnP/7R/u5euV7LKVVlOY1iU9O3t7fzPF+slnGa5HleqVQAFxenp6ZudE/ODFVLkmSxnFUcr11vlUueZVkIS59+dn88nFzbO+j3h7blAMZpkSFBaZHyguZ5LiDgGAoABAQpy4imUsH9MJRVAiWY5gmlNIgDSnMJAcgZhghw6Fg2L6iKiS6T9VbbVLU8yhzD9CynVqkqiqLrep4WBEtFlhUZzeJUgpjmRZIkRCWUF3mWqKqKZUJUFROpKAoGhVsuZVlWa9Qbtbqu66ZXKTjDRGFQAIjTLNN0pSgyIUSeZgCAglIsS5QzgGBBeS5QzqCimrJKsrzoXvXuPz6+/+zwpx9/8Dv/9nf/0T/6R++//94qDM7Pz9v1xv/0d/6OpWtAsCwJ0jCYjkd7u5vtdkvTtP7VlevYq+UiDqPLi+5kMGw1mrZhqkQ2Tf3q6irP883dzavehaYraZqenZ0YhrG3t/eSnLFMO0kSy7IswwIcttttIUQWJ7qu67o+n881Ve2srS2Xc8Myi6JYrIJquZLGyXK5bLYbKSu+eHDfNE1K6aNHjwzDqLeavfHwotcvlUrdbtd2ncFoxIQYTsaqoSuGyYBYrJaSJF1cXd68fSuO44uLi87GukDw9PTUsqwsTu5//plt26WSSxC+urhQVTXJ0uF4PJ5OJUm6//BhQvNGpz2dTgkhlmU5jjPo9zGAnufpiioYtwwjSpOkyJMsLYoiT9LNzrq/WD59+vSlrm44GXMgHM87OzuzDTOO0pe6UN0yLdccDofL5RxCOBqNHMduNBpRlMxnyzCIZFlZLBZB6DfaLd/3MUSWYcZJCjF6dvRMN43W2tpgMJxOp9VK5bJ7kQRJlqatemO1mNuaYelGFITf+b1//9V3v+a6rmHotm1LMgYIjmfT1Wzqz2fDYX9/f38+ndZqNU3Tfu87v5/E8X/+n/6NTnvtf/w7f/vL+5+/OL384otHiqY/efKUEBljnBUFwnKas5kfRjklmg4h1FTZcSwOQVxkWFYYBxxKSSGEpKUcbm5ve5735ptvy7KapoXjePfu3VvNZ5Cz2Xyap9nW5ian4sbBdcct6abRaDUuLy/bzVan02k06mmaFpxRwaM87WxvrqKQA4GJtLW1BRg3TRNjDCF86403hRCD8UggSCk7Pz1da3c67bWyUyo7bryKlvMFpTzJ0ka7lRb5Kg6H48F8NElWgaZp0+k0iKNVGDAEMsHK5WoQBJxzy7Je5sfeu3ev5LrNRuMlJLMMfMdxsiS5du2aLJHXX33tlVs3gyDY3t4s12u6Yy0WMyzBWrWaJkmR54SQ09PTgvIwjkaT8e1Xbjaqtavu1WI2C0M/9IMsyW/duF0ulYoiH08nkiwFUTifz996563bd24bjrt77eD73/9+nmc5TUxXv/vGG0nBNcs1Le/g4Ean0ymXy8fHx2XHa1bra4369uZ6u1kvuW6v2wWUPfjicb83lWjB33vvvYOtreV8ggkZXF1eXJxv7G5f9Aeb27vz5azIU8hhtV7L06K1s/306HmYpMMv7jcaDUKaqqVd9M9UU0kz+ukn97/+1bfrW9sQyY+ePqnXapTmy8X8/sMHBYQIScE8aLValu32h4PpxTQMQ63kbW1taoZ+dnZSxCHRZbfeKJhQJPyzf+Rnnp2epXn26is3AaXbB/t/8+/+rXa7WbFtTTMoFaqibG9ugDyFgF+/eevps2cMTavlSskqhavQ9vRC1VOh1Kubkb+gIndc96zXOz5/TkXMiaybdpoUkAvd0hGANOGGZszDKKZF2S6Hy5WkgjyOhcBIMTDmaZ5JiHAq0jQvcpHTIi1yuYSzOBmsEkIUxgok4YwmlDMJSr3LvmuZq1VgqnqWpDKXCJSDKCYYW4ZRFMVyuZQ4AxKmFChEUnUismw+nxoEiziShLS/2ZmtlvVm7WLQE0iVEKZJEucZZ8zQFcqAkAAHiMhymiRYUuMwkVWdcxDFlNNCKJLAnOgaE/wqSopZIGMoYXj4/MTQ9HLF+/GP/lBQVq2W//Qv/9LGxsY//1//xV/8z/7jj+999vFHn75x+67C0Vqj3j0/rtiuxIFnWKvppL5TDeKAcn6wszeYjc2yy4oiDleVSuni8hwhhBBSFIUx5gfRahU4ilEUhSRJRVGUSq6iKMsgqLeaiiL3rrqaojRqtePT08Fl7/rGDi9opVxKsyQtUk7AxWDo1erlavXZ82dupbLWar148qzkerIsH5+eeOWy47iT3nA4HPzyW2/2+4MwTU3HKbJscHWBBYjDcLZYEihvbe5O5/N5tNra2hIyX8Wr3nhw486tRw8eBavALrkAgCAIarVac2P9u9/9rqaqRJIuLy9zRrc3NiGEkPHdnZ1hbyBhUitXZEnCmr7ZWRdEooL7cbS7twMxPumeK6peKpW+/OgTQ9PSJDFMm0iKYVkI0vF09OortwnCh8cn5XKZc+77K8sytna2F6PZbLowNw3G2NlFt1wqqRDLAqqqGiRJVBRpVhiaKWNJI5qmGnbZs1wHECgTrGiyEIJwbmn6T+99TAlQiTybzStOaRX6497gL/+l33zyyb0kDesV7+MPfqLLcDmd/OAPfvhyL/PHi/lsMh9OIIGyIVMhgjSXOPeDIC1yDoCumQSrhChQFGmcQAijKOVAAUQNs8IruwCAarXOAbRcjzEGBK1WqkTSvHa52lwjivzRh+/FtXLgL/IsuXPnTqlUOXp+nNNi/9rBZ/c/f3L4LKPFar6gNOes+OLzz954853Dw8N6o3F6fjYZzq7t7Q56fQnD+WqZFnmr1RKQ9y67EMLRYrK22SkyKknING3BWX/YS6KwUa9TXpx0L4QsEr9oN5qL6WRtbc02neVymaeJKhOEpOliEWmKrCqWXSq5HpGkzd3OyfkZxtjx7I8++qDabJmmWVlz0zSt1Krdq0vbMNvN+nI8JoQYun7aPWeStLO1VTc1joRparVa/UUUyLr1yu3Xnr84LDK6v7fTu+g26lUEyfbWlmMon3/+uQZUmhS9y/Nf/5O/PlnMZ6uZIsoQQsdQDw8f5wCs7137hV/55d///e+cnJxFUTIZhXdffd1/+Oinn3x0c+9a4C9Xi7mhyIqm5nnertYWi/lsukyTJEiSLC/Kpfp0OpV0Q200Gv1B79XGq9PJLImib33zm1u726PJbNwfbG9vrpbzYW9YrzcP9teKgmVxZmpmfadkWcZoNJIUybZthCTZsABBz18cNWr1RnttZ3NrPpvUKmUo0K//xp/4wY9+4PvBoxenTIDxculHoaebX//GN87PXjglZzabzGeTPI6rUmU6Hg9n07u37rquvbu5ffj0qaapq8Xi//Odf+e6rirhVZoss2I8npYs11RVUaSO40yXC4aA7blZkV9dXSmamixz3w8N3QUAaYqW5ilj4qx7kqaxbZuc5kEQYSRDgQTjcRpXTCcvKMESAphgmRW0yBgUTFXUJElURYnTjOVUURSESRDGsqoSQtI4SdNU0+0kTQEXRAiFKHkcqLqCkPryWgkxkYRRtVzJihwhJCvScDyWVUXVdYgRExBjjBDKKSOEAM5VWaEgM3R9sVwpusaA2Nha5wD58wXLtSSNEC0CP5YkRFRdEigJIomQJMkURWMcQAglSSa6woucciaYyIvctEwGY0kioGAMiITmR6ddgrqKTL74ori2v3d88qLfu+r1r/723/5bBwc3WCJM07RLhi4TFRLPsA5fHAFa2LqBiSQYH0+G5Wo5yWLIOCF4Np94nlukma7oURQzJqr1mshZsYwJwkVRKApZrhaMFhhjRVGSJNJ1NZj7ghWUUtd1Lct69uyZLMv1ZnM2n44nk3ff/YamacvVXEJSq1Ffzueh71NKdUXNM+ra3mQyqVarqqr0Lq8sx261Wmma1ipeqVSa0wkr6NraWhCFk8nEq5RUXZ8upjnd5JxTztI03dzcVBRlNBp5nifLMs3zfr8PIaSUFkWhKEqv12s0GjpQZ7NZmqY7OzuMsTAMdVWbzoeHh4c379zRDN2wrZxSVdfazSZUtPl8nqeppml5ntPVigsaJ6FlqNVq9fDZ0euvv97pdBhjlFLf97FMfN/P89xQNd8PKKXb29vz2SxLsu5Zd/P6frlSGUzGpUodS1KcppJEWE6XC38ymWztbEoCTsYD2zRNQz49m2yst8pl7+Ls3LUdyzQH/Z6uqBiiH3z/R1CwME1++v5nkobyjBMiO7a3CqLvfvcPkABIAgWAnLGcMlXTaMaB4LKmcc45BwCgomAcIoBBUTCs2QWWiaqoMmUC7e/vm7bllSqj+dwwjO7JUZqma1vr/d6oebd6dHzMGEMIBb6vSPjZ42flchkhCSE8mc9K5cpqtdze2SIEt1rrlxfddqOJIdrorDuumcQhKIHVanV+eiIrkuuUbr5y+0c/+kG1VK67pel8+do7bw0Gg1q7Pro/TPJsa30jCAKiyKvVKmVZtdmI8pgxOpqNsACKopXL5TiOe4NLy3VqpTrlbGt/ezwel1wvDpNCIt3eVZbnaZqutdtZli3nC4KlYOXneV6uVhrVWp7nWxubo8te/6JbbtZ3d3edSmVtbe2Ljz+mNG621p48eUYIwYQcHBxcXnWx4K5lJoE/nsx29vcODw91CWiadnl52Ww2S1X3o88+qjfXBMRrjZbvL58+eWgYph8XsqZOftTLopjFrFlpaoYTBJFTcmiWH714ami6ruiMFZe9K9u0Pv7o03azBSEsUlqu1KbTaZik1XoLnXVPoygYjyaj4fjzz78kkrJaLa5OT188eOgQOQsSGasAoGC+nPeGOhMGlNZba6ZnFVDUW+00yhHDe3t754OuUHHMeRAnOsYwTwe9/mS6hBz/k3/wjyu60Sh5u/s7Gzvbm1vru1vbAJGLy74kk3uffPj86YP1taaka1fTmaIoGxvrDPHf+q3fuv/hh3euHUwXi9//yQ+DONBlQtNsOZ3tbm3atqnrGuVct70gSYUCSu0alglWVIrFKo2I42BdDYOpJDJWpLubG2mwBHnOKQ3DGOccU8AENW0jCv2SZac0yWh6bWvHVczh2YWjGQaQXdVgeREmIRNUNVSIYZREaZGqhprTDGIAMbBdaxX7kkqgjArAl8uloigCo0JwBgSjIkoT2dB6s6lf5JJKJFXBtk4JgrKEoICMakSiNJcQlLGcp0UepGmaT3w/hixnlFKaJSllKdIxcjXJsxTPdetVRTeKXKRxgYEKcoy4BACAmBcsZTxLo5gVQIYKy3IFy0RgzFFOecyYYVp+mMqWqThuQoVi6X/z7/0v//3/+A8/f/j0T/zGnz67vPrtf/udT76496Of/Oif/bN/9uFPPvjx9//w7ddf+/N/6k95utU/v3j+8JkqSSXXRQDKhGiqzFmhEHJ5eSmEiONUUTRWgItufzSaICRBCOM4bjabqkwUgvMoaJYrNGdZVpTL1SzJWVE0m43zq4vrd27NVsvZbG5bzubGxvn5ea1WkSVS9TwJ4DiMIcaSJIXLVbRcaERSiXx2dsa5qHiV0WAYLFeEkIvLXlrk7U7n5u0bh6cvJFUuV0oSQrqsbHY2i4wuZsudjS1O2cVFVwjuGFa0ChhjjUZDRRKiHGFcCL5cLkulkuO5gOA4zziCDIKjsxPbtuM0SbKUAyEQzPJ8Pp/fuX2b5sV0PGFZHq+CtVbbtm2vVGKIQ0m4rt277D1/drK+uRmnqWWYru1AjMI4mk4nzVbNMBQK6M72HqMgjqJGrT6ajGuNevfyAiFkKsZyskhoruhanvE0LnY3t2olj1JerVXeunuzVXU+/OQnXKT/xV//35UN9fmzx65tXN/d+8Pf++6nH336B9/74a//yT/z4nz4Bz/9DJkqhQQqMkVisBzN4oVQJawrSFGxpBBVUXWdUoolqaA8zouUskJwrMhQUguII8Y7t67feOcrdr2Zcn79+s3XX397OvHrtTYCGBVg3pvQiLKc+ZFPNHTv05+2G94rNw6KLFdkHQhycX754vkxB2C+nDue65S8zc3Nra2taqNue65b8na3d/oX3WA2C6YLDRMEoaqq27s7WJK6lxdPnx92Nre8SlUxzHp7zU8SQQhRNc2yd/avcYTv3n3NcbxCAFk3CgA4gJqmFUURZsl4Ph2Nx8PRaGtvVzX01SqYTxd5UQAIBeeOZY2mkyCNLwa9Sr0yn0/jOKxVSpPpCCn4na9/JfCXG+1Wo16ledppN1975c5iMu2dnqI4nZ5fgEJgSUUS3t3fuXn7Zr/f/+STTxrl6q39g+FFfzpbxWk+nU46G51cwsBUoYxynpbqlf5gcNnvXdu9FsxW/bPu7es3aJG/cuPaTqcjMeHIVkVzb28f8Lho1OrbG5udtcb+9uZWp40IqjTqQRxNltO9G/vj6SReJfEqOT+/0GwzYnkKqVSk6WSxXF9fPz+/SPKsoarLYPl8cigKuppN3/rq19vrHU5ZGgYPH3zZbjQ13TAMgyjSPFjZjiNvSeOLy+9/77u5ClVdMQxLluUXh88NXXUc5/DwkBWCZUmzWuaAvTg9uWXbRZ4ePjncP7h5//4XgqVEgvPV/LR7rpjW66/cCsJlo95aBMHu7vajj+9/Z/C7dqO2tbsTBavZfG6oGkBSEASA0TfffPPeJ5+Gcbr0V+W15vnJcbvdWczmeRZjSeKcx3GiMErz+Ozy/GLQVQiZT0e6pqgSRimtVkvLIstZXnbKRZELwRSMeZTePbj5RXyfUq7qMoeiUnE4BIv5ilNmOlYcx0IwWZYgBqvVQrINDIFhGEAAyBmRJC5LiqIkWYwhyNLMsQwkuIBAYEQ5S/NkGS0tzxVCGKpSJKkQlOeFZZpBEBQiR0wAxGVZXmapZulhGOm6WqRpGqaKZXDOFEUhCiIy91rt6WoFBAoXAWOM0TwpUoABkgDlQpYVzgBlQpMVhMBqPieqjGUFADCezUuVSlxkggmiaZxxxdIsy0qDiAmhaqpJ5Lm/0lW13x2XPWc+Wf2f/y//ja7rj548dhzr9TffOr/q/fwv/dLvfe+7gheb7dbzp08k1Vhr1NdqjaP58/7Z2c9+81sZLb787N56u8GLPAyWCFu2agPAS643nU4rtdpaZ3PYH0yXS8dznz9//tprr80XS4SwZVkAoDzPXdddLRejQV+VFciFY9mu68ZxDHPaabUn44nneYSQ8/NzQ9VkLFm6DhEolSsvjo7mk3mzsw6R8INlmsYES3EUSBAEK7/Vaq0WCyEEJtJsPnddt1QqFUWRZRlgXJEIQMiznfFgiBAqsjyIQrfk2bYdB+Htm7eG4xECkNL82rVriqLMZjPO2YMHDxBCjVodYlTy3Muz8ziKalCouioRqKqKrhrtVocV/OUDW5ZVa9RrtRrGcDoeRXEIAFgul5JMfH9F87zmlHRd36mWppNxkcQEAlfX4ji+vr+9Vm+/9+MfPT189to77zx+9OTf/Na/rFa807Pz9nrnb/yN/+zoqEsBePjl09/9V7/DmMgYe++9D3ghCgQ0wwzTGAlACClormhqnKUQQwZxmqZYJqIAEEIAoaLpkHIsoSSJFVkLlkvVdTXHViRETIvoumbbNUm66g1UxVpb6yxmyzSLDUUrkrTVWouioNfrvf2Vd85PTy6vukRSOp2Ogshau/07v3NeqWmPHj1yK6U8S7Mo0iQpE0IxrG63a+v697///fX1jdfuvnrvk49XqxUUoNNcmy5nAEtrmztpFtdr1cMnT0tuuVKpPfjkM0mRiySFEBZFEYXhs+nsrTde/+lPf2o5DocgTAgvqGl5y9UcYzIajUqlUpqmru0ENCg5dhxFgvPBoJdG6Te//S2E0MXFxcXFxW5nA3JR8EJAPhwOz88vyk7p+z/8sec4jmVO4mw+nwshGtXa0ePHb7z59o2Dm4vVUjCuq9qXX35Zq9XOzs7a9drR0eFqFXjNloDwlVs3giBACg6CoNZszMYjSZfdUsk27KuLXrBaRXGaREnFK68Wy+fPjuJVQOO0atu/9f/+7fr6xsRfDie9u3duaQiNl6OL/nBjcxtisb65fnl5pTtWvdyaTqetUmc8n1HBB+OBZMnyZBWcnZ9WKzXTsiRF7rgdx3GKLB0NRoBm036fJQkAwGvUjEZltQo+ffDZwc41DGBC48HkUuTh66/cGvrL4XSWw4jKSrxatuoNrMiKohQy5BBcDkZCMNc2H355T+R0ufRt07pxcC2i6dXFOVa1W7dvh1mU5WEcpbbthEGynM6+9a1v3fv8s2qpbJXcJ4dPnVrDcaxJ4A8n4zRNt7c6aRKNxlPF0OaTuWs7x+dnO9ubIRDz6QJeDRxF03V7FfmKa1GRAwgNUwE0Z3HKKRxNZ2GeK5pMdKIS1VRtWmQwyRgKdUufLleykIVgmFNJJppmUM4kVVEgkKiwTV1ASCRg2/ZwMKYZh1xYmkyzRJZAsJgahhXGEZFlgYRgTBRUlaSUccq5rmCSpQjgMF0CAFUoZ1mRhomtW4kf6pqe5zkQTIIACiarRNVkCYiSZ8dJgoWQOMAQUMFywTCGEpFtt7Vc+mEYGoobRRHnPM+ylHPAgarKABRxHBm2ked5HPqWZWWA5zwHguU051lhmmaWFHQVAAAAQIiDJE4JQFGWIV33OTNqzv2j54TImmFcrlb9H/0IMfiX/+P/hEFAkCip2v/hr/21f/6//gtD0xcnZ1//6td+4etf+/L9H9++8wqMw3gxqjYbrqXEia82Kt3Li+F01tneP+9efvr5Z3fu3FFt+8H9RwiraUQxkcpuFSEpyROAgK7J/nyWpnG57JmW0etflb1SGIa1Wu3J8dHm9ma5XBqP+mudZrVWWhxNCYaOZY8nSy5wpVFfrFa84IiJVqv19OljlmeWrkGJHJ8cObYHABACMMYBxqfdbprntWo9T7NRb9DZWAeUvZw4JwinQTTuDTaabRni0WWv7DmC8SzLut3La9dvvKTrbNssChYEgSpjmeDnx8+9UknR5GF3kMWRa5tdxjjNXU0L5wsIkWFZBWVQgGC5yqJUlVVJkixbVXXt7IpZtgEBS0L/6MkX//V/8998799/5/69L29urv303/27d7/xrR8/evTtn/+FnWt7//3f+tsCQ4xAtzerlp3Hj88QAbKiYCEA4ylnQCYaQis//Ne/+7tEVZOsIIhgjIsi0zU9LXIAISISFwLKSFYJIpI/XQKkBBkDEDMKZMNFslr1ShKWVctQPDsM492Ot7O28dlnnxEsx0GoqWoBoYSl4Xic02JjfY0B5gfBRx9+UqtXNE27Go5Klsei6LOPPlBN03ZdJGOM0aPPvrx57SBerI4uLzSvdO3gIF6tHEP3auUM0F/7U7/xu7/1O1WvTJDsmRUI5Hm02rt2vVP3zo+PbNsWjP/KL/zCl59/ES/9/f39Iku9Wm2xWi79xcZmazaZmLodhalj20kSl00vSZICcp4X48U00DTHslVN7ncvAAC6rGIJ/uAP/0CWZVvVeVLEYbJaBaPV5OD69eXEf/D5w+293fXt3eVoDCE+Oe9ubm4CyHu9geU6o/n02m55b+vu+fnpi5Pj1Wp17eC6JJFGoz4a92gccCTKnhctVuFyVSRREkTLRVBprMXJCgAgKPvy8y/Wt7a2rl1DmCxmQ6/SaLXX02papElF0b589JjoiqQSALjv+7M4/ezTT1974+2j589MXWk16xdXg4Tm42iGDDycThqNBmMFY0zqXvaq1epkNh0vpq/ceqXRao5Go8OnT3XT3NnZjsPk/oMv1je3DQVneR6GYZKl7XZ7OOxDJHJU9Hq9mmmrmn725Rd+GFUrdcgFlqRVGhNWpHmCZC1Iwu7FVblkG65V2do+enr4yo3r65vrL7rdWrO5WC2Xk/FHH33iVZw7d+70Loa/951/Xy5XF7P5paRYnmsYxnQ8UVWtXC5Pp9NqvR4s5q1W67PPPi2Xqxtbm8sHD8rl6mA8aFYrgHHKRFEUNMscr2waxmg21Q1dN8unLw5Hg4GGYJElBQNxGEuKIstqnw4sQ7d0w7aMnY1N1TSsWvmnH33sJxEh2MYGBqi1szdfLF6cHyuyZBJl2B9QzgxD84NosfJFDjBEQbACSGSMyqqScwgxmk7mskIUgGRCChohBQOMsjhHKhGCMiIJIcIo0HUdI3kZRiXHnU4mrutmRUoIIVgGvIiDmEAcBRGWZMSopiuMMcYFo7kkSQKJKI1US7FcKwxDp+SkaZokGWOM5UWeJRww1TTSLNN1AxYsp1xW9XDlG7oBAAdIokwQQiilkiQzwZM0V2SNQMQYo4IhiMM0VSyT5YwCqBhWFEWeZaM8x5KkYIS4+H/+w/9lMpupsvQ0otPFcuGv5svZv/vud1zXvajV/STkMtm7tnfVH+xdu97q7CyXy5sH+7/8iz9/ePQ8z3MZi52t7YpnnZ+fVzxbwUgAMR4P99bXbENvtRphGI7HQ8556K/W22txEKqqqmCpf3GZRrFlWVmWhWEoSZK/XNWrNQjwcjV3soJTbmhmEEeEkCiKoigyTfNlN1zXdU1RPc9TNQWHwLGsLE1mq5GmqzzP4yDACNarlcl4lASBrqlZHIdREgZLjnm9VoKMlh2z5tpf3L8vBCcSGl5dWrrRajSKLD442BeCQZ77i4lr2dPRWJUwcWwoRL1SDaLQtB3IxWo2V5BUcR3LdJ4+epQX6U++9+NFEPb7/UlvsFos0rz4zd/8TQh4HqV/92//PSHAs6N/KgAYL5flep1DQCQZAKAa0nzhSxIGGGeUEYQBwlyInOYyUYEQsqbleSZLBAqAAFSIXGSUM16t1Rb+igrBuAiTHMRFdWt3sViqii5rqm5Ye3t7z54981zXMAzK2Lvf/vbp6VkwXVycX1TrdQAAK2iRpEkSx3GsO4amKYhIUMLVeu3q4tKxzel4kiepWpVny5nlmBGlH3/28Wt3X5UwXswmV1098leCMZZl3dPT3c2Nr3/tXc2x5stFEoW2bSuG1u1euG5JlZXtyvZqPlMAL5WqqyA0NOP8xVkax07Jm8wnNa9MKTUM7ejkyLHMJMuGveM33nr7yfOjMI5v37zOCvrs2bNOvWaXPFVVszRdrVbtenMV+kEQuK47nU4cw+Qg6bQ7aZzYpuUnvoQkBKDr2AAAAIBmagKgSr1+2e/vbG76YcwJ9k+O84zubu8UWR6HSa1U7p6fcc6jLBaCNdc6Gxsbj+4/6HZzBGAQZ4QQ1y2VvZIsk8V8mvhx2fNW/nJnd7NRKk0nkzxNK7atquR8Ni38oL3dMRzTq1UsWzs9PnFte+/aDT+INM1wS+7Z6UWcFJbjXPUu19rNer26mk0ZY76/hD/7J/8sJihIAgnCtld69c7d45PzyWI5ni5ff/MN17Xf//CnmqUlYfT1N7/KcvbDD37qlpyb1/YWi9n5Vc9xy7pud7vdJIpVhbTb7TD0J/MZUojvL8u1arlc9ucLzLitGQByx3EW43lnfSPIaUgzJCuHT59sV+sSEJZr5zQrGCiXy0mSJUlCVG22XGytb/lRWKlUXNd9/ODh+flp4C9/9Vd/lebZaDRpNNtn3YvhZLzWbAyuLjnnWNZ839eIKoTAkhqFvkkkP5g//PLeO99898uPfkpZppYct1ThSc6YIIqSFTRMijQIvHqpbBuv7O6WSqV//8OfZEUO8xwLLIii62YWJYqipHnCGBOAYYLivICYNGutrChGy5lpWaVS5bLfAwBUKhVd0eej2aQ/2N3bj5LYD4PIjyHGnlNaBUsmEkiwDDFjTJbVOFghQhRZVpAURj4mhOa5LWuqJBVFAWWUcaGb2mo51WXF0QxKKUUcKSSKIlVVJQEVRZmuFqquykTJs1QCkBYsKShCEstyfxVKMqFxhGRVU1WJMcYKTJSsyNMkMS1LCEhpTgjGAuRxjhCSDIlSChhACEEoEEKUAghhLjJZlouMqkSmeSHLMgQgp5msaFEUYJnoupZlmb8KAQSGRghREBDzedRuV0qeNen1aq7brjUKXrQ7axvbW19+8WB/d/eDjz75yrvvdra2j85O+le969u7hmEcnb1wXbdRb6Vx9sEHH737la9CiObzeb1aS7L0+fPDUqm0vbc7nU5PTs92dnY003r67JnjuWvt5oMv729tbWqaNhj2BoPBH/v5X7i6uppMZgChVqs1n89VTSaKNJ+NVSBZlqWZxtNHjzdba4SQ44vzrd0dQVkYhqenp3fv3tUU9fj42PZsz3Hjpb9arSqN6snZOWV8rdWWBLs876qGKTDyY59z3q7Wu+eXcU5109BldTweSzJRNLU3mk6nU8yBKiuL5cS2ze/87r/963/9r3/4+ZNS1U7yrKAUSSSJU8/zAt8HAOhEScNYIYQDLmvqPIwgBBhCCDGWpDRNZVnmnHMAhRBQAEmSMJHyPOe00DTt5U2+4JRgiRcUEZkyoZtGkiQcICDJAuFSrSmpiqRhzVSWwdIxLU6FqRmcc5oX85m/trZWrlZyxk9Purv7e4yx6XhybWd72O8Pr3pZliAMqs3GKkvdShkyDjJa96ph6CMFYyIt5yvXKem2ntHC0U0I4eXlZRTEzWp9MZ1urLeyPCGajhFp1Cqmafb7/QyAXEaz2Wyr3qBJkcQ5B5LluKdnZzfv3krDIF+uTrtnZsVyKiWRisl42L+8MEwNIDkIw3LJDcI4yHIhIGas5HqGZVxcXZqOrWJsESUOgwLwKItTQU1NJzlfbzQvL04RQgDKguOMFhAj23VeefWVyWQSx/FyteIcVMr1Qb/vea5XdsuNiqLJWOAXhy+q1Xq9Wqu4zk8/fh+qcsZ52avEQfjynJvNJtvb2wirEiIKAJP5bOvablEU/mhqW8YyDOI02t7ZfOeddx4/Pbx94+azhw8ePXo0WSyRKmuOVavVusencRBXvJJp2kEcFDkDAMZxKiB3HCeNMyx4o2qNelcYyWtr6/DuL/wxzdAB4I5trnnVsxfHkqrYjkcUoz8cBJFvWYZAEFC209qolMoXw+F4NrZNRVGUVqu9ub17ftm/6F0BxmezSRYnbsk5Oj81HBsA4Lm2rqjRylckScYSK2ieZLVqFUtyBiHR9Yt+n2aZlDMZQafk7O7tWa6VpimEWNf1F6dnX/vG158/fT4cDi3LqlQq9z75eDgY2La5v7OrG2qz3TnvXh4eHrU7a7qqLMYjy7JWaboKo2jpc8oRJpDRjWr1d377X/29f/j3P7334T/++39/fWs9xVxAqHBkmSaUsFMu2V7J87zj50fPHj10NL1UrZSaa4eHh1JWQACwanDOk1WAELJKbl4UAAndMhdLP8pTz63otsMQsB2PAMkwjGW0ev3116f9sa6ow8GIc5BTXq3XEJD6/T7GuFQpjWfDeqMarVZCiDRNAQCOZ48Gw2C+RBDmjDKa0yBOgkhRlIU/hwoREGKCTVVJlj4vKEcwF4XjOHEca0SOoshwTCEhyhhGSCdKUTBJM4UAMpCEEKsotF1nNhkTCGGWsYIyxGVZTtMUY8w5IAiyLAdcyBKRZJLTDBEEORZCCMQYY5DjPE8RQaquAS6SJFMVHUk4yxJZlgFGBaMAgKIoJAnRopBlGQmQRLEEJdM0izwpksRUECx4lgKMAMQgLgBEQEIAIFBQ8Bd+888WlP6zf/av97fbqqqmRe6VSjcObv7gBz9otzs//+2f63a7ZyenhmFUq1VZlpvt5mw2uxr06/UGQigIQz9Ybe/uBkFwdnJSqVQcz7u4uGjW6wghKEB/2Gu1Wp7n3f/iSyTBN996/dGDB8vJ/NVXXz256gYrf29jK47jk/Ozer3uOE6323UcL89TgfDp6XGlUgnmy/l4cufOnd5o+DJ3N/SLmiMncV5QACQgJMApYAWQJSAAyCiAACAAKAAAAcqBJAGJAS6AhADnAALglbSoYEgmlHJFU5OiCINY1zRacMD5yy/CigxKMCsK1TbiONYlmVOBMI6SRNYUIQQQSDD2sg7jAEAIMcYAAIJxlEYYY0aFbZpMwDCIdMcu0qxSqcaUrm1u1dsdKONer7uzt/3j93/kWnbmJ5ubm6sgUBRlNpptb+8u/NX61rZl2qqqX3Uv4jhUFYVgbOp6EoXz8ShlBVcVCoRrWrBgrmoBIRbRYhn42zvXxtOJH/nr7RagaDGbbe1tT+eLxWzRqtY1CVBKZdNkjPG82NvZPjp+wSREVfnGjYOnn31esd1mtWM5pUKgJ4fPVEtbazePvnwkyXgYTtc3N2mULKYzBYP5fNlc3/LDIM9TgeBsFUgQlS0HAWhYRhhHSRLFYRTOl5ahRVHEINAqduD7NiAygGkSCAhU3ZaI6hrOZf9Scy1C8O7mFkLo7KqnW7ajO1/92td+/NP3Gq3G6fmJ7y8NXb1x41aRco3IIssUTR6u5v3RWAi4s7VdJPFg0EMKmc4WrdaapZnRfCEr0mg2b62tLUZDWZYEgmEUtVqtVmfNMDQM4fnhizTPvHo15+z0rJtlWRSGumoggWq1WlEUlNLpLIAYMVbouo44fPedt588+lSRpKLgiqzBr/3Gn6jX68uFTzPq2p6EcJwmkoRUBSOENMMMg3gxWzWqtfPuGcBgY3NbkiQmeL9/5ZiGbeoYQo6Q0NX3P/io6ZYHvb7TLCNMEj+u12ppFG6sr+VRlud5WuS04IamrYKws74ZJbFumbIsn5+cYojWNzuXVxc/90d/pnd5YRhGEESj0UQAYFhmGIbL+ULTtCiJy15JlXAYrO7fv/9X/up/8sFH96bTabNe3dpcD5YrwzBUS7saDrrnvSAIXNNwTePq9GytUf/zv/kf/rk/92fb7U6lWh8tJ6qs0DhmRa5ZputoBzsbv/HHf/VXf/U3zvujv/Jf/V/fu3fPxkQGgGaxrsgMMgwgApAQZbpcmK6NiRLFacFFkmcbu9tZQS2iB8ugUa4TQlqba/cffS4DuLu7u7G1nebF4Ytz23ZlLFUrJSxJlUrl/v2HeZptrbXX1zunF12skDiOkySxTSuOY9t1TrunSADGChlLtm1DCM8vut3BwHVKNM0A5aKgjBUAgCzLCMZvv/32w8dPB7MJVEgcx1kU27adMVYURclwhRA5pFRwymkW+WXDZFmW5QljLE9C2TDype/YLs8LhBATglIKABCAWZa1WCw0XccYJzTHCMgccsYoApIiZ1kGIGSCSwiDnOqKWnCGEEqSTCUyEiDLMs2x0qzglJmqhiCDjCMhoihCRJJVhQIQxbGqKILxPMkF45pCCk45F5QBBICuEllWkjwriqLeaPX7fYyhEGJvd+f69esfvv/ecumvr7V/+Zd+5X/6n/9+uWK22+3nz57rpp5TvvTTatnSNG25XGqaNp6sdB1WHK8oiqLIIJaygtm2nQZJpVY96p5pmsbiFEOUcUaIRAhhjMVxzgFAEgAAcAogADJBL+XPBWcIAUkmWUwBALJK8jxnTBCCEUIAcpozXVeyIgcAFrkghBQgFxwgCCSEQAYwxpKGsyKDGCFJ0nVjvlw0m01/GcRxrEgkLXIMIOSCMS6rEgWCAiHJCqGCU8GBYBBwKACABALIBWOMEJKmGZAJAAggCXABhAASBoqqKqZhGJZjO47FKJ3Pp421xvll13Gc7Y3N0WV/Nplvbu3kaabKCCnkuHvh++HB+obtuYrnyqpiy9rjLx+Yul6rNibLOZIkwCkUADNx2eu3Nteny9nO9jZB+Or0QtO0ebi0PHcymxuGEcdhxXN1pEmSFOWxrBsn3W7Fq9TKpePjYyThm7dvcsrCYPX2G29e9K6EYIvV0l+ubMP0LLfT6ZRLVd0xJEvt94fv/+CD5cJXbE1WJcRSxDngYLkKEsZK1dpoNDJUTTBarZT8+aJIs+FwePv2bUppuV77/NEDwXnd9rrdruxaGS0apUoaBpamTWZjjqCia/VSbTQZq6aBIJQyXitXZNP0wxghybStwXR685XbgNKHD77c2upwzm3NiYO4Ua10u13dNHTbCtOsoFkR+nEcDxerje0tzgpTN/z5rOS4cZzmeT6cDBVDlzW9Wq2Hc59zXm64lmFCilzXHU373bMzU7PCMMx4rpvmfOLrqubq5irwLc8LknRzZ3u5XPrTebNam0wmW1sba+3GF198Bn/5r/6lxWKBmdRqrSVJSmQ5TqLpeGSZarPZ9MOIYDlcRowxSZMlGXMqbt6+BbH05YMvD3Z30sg/evoUq3IGhKIb2SrWdZNJwPO8o6fP8yRt1mtEkizTGU8nlu1ACfd6g3qlqsqKbdu2bSMJx1laKpWyIn/25HGrXip77mQyURTN1MwvvvjizuuvPXt+SPMiz3NFU5MoVmQpWMzX22vPnj2vtdqdTodIYDAYtBtNwzCuepdJkScZ7XQ60/GwSJLjx09evX2LifyHP/mx7Xgc44LlcRjZqqopcpInrqmZCjZl+d23v/5Hf+U/iO3SP/in//yjP/xDyFgQLioVV8JAlWXBoWaa49k8pQVRNEq5JHCcJhsHO7brGZL++Yf3TM0s18rlZv3Z86c6kXa2tvauX0cSOb8aOLabxUkUBu1Oazqdj4fjarnG84zS4pd//dfuff5Z/6qHEKpUKovVcr5aAgjDLDJNM1os1ttruq5/+fBRbzK5tn8QzBY0zTAHlmWZji1B1G61lsvlxWVva283A8wwjEl/uLOzkxZpt9t99fYrURT94Cfv/epv/NpgPHz/Jz/OfV9XtVqlNF0uYpoLIViUJkufM0aIxCAoigIhxIoCQwAhVHXNX8xl27BUXURpGieyY66CpWFoXDAAIS0KVSBOmUTklx66NE4UiDkQsaAAYigA4BwBSClViKyqapJGBWcZ4wIIwIWEJV1WREGzLNN1NUwiXdd0WQYAAADzPKdMhEmiaWpRFKapx3FEcw4FMHUpSagQAEIAAOACEAlQCoiGIIR5zjgHqiwxwSnllqUVSaqqcpZlsqamCeVcAADSnGmWmqYpLICiSABCRdfSIJFVhQouACgYlWWZFRwBSDl7yYArqhpFvkQIxEqR5xABCAVgCGPMAJMQYKzgjEMEIERASAAAiBjAgAkuqFCgyjlnsOBAaLpaFIVhmVmWESAVRSEhnGXZ/z8voQAAIAlLqpJQCgDQkFIURZ7lQJVNz82yrIgikOcAQqzqRJHTOLbKZSFEs9IoVSsM4Vq7XeRcN6zz7mkUBZxRALiuq/3BZa3kOZZNEypjFQgEMFqtFqtg2VhfLwrGksQwNKxpjmPpRCrilOd8OB7ZpbJpW6vFUpKkYBmmRa4ZmhAsZ5TlRcUpF3lebzdG82mc5YPxaHNznaW5q9sqkTVTe3F2bpfcKIwt3Wq1WlES3r596/z8NI5jQ1ezOLl54+D3fu+7sm7U61Ud4/FodPPmTUiki2F/fWPLNkuT0fSTzz4vl+xa2Xz65CGjIC+YYhlJXqRpWnGdNIgkIIiq+Ys5YBxjLMkESBjIEufcQIRSOg6WsqoQiKMgkBCWZEl3NMYYARLGJKWFoRmEwtCPBrPJxtamaWhRFP2RX/il+Xzev+rtbm8CzL/88stXbtzmBRcczqbToihM2+hsbDx68pDnWRRFnChb29vdi7Nhv9es1+rVWr8/oozplt4fDirVuiTJlmxaptEbX/lhsN3Zeeettz/86MeL2TQJ4vXO5lm/qxuWhAjNqATBbDEXhDTW1zTT4AVNw9gxbUpBlmXbO53RoA9/6a/+leHlwFH0arU6Xc2QhE1FI6oiEAniaK3Zmk3GuqGOx2PFsBeLxdffeWcymfRn43K1/sqtG7/9r3/LtcwwCBRFIZJycXFlWrZVck9PT1rNaqveWM5XkiSdn1/88q/88VWaXPUGWZYQiCQultPZN77x7tlFd2Nv58XJcRRFJdfDQLiWmaY5BtDzykEQUMoVTTu56o7HY5amN65f7w97SRQvhsNKuVypVV6cnECMvIrHac4KaioWxjhME6dS8oPQ1o2jLx5sddbG89nTF890W81ZQRnHCNm6zrMiz3NVVWleuJYzG0821jpvvP3WtesHH3z82YOnj4NkRQWzTZVgGSM1SFLDsdM8S2jK0rwkmzKW7rz5OjFtwyh/+NHH1VpFIPHsxbHjuI5i3rn9SqVaCqNIUomiqpPRhOWFALzVag3Gg4Mb1+Moff/9913XNXQdUr61sdntXc6WizD0u5cXN++8MlvMe2dnN27cuOz3FEVZ+T7IKUwL0zAkVeuPhpzzRq1e96rdbtculRgQlNK93V3ERMmzGcguL7tFRj2vNBxNXnvr7f5kFAfh8eFxrVKJklR37YQD2dDCmb+YTKsVNysSgJGAAEFs68ZnH34gy3IymQDbJgRJHGQLX1e1RFCEhQQplqAQAnIBBZAgEghKkrRarXRF5YxBCBnnAEuAcSq4wJIikTyMMQKShKjgVAKcCyQAFCjPc4yIoqlREEgSoXlGJFCrlpcLP0kKhADGCAAkSVJRZJIkAQAA4wAAxpgkyRwCjCFjDCDIGEMAQwizPDcsM0tShBCRpCSJbdfOsozmhaqqQoiXI0WSTApKMUKCAc45gFCSJJblsixLuhpnKWMMY0yzFGNc5AwAQGTCgGAFxUjilBFCaF5gjJngGCIoBCGYAlHQAjJg6EqUFJIksSInRCqg4ACpEknjBEpCUeQ0yVVVFRLSZCUOQgxgkVEBhKqoFAiOQE4pJhJjQiFqmqaG48RJJiCSDA2pcp6mSGDLMBtrLUopAjCnBcAAclH3PFUhheCqbkSpcL0ygPyjjz7Qdc3Q9dT3ayWP5QXnnEFgOnYQBNP5rN5em0wmr999/cnhs1KtBhiTRSFLGMlk0O9XDCeKIqPsKYqmCyUv6KU/85Oo02wKWsiyauoGT1N/vtjbuzZZzGfBwiq5z14c723vlEx7tZyznFmGTjkLVr6MlY2NjYvLs3qrXiuXzi66jue8jCzv9foxpdVqNV3OlvMFY8ApVwSW1jodx7UM3frJjz4YD4ZvvHrj0dOHaZ4FUby5vUUIWSynuqq5qj0aTXII19utsmGfHh+lkEuyZKv6YjHzkwgrsm3bQRA4qhrH6dwPvEpZsERXNZEDTTOznBYCSFiZLRfrmxtB4NuadPPatWgZHB4dd3a3SuVqtVHXdH0yHvv+cn1986p7MZvNlvMFpzRJklqr6ZZLx8dHEKM333j79ORFnkZepZwV/Ozi0rXsKPRLrjOfTnc297MsG82njmOtt9pxHB8fH+u62qhVL7qXjlf3PG8yuPJ9v9leM8tejoQky1/c+2h3awtwMJst6rV2r9eHAKuqKhW5aDQatqRcnJ9hnexu7kkUQIkUAK+i8Go4iKPg2vW3JELOzy8829F1/erqKsyiP/pHvt3v99fW1ofDoeNWCIL93nBnZzcvCohxrVp1TOvyqmvqVhClm3s7R6cnyyBY63SyVMmTJA/Dzc01XSOCZU8efLHW2WC2bVnW2fGJa9mmptu2TQUAGE1GkzRNGRS7W9uzyXixXC4XvqqQUrVmWEZ/OGqttafL5UWvL0EgY4kqXJHVjOaj0UgiKgcoBWywWqZFQQUoOBNCSARDCKfLhUpkWSE554CoEYNmrR7m+Xs//sGX9z7Z2NtvNuvLE58yHsapQhgQ1LRLBeWyrEKMKCSD3viPfvvbq4WvC3x8MkSSKmvqdDZrNJquU3rr1TefPHo0Hk9LFUckfHt3SwBquka/P8xoIeva/YcPv/a1r7/11luz2SwKw7LjciBUQ6ezaZYV25tbiAtYsLV6++vvfPWjT+9N5xMFS0DBBMtRFJkI5UlcrVYDf6lgKeM5QLzZqBMsjYY9keZp5PbHl4ah1cq1e59+wiCM0qSzufVSnLK/d21rd+fDzz7zNI0yQbj09ptv+8sZUQmlNMnSerORRLGu65fdC3zt2tXVlYIxKJgqGxhjImjgLwVAYRCAIse6jjEOoqjebgfBStL1VRgpRIYAUMoAZZALDoFle3EcIyJZhs4FhTRXZMI5LwrKObcdZzVfUCiQIvOCqapiWtpkPgMcEAURIodhSjBgjL3s7CuKkuYFITIhcpxkCCEqEIeAUyZJkhAAI0wUmVIqIIAYJWmqGkbOBSSyIslJmkLAhRAQIy4EY0yRZQYZFKgQIE5SQ5ap4IG/UlRV0zSapYqiUFYQghBCBS2YALKiUUoFFBAKTKSX4k/OCkUiRcG4hBRVBTlNkkxAVDCqYCUvCoYRwDBOE0mWMIZxnBOJZAWFAAEAIEKUCo4wUeQwyWVDy1gBEBaKrmJJwYpquIbnXGuvLVZLoquqplNKK463d3Ct2+uPx+PYD2qVymQ5T6NIte2H9x/cfe3V8243ifNXbt6wLItn2eXgyrOdTNam44lnu4pGFtFK5SzNk83NTd12bNsOFqud9U2oK6vFbHwxdlyLaCqRFca54zg551CwNE0v+72NV24AgjBjlm6kSXF5dWHpmmzqL45PiKpYplkrV2RFH41GqqIAjAASYZwIwXRdNzWz17sSQhw/PzqXJFmTM0bzPG94ZV7QwPcrlcpVf8QYg0hWCrFeaxAsD/qjGzfKnm19692v/f7vf2dtrTNbLVvr+pNHjw8ODmzbliWiqIZmxvPReDqdGlhGkkxZMhmPct0KggBIWMYkzQpClNUqcN2SW2uMp2OVyJPJolFpR3FOCImDkKKEAZHluWmaTCQPnj5EBQCQL2fz2Xy5CELTNFVVFhAslstaqzkYj27evjXp9Wazhe2Wjk/OFE0NVv5sMGrXWn4wP3z23CpVNzc3WV6kUZyGCUvpYjmzHc/SDV1Rp7PZ1eUl4yAv2GWvt723+/T5uQB4fXMjDFaTuS9bhqTr1/Z2Krb+kx//cDZfWpY9WczfffdbmxtbnAP487/5V4PFrG6ZJc8FsjSZTNeqrTCMGQRUgVTBs+lip9ZKwihN48l0vlj529ubrXrtsnvmOaVypfbo8AjLRMFSQWlSpKqqhmFIECSEzGYTwzDccsUw7VUY2IaFIcrihDO6Wi22tzYqrjOfTra2tlZL/+T8rFSqSJIU+JFpmpSzq/EYy6TdbgfL1Ww0sW3bq5bPz08rXokXdO6vwjhK4tzzvP5kKMtyr3tRLntFkTmOgwCWCcnjwHSdy9U0z/NoMosWC10mtMiIImW0gBgSQrIklmVVxkqWFkRVFIVISEgQKVg1bG8wn/uRz4pIUxVbszFSBASSrPphmBWss77peV7gLy3LWcU5BLheLiVpxAHPsqRVawWrlWEY7fX23J9qpu44Fuf85OS8Vm8PJlPP8VSAizQzDCMr8lUUFkUBIcyy7PLyMo0jkRUVx5Mk6eD6fp7RwXT8onvmx9Hm1jpPch4mNa8sKbJum7Kjf/HwAQCgVq7kSSwhtLm+0e12LdtNkqTRaAGMiKohmQyHw9l0vNFaM3S93mj4vj+dTDzbW61WgR+2Wi0hRLPZPjl5YTqW5Tm9wZAxZtn6ar7Y3twZDAafPfpifX1tZ23ryYOHVxeXnPOcFn4UqqqcFTldzgGCQJYBYwAICKAoctd2ABcAIImoVPA4jR3bzKMICiYYF5BDIq8C33YcmuWioBhjmheEEEWV0jR+mbXGuZBlWXBIKX1pcwUAYIwlSU6SRJZVDliWZ7KmYcEZYwJBlueObSVJ8nLPxZIkEFaJXBRFVjABmAQBgAIAgBBiBRWMAwAwkVJKMcY8LUzTSFiRZbmhKC8bMhBCwCGWpCCKAAIQSoJRgDDgHCAEgIAYv3wFJiCAAkgSKBgQQlHlLE41zYQQUiAAwYpCCpqlcVZyXIwIg0XOc4whSFmaUI4k2TSBRAzbsRy3tdbujwaSJGmSIij7xje+EcTRhx990Gw2O+vrJc+7ODmLkngVhYqisCTjArR3Ng+fvxBCWJazv7/fv7ocD672NjYQE2EYTxZzAYGiGldXV6VKrVIvP3n00LH0kmkuFzOvVEmTpFVuSDKeB6vxfIYluVQqzUbjWrVMALq4uCKaatu2rEhRHBNNt22b5wXGmGhGpVa598W97e3dLExmk+nOxsZoNFAMa7aYD6ajRqPGOfAsFwA0n01onhua4vvhrZu3Hz97Wq1X/SRqN5pKxmmWZ4gxBJK0mM2X1WarWq54ptGsVd/78KMbNw92dzaPj49Go0kQhq7rzhdTSUBCyNnlleO5pqmP+r2N9nowX85mM8XU/SzZO7h2/uwIAqDr+twPBrNJo9HgUV6rlEfzseO5JdeL4zQIMkIIYKmmk6TI0zStONVl4BuOphp6nlHIuKlom5vbkMiPnjz+xjfeXVtrPXn09KX4+vjwuaNp5+cXnZ2dcr02nQ2zILCJPri4wopul73L0WW5VvUXS03RHdPxozAr0oxmuqJimeR5PhiMSnYZYaiosFxvJBRBgA0ZGDIZXY2Lotje379269rHH75HBcty1h+PWltrAqAsyiDE8M/8jf9icHmhywQK8M5Xv/Lpp592Gmvz+bxaLecY2K3adDq/enZqqlqSRFRwLKm6ro+GV5pCJAHb6xtxwYfTabNW13X94rLr+8v9/f3FYlEUjHNKiwwitHv9ep7R5WLRabVt0+p2u0KwyXgoQUQg+At//s//m3/zb1Z+KCnyWnMNIBTG0XAyrtQbx2en+/v7i+lMI0qSJLPVAgDgWqaiqi/OTx23NJ8uS6VSksVhGBqarskK0SQhGKRoOhyst5oxKyY0hkSaXw2GZ2cSzU1NBQAJLBjkVHBT1wAHSKClH2qGzjnVNcUz7eV8JasaIGpa5LE/E4y5TsUwrILxfn8IAX797XesigcgnA+HiqLM/NSx7Nz3dUNZ3944enGImKhVq6++9tqTw2fLcFmqlkuufXh4uLtz7fj0vFJteo437Q9khGVZTmkhGWqtXp+MJsN+Lwpj17K6R8euY5XK1SBc6arRbLUWcXjau9BtKw2DdBZoRNre3jY968XFecqKfu/SVDTPshzH8Tzv+PTM9+Od7b1er1+uVm69dnfhL/35QoIACFaplKIouuxemKrG8sKrlAeDUau5vlgsPdsRCBKCjs5OFnFy48YNVUL1SjUKk4dPH4csvXv3lUV/koYp5EDXdVmWN7a3Prr38XQxtywjCAJJQqv5ghBCs5TnRbBcMcY4BxgTATFWpYLlIgqJrBZxrGgaxCinRankLucLAgHPM01RiYwFFFEUIYgZEGmcEUXmQjBKIUJCCFVR0jgBAECMIIdEUXIARJYBBAHngHOkIEnAomCapmCMGRB5wRSiUs6yNNNtEyFQZEmR5ZZlRVEkYwlBGCcpxwBgpCOFMZbRgsiyKKipG4wXL3/fkrywLMsyHUopQDAMQ4Ggqqr+YmlZFucAIZRliaTIAEHBIcY4zyJNVhiAhmFhjMPQxwrOMyoElCXCeIExhBhEUSQjebmK2htbllfpbG5ljKm6STmDGDHGTFUfDQZ//Jd++Q//8A/9YEEpvXbtmm2Yp8cnfhh0tjYZYyIrKpXKaDGPsrRcrmKMNc3odc+zaFWkSR5lWZZtbu9MZnM/ilVDp5zZts1pgQSgSXJxfuZ5niyhtUbrxYsXjBfVRlPzvPF4CmhB8+LlePBkNgcYrK+vDQYD2/Gmk0mzUlMUjQOBCZlHfrlSGfaHzWotj+M0iJIkjbLUqrhepbxcLiWIWvU1zoqfvPfDu3fvLuYrzyvpirq+tTkYjyil4XiWp5lha34cAaJhRUVEOtjb9ycTlci2WyIyRiAbToZHL86IIhdZfnXZXau3TdPujSa248yCCUHwzsGNJw8fEVWBRJIUWdf1LM6SMFJlJWPccKzVaqVjZTQcCIwkmZS9UhjH7bVNAPhyMrBMpT8ZCAF31ndczzvsno4Xs3arMx2PK6aT5/mrr72h6krZKykqWcxXeZZlcXJj/9o//X/9483NTaLpr7/5xtHJ0XQ4ymcLU7eQolsl1/fnURKqqj6d+4qqAYwkAm3btFQzjuOHz57UarVwEZimOfOnpmO79Va1VD58+EiXpO21Td/34ywDQHzt3bd/8v5PgUSo4Otb64PR8Be//XP379+Hv/qX/jetduf4/CpJskapxItcMRXOCpHzSqVSbTY//PgjwGESRqZpt1qtbvdybX398YvDctmzdQUB6Ja8Z4dHDOCdza3Dh4+r5ZJmm92Li3qjbehas1HLssSPQlXVkzxLozQMQwTgZDLxPO8rb7/z/vvvG5pm23bG6GI239lYT5Kk3mkTRaY5GwwGV/1eq7N2cXFh6QZP883tbbdeevj0iUbkwWCwmq8swzR1g1KKCfLDAHBIJLTd2ciSZDydR0lsltykyDNOnzz4QqOxDADgKkNAMdU0TxSFFFkOOdQ0jUOuYEmhIMuyTOIFFBArWZIYWF6tArtcdcq1/mi4sbFx6/qN4+MjgGClWg18v1SurhJW9pzzpw//2C/9wh/86MeO4zia4bjW2kbn/oOH5XqrSItoFbCiqLfql5fd1+7cPT8/Hw8nkiRZjg0wqjbql5eX8+nCc+39nX0IxU/e/ynlRaVaF4IRRBbTGc2LerNxfH6Wi0JA5PvLVqPZbjTHsykXdD6ZtuvNqle6urrKAecCJlF648ater1+dPyiVK1wzrMwpjRHUCRJZLvO7du3Pv3kk7Ozs3e//vX5wo/jXMLEdd1mo9G7vNw92O+OhnmeD87OtzobSZYhjMf+XNO0wo+iINQ0HQGxu7ElEAzzNEpigqUsy/r9/sbu9pPnh+32mmfap6enGaeGaVeccr3VPr+8iIvk4uLCc1wC0YsHjyzbChYzwCgQBSiy3YPdRrl8eXGuakae56swcDyXC+hHISYSpbmu62kah0Fgarqu66Ef5GlhOW6asVUQWpYJIVQw4qwIw1CSkCRJRZqpql5whoicZAUvGJalLI0A4IAyTAjCGCHEkkwzDShhPwqRQAqREUKcc01VVVU2VIVoJC1yVVWb1TYG0LPMOEs5Iozz8ahH0wxzKYoixdCjxGc0i4vMcEtpmoIicyzDT1LOOUgL13WDNBQIK4qeJAmABRaQAAlCNA0DVTfTIJNlpVSuvvrmW4KQF2ent2/fDlbBw4cPr+9fn09nmqbtHex4nvPjH/7IspzeoE8UtdFpS5K0WW/5y2Wcp4pCiiI7OT5++7U3LMs6v7wKwvBl0vdyvqp45VLZi+MwLfLlcum4pSylOWW6ri9nY9c2RqMBkaTbB7devHhRXqsv/ZWhmYLDi4uLn/32z/zgvR9vbm46qjWbzfqzYaPRMLE6HYzWO504ScIiVyxjuVrs7OwkfpT4oSpJDIiXGyiGCHFmqhoraJxGgGBI5NV8kfuJpRvru9tH3bNKrXpycqJK2LZtIMkcQaLCWzduSkI+fPSss9aaTMbTcX9tY/3p0QuE8XjUUwghkiYh2dC9MIqAzHRTL3vudDqdTqeqqhq6nsVJtVo+PT3FMimXq7Ztm5oO8nw6X/h5IRFFV7V+r7e9sclZMeid25Zx/fbNOI4fPXq6tb0NZSmMol5vUK82siwfD4d/4c/96YuLi8t+T5Jk0zQtVZcZyLOMaLKqa4NRfzybCiC5tmNgiBAKGV1bW9uo1R8+fNjtT5CmpkWe8wIUxbe++o3Dh0er1aq5UW93Wvc//dS0rYRTRKTNzc0b1w4Gp5eX3Ys0z3Ja7O4frDVbvKBPD5+FedqoN7/85F4SBO985Q1FIfBX/qO/mFFaCOTY3nw4hEhALFrtRrwIS67nWc7DR4/KjdpgNFksVtVq1ff9N99666zXG4wHlizLBKd5FsdxyasRLI+v+rVarQBU07RKuTYc9TfW2qVS6cHDLyVVcSvV+XJBsCwom45n5XL58PlRtVZ55dbtyWzaHw0lCfXPL372m9967a03nz0/PHz6fGtraz6fAwRny4WiKGXTabfb958/StNUITLN8tF0stZcOz98YVmGUbYBRokfp2HEOV1bW3v+/EVOueM4um2a1dKDL7/w+xeyEDKWZVleRIFXKQPGGWNAsDTLTMccXPQaThVAnuKcQoaRTLCc+jGApLO3X200R9PJxsbGeDxezWe3bt2aTqdACCIpa5vbo/FwvVpJ0rCAYDwe8zSzTUtAYFj23sHN5Xz18PP79VqtYEUY+hDwMAwNw0AIWJbTbLWuBv21tbXLq/6zp4/X2+tpkc6XC922iCJDAVSJ0DQZXPSbzSYx9d5kpOqaruuVUpkXVCBIED589oxgyZDkUrXSm45VQ4+W4c72NpYkQojrummaDofD+XyeJpGuq7qum6bJgZhOp0jClukEcZIk2SuvvJLnOc8KJOFSrWoYxg//4A/X1zoYY9000iIPgoAgfH56Vq9XFUXpdS8URdm5fs3zvC8/vw8h9P1ge38vyvLnL44c3X7l1bvLwG+vb1Qcr2A0SGLTsQ3DkCBqVmuX593TF0dxFEwmoywJCRJFGqVhkKcJhDBNU0VRAIJRnL6kJ3OWE0IkCDjnEAAIoSYrcZhgTPwgUXX9/zdlxoEATJbl1WphGIaEsCIp23u7AmEhRJJkURQFcUCzlCBSFBlFiNKcCAwAABgxIF4a/1VZy/NU0wzfX2LEDUvPBdM0DXMJFMxUFVlTddvLacYKaqn65fklhDAXBaV5HC+Z4FxWBACtciUM/UWw0lXNQITmRcQLzdBpIRCGaR6pRNagmiYZMJQkK2icW7o1m8xlXfuLf/Wv7N+4/tu//W88z0vT3NAtwGCSJ5jAXv+SFVTTDCzJo8n4nXfffeXW7R9873uCca/kPHny6O7du5ahdU9OdM2gmPSGo+liDgCwVL1armRJwjlVFOXq6sq2XUlWKs01zvlyPoWAhXFEs7Tp1VbLpVmyBRKDwWhzczNNcknGE39Rr9cXwxkhZLyYua7bKteGVz1b11Z+qNjWcD791V/7lcvLSyykYa+/Go9czxZEuepfNZtNS1NVAEPfz7kwPIdoahxFdBV5tuNUSrKpP3nxolQq5XFaZBmSydtfeevL+/cIIY5eXm93nj19tJxPK25FVhVJVY6Onwf+EiFUKlehkDqtdV3Xf/LhT7Z2txzTCpN4vpxFQSgjPB6Pt7a2prOZrKmmac4nU13TaJIxAahEIEKdtTXXtk+eH7UbzZPTF0RRcloIBB3H0TTF0g0IwOmL4zSnuuvJMlElWK/Xx7N5Z309S1MsuAnJZ5995tZqhqWXy95oOhmN55vrG73u+db2xqPD57VaLZhOOQMUSdV2y3Sd+WLG8iyPc0s2Sq533jvfv7Y9urxUdI2YZpiGURCst9owBwpR5/6cQ1CtNTRFv+peAADmUbizuXVx+OJrb7/9/vvvaZoC3/rj/0FeZI6pFTRd+Kvt7W2aFc16YziahCt/u9pcLBbzJGxvbSCEdMM4PTuer5aqaWBMwulqfW3jcjSoVkomltfXOi+6FwWlqqLEYegZ8ubWxul51zTNyWjoVSu1rc54PsuS7MEXD37um98Ow/Dw5EQz1JfslG5qrKAi5wSierVRrlRyWmiaFgXh8fFxqVSazmdhEvf7/Rt714o8xwQphrmM4057LeiPNVN7ePy81qhbEnn29HG11XBd15DVxWKxDKON3W2iKqZu/et/8a8sCaN8yWjGFZ1zpAMFQxTTEMsEyUpaUAxEEvtEBpah+avYtj0msB8nB6/cbrfbLKW7e9d+9PEH9WajVqpMx+PlZFZyXBkjTdM0zXr//ffKDS8Kl6pESk5JlbWnzw7dallTjT/9J//Mhx9+nBc0iKPL3qXr2nkW/cw3v9E7vwRCDIZDpJAMwiDyeUpPT49N2zJNs1quyAirspIlcZoXiqYGWVIUBU9ou96QJSXOMz+NvXJpOV0KxoqsqNZqvcmo1W5qCDSqleliuVytgiCwvZJpWX4YAMEUJLmO0+l0TroXeZ4Hc79aLQd5tFytbtw8GI+mACBTN/zVzPPKUVrU6/U4DBhjvcEQYzyZz6rV8nQ+6qyvKYpCCJGQBCEMg/j4+LRarZed0mAwitJkf+9gMBi12+29vT0u8nandfjiiDFWNq2r84tvf+tnjk9Pur2ra9f3Oefv/eiHrml88sEHjmMF4crWlTSOX6KQRFZfRjthmVBK4zjWNM12nfF4KknSS4ZEVWUMhCRJVPAgCDhggnMIoaIojmmtr68zxlzbsW07TfM4juM4nk6noBBM0GWSxGnkqBZ/KRfnzNR03/ehQBywLCugBDVNYbwAAum6jhAyVI1TQQiRZKKqsuWWZ7NZ2XX6o2EYR3EYIcF1XZ/5S1kllBYQAAhQGiSGJCuaHIuiYDkBkm3bYZZgJCHKFUUZreaGYQDGIeWQiyAI0iL/ylff3dnZmyyWTrmy8qOnj59XKpXbd248ffY49APPK6uaJssyAGB9fV0I+OWXX1ar1TRNkzx79dVXx4MhlPDFZa/SrB+fnKyvr0NGV6sV50BT9U6r88W9zwAt0iKrd9oFo165VqlUnj1+IkFQZCmnTCWypMiyodTKJRlIURInQvhJlKZpEka3btwihDx/fogQ2t/bUWTt2clxvdnQVKXqVYqCvTh8nge+YRsMS8+On3ME3379NZiko/4A6ZZEFD9YpHECuNhcXzd1YzoaC4TSNJ3N/Uq9QjTp4GD34uw8CaOtrR3LsobDvoRxPPXTNO1srx+fnjx//gIR5HhWqeSCnHteSVIVQ9WG55dBFBoVr2AUczAcDpdRMF/6d+7cIYiMRqNyuVxyPT8MZFN/9uzZWrv57le/Eq38Xn+sW6XLXt+Pk1K1NF+M2/VqOB5hxjPfj7JcOM4bb78zOD1DSLJKJSDBRsWbDgc1r3x8dHLWvajUaoqp7uzsIITOjk6TwLcsp98frnz/Wz/zjclirmiGoqnj4bjSqB5fnhEZb9Sa08Ho+MXz1+++CoSYLxa96bgQ9LW7r+Z57llOs964urrMsuyqN54v/Tfeev3i6hICaXNjw5RJ/+JS100JE/jVX/nTuqGkSSDJaBkFpmnmUaoqeqVeuzo932msqap6NrjqTUaaaVRr5ZfL2Hac87MzxKVOZ2MwGSdx3LCcoih8SokiYyai1QKCAmMsqbqlG8vppNKsxZDlnJXcEoFoNpgrmsoRJjLunp1eO9j1w1VzrX387JhlLEtyXdexKm9sbPiLZf+qF0eRaVm6Y2EALUWDUNTaTaLpg8lUUP7ws88lSVrb3hxOhpjSgmaaaQRhaKpqu93GsjydLZIkadTXZgv/ve//Ac5XqoqEJBuaDROGEZINQgUP00LW1CgKGM9EkZVcmxcgp2w+WX79F3/x1it3Ly8vK07ZKrk/+OlPq7VatVLJknQ1XfA8n436nuvaTjnN8/HkKi8SXVE1RR9c9a9du6aZxmXvigmxu7t/ed5TNH00G6dpvL210a7XFCTdv39f1nQ/jhhBumWOewMZSzs7OxcXFxLG7Vrj5PjI87zJbPrq66+fX11wBgyi6LISBfH+wTWBpQePHhma5jpeEATN9TWsKc+ePuFRCBk1TRvLBCLUG46aa20/CosowUB4rnv9xq1nL44kScYMuK7jh4t6q94fDgAAWUKXs6lpGNcODmZBtL+/f9k9n89m/eGkXKsu/RWAjPP81/7Er4ZB9Nm9e3fv3j16dvi1r379937vu45lJ0nWbreTJCWSgjHe6Gwqmixr0hdffFZt1Lc3NvvnF1ud9SSKHz15nAhWqVW3Njbvf3bv3ocfcEoNSy9oJiORRJFKFEmSMlqkaS5JkqKqsixHUaSZBqU0TXPHcebzpSLhZqMiIajIam840B3rZaUCIXBdt+x6tVpNIbIhk06nAxifzBb9/mCxWGGMqeCLOFQUJVr5aRTbnpvnebVUTdN4OJ7qpuZHMYRCN1Tf9yvl2mq+kAjynFK4CnVd1zSFqArChBAiyxLAiHK2XPppnCVJQmnKAUvzwnXdZBmoivLyOIGaBBDI04zlhW5ZeUYB47IsM0kkSaIrqgRhEsWqLHMAut2u63pvvvn21t7BaDydL/w4jrd2dxSFDAYjRVFeEvE0yzXTwJIcxNFLt6XrltqdjmEYS3/15NnT/RvXdd24vLyUMDo4ODh6/oIxEQfxrYMbn378vqwQBrmA6Obtu1/72tf+3b/5HVYUwWr1/PnzklduddaKIrdNXSfaYrHQPVfVtdliliVpsApUVV2sFrKq1Ku1rMhb6+tra2s8K9Y7nR/9+D2FyNWSc//+fa9aeXF+ajvO5ta6BtHl+aVXa2BMRv2epqlO2Xvx/KhVqSkSWSwWiq4XlAsEiQJr1XLvoqcQGXDKgLA811C1y8NTVVEykDHOISQCwlU4c113Pp2lefbGG2/Uq7WffvcHk8kEu8b+wcGLZ0d5npcrFSp4FCW1Ws1UDMbYYDB47Y3Xj7svwjhYX+tcdi9uHVznApZqLQYkzTAfP3t84+b+fDxkQagBePb8SDH11o2Dp0cvdKyqqrp1bR8iMB8PkijM89y17fOzq3K5XKnXVqsFFiCLE4KlNM0BQBChq9Hg3W98YzwfZ1mWBclitVJc/StfeXsxGS9G48HlFUYSpdzxygyBKAkrldLBwcF7P/7J17/2tclolEZJTsHcDxJe3Lp1y1INDNG1g+1/+S//5WZnu9loSSKPI5YKBKMoMw2PFnlrfePs9Dg/j9earSBLBotpxnijvSbJZLnyq2UtjiOQccSEaau9XpdDoMmEI5yJ/GJwVWTJVnNN11Wr3IzyfLmIslViu1UBZX8xWd/snD0/0RS9Uq1f9XtJUXBGb21voTjHWV4sfRXh+tbabOVnRb6+vj6fz3OaSzLmEcvSWMWIKApQlIte7/6jh2+99VbFqSRZhjUVQhCMphYgSJZTKHUaa4eHh5bpnp1eJGFSdstl21kMJjHNtq9fA4wfHz4BccQYa1YroR+IjAOAHENfrRYahhxKTJJWy7jkeqPxoL1z7Vd/5U8ePj38+le+9eL05P6Dp45uGUSVJdmqWNPJQrfs263Go4f3E8o0zdjc2D85Ox2tVgRlumnFebGazmRTn6+WZ5dnIqcl17EMU5bl4xeno8F4s7OeChjnmeG5k9miVDHr9TZB2LI8RZkpuvbFs0NDlsfz1dbGNoSYFtx03PNut9NqVzuty0FfU9ROs5HGMUTcLtuj5fjisJ9lWadUgQUTHMpYVmxT8cNFEF4/2D96/DRPUtW0nh49L9WrSZIoSB5MxtFqefv2nYKC4XCIKW9VGvV25+j4rNFqmrrZqK3FUdFc1548P2rWG5pCdAnff/+LeBWZmvbFR1+0Wq17P/24rJmmbtI02d3b7J6dX5x0b9y4UbAw9fnoZDyZTDZ3ts/Pz+vVxvHFVRRFmlupek6axmWvAiijebHWqg9nA01TWMpUoim6FoahoqpYUf0wwBgUNElFDgpECMlYEiXYsHSaJCzLvLIXJxnGiBf08vKSyLJuGBBhKElJFNfXSyxJsihs1qosS3PXnU6nfuQTXaW8YCn1k4AoJMgiWhSKv3QcyyhZGc+JoUDBOOe2YWQ0YZBaip1FoSlLNI1DmpJMlmVZKAqNgG4aTLBmsz4azrAkxQlkjFGGw2Vcsez5bEI0PeNMLUgQR4pBsKYCClSJxEWCFOL7K9O28zwP4rjseFEcMM63ru37s9X3vve9n43zr3/jW93xCCvK48dPS6WSRBSi6aPhZH2tFK0mSGJuzSibpmu5s8n09ddePTw8vDw7TVlhuc7x8UkUhK1WK+dgPJmnab63t5dl2VH3+Bu/+PNpGl9d9Lrds9ls8vTxA85yglG9WonT1K5V/DgiDMYpm/mLVqvx6P6DWrlkmYZAmMiIyGhrc321DF4GlPd7V41KZb3euX/vy91ru0VRFEmq6gbB0nZnI4liFcrdq0u7VBIsVyVctWzLsmZBAAXAGDLAEpETSeO8UCS83tyoVqv9y2GSpe1q9fT8bLJctdY6Tq0MBfCnwf71g+fPn9uOVyodRFk+XPUVTZbdElSNURAe3H1VKFJ/OrWr1dfvvjruDSDED4+eeZUqjAvdsCWBjg6f265x68b1xXT1lTe+yopsMhmdLZc7u/vJPHjnzoGiKNCycqQ6ph1EhawSmEND0qAkhWks8vTh/S85RLKi2K61CCPDNgxdbZZKy8E4pbRWqzRq9eFkQBnLGe9IraurqwwUnufIALEiv7Zz3ZH0o/P+3Tu3NSxnOZ/MVws/sxy7Xi1ZuvLw3sONta3B1ThcRYpKXr1z8OWjx4PxatKbnU4O69VKEPitdufWnVtXVxfwT/zmX15GQZhmaZ5ZplMURc5yVZZQXhiaFkSJbtlRFNVbTSzJeZYAxi+7F5DRVqtllWzTc45PzhgTO+vbn332KYfg2vX91XROCJktV9VaIwyTWrkukqhS9YbTSZLFvOC6Zg5Gw2q9VqpWijShfhD4CwqZ63lYVlRNTwrqh1Gj0fjgg5/euXPn8OkzW7P2travzrsYQ0SkSqNuGEYUJZqsffDRx3u3r89ns3S2MjXdcO1Gu+Ev5vPVMkqzZ8+e3dq/QbAkctbc6Pggp4K3GmtPHtxfjXu0iC5PjhTXJUAGAkGMMAKCFkIIJGsY4ygKGBO//Kt/qlJtWJYVpUn38oJzvt3ZoJQ+ePrwq1/9ahynl93zs5Ojzlrr+OiF67ob6zvnVxeKoff7fVshL3viHAKOoaHpdbtsGSYFcDqfIYTyPL976/aTZ88imnnViiRJO5tbs+lUUKar2tVVzyy5h4eHkHIEQbNRS5JE0Q0/Ci3HnM/nNMk0RcVMMF5YpuN4bsbzi2E/KrJGo2ERPQ9jxCCS8DJctTfXp9N55AcqkVr1xnA6URRlb2/vpVvq5MWpP5/dunVrPF8ghESSpHGmmGYQR7/+678+GAwuzi5k3TBL7hePHiCE7ty4dXFylIcRy3JVVUuNWhBFiDMEgKqqBS8Mz3n48GHJtJvNZrOzPp5Nz8/PnZKnqLrv+57rhsvg7be/8tEn997+6luOZXafH/3g9//9ajlzPTOnSZqmkAJN0QEAWZEjIuW0kAjRdCUMQ8PQfN9XFEUhcpYVEEKZ4JJmSBimWZYUdDpbIiIjCSMEiIQc06xUKq5pJOH/t6UzabYsuw7y3mefvr19f+/r8zXZN1VZqipJVa6SZFkWsiQwBoPtCAhHADMGBAPPPGJGEwQwgCEBjmCAsEXJskpyKZWVlf3L5r18/Xu378699/TNPmdvBsmPWBErVqzv+7zf+uZHn33214cHR6VafXl17aLXdnyPYaHl2JACXhQs1ykYeRjiIAhYTQ4TTADlORS7viwIKYK+7xuSEnk+z7Acx4UklRQ58DyJ4zXFwDjiNElSVY6VPC+IcBiGIY7TxXyuCxxCaBH5kGFBCgRBoAy17HlGUhkIF5YlqDLP82EcxQlRVTVwbEmS4gQLgsBBFNmuM3c0Q89US9du3jL03Be/ub+0tsogbjIxW40WxUmSpplq6dGjR5VCkWeY2zdv3bv3hWkvCGS0bLa1sjwdjSmFt2/c2dzcfLm7e/+r+67vrW+uX965StMU++F4OCIwASRdqtfu3bvHAKTmclPPqVUbEmU2L23bof+zn/2MxtH2xjqitDfoxiRtrSzPJlOWZRMA/SAMQj+rGd/56JN7X97/4JNvHp4cx14YeB4FaRrjQq5o2c5wNtV0NaOpljnNKIZlOVDgZotFrVKKoohyyLLmSRQaiqpK6pvD4/rScqPROHz1kmVZwvNxgjOy6jiWpmmO78R+UCyURUEDLLfw/cZKs1ovjfv97uFJq9XykzihJHSDQb9PMc7n80ahEAYB9aPJcFipVU/OjjVN0jLG8sompXT3yUPD0BiGzeVy0+l4e3s7xPFstljMPRyniiotLy/1O11RU3hVpgw9PzyuV6oTcx7hOExxksQsIfVyReKlO7du/+IXv+AEHkdhJmd0hn1RFEvFGivwEU0zGd0eTe3ZPJ8tcBzHssze3t5HH3107/6XzeW1QqU6Gk18z1ltNfrdNsdx08lM1w3HsVkOuGFUqq+Ui5XzvdelUiHXrCw8CwG6sGZsiODUXlCaaprBcKzrOISmQRCs1uuyJBFoR3EMIdQlZdDvZ7JZNatFUcBQIKnK08fPivX6+Xi0vr5+fHasK+K7d+6kCLbH4+Vybdjrj89Os5Was5g4gxHxbNO1OEm2PR8Tqmiq77th26nVKlDmsccAxANR6Xc6tVqNEIaHHIzpOzfuCKpIIcjniymFFCICQOi7qrzkBVG71xdFcfPylrtY4MCtNKvmbGb2e3sHe5osra2tsYLYbDZLtWohl3fmM0zjub2wHWfYHqw1m+9c2Xz87AmrGm4YTQb9SrU8ni5ASihhGARExPoYX7nz7sbGJgOF3dd7G1ubAJCttbVBv2+OzCDy5+bs9e6znKp75lQRJW/h3tje9gPv5GwvoRSTqJQzMpICAEkpccMgSKnrRYgJOv3J1sbqOzdvnXXacRhX88WX/rMbV7fP++1FGPjFnGmORI43JOn29euPnj+vliu+45eK+Xw+M51OXS9McGrN7Wa9YQ6nSZKIGldvro1Gk4PjI03TavmKbGi8KEwHE8v1BEFAaRJEvoCYaqH0ejBe3trAaarKmmvbx89fsDznA1qsVQzDODo7X11dffnyJc+hrG6IEvfh1z8lcRj7bqVciAnJGArLQE1TeZmFAquKRlZSF4uF5Tiu71Wr1UatjiB88frF4PyiubKaBJgRlE5vHCX4w298bDm27fpuGCeAGoXs2dlZ4Nh/8/PP7ty8tr668l8vTre3NoLIDaKYY3kCUpwmADCCrISeK0kiThN74bAMIjGAKWQIQxIICCSQWHbAIzYKAg7BCKecpPphpDEcwViXJUkQ4zS2Qri2tPof/vN/G5hWqdrou1H3+Yt6taJoKElCVZGn5gJTQBg0s50CbygCb89tQZcWvguIUCrWBoOeoCnZYpFiHPoEIhgR4scBFdiEoQKLZnNLUmSYJI5jy1wsC5IiGmwh3+21y9UcSaDl2IKiB0FASazxymg0kSQxDENeYI2MJsvyZGzKmiryjGs7qiBinDAIQEhJEvMSn+FzAMJhr3t8cPjBB1//9KNvvDo9IjhQFUkS+NF8kSuWZgsLifzh6cm7t26fnp3vH53c+fBriqFPx+alzW09mzvaP2g1Gg++uPds92nKgD/5p//k1atXWTVjTSfz6dwQBNONLMsSBSFXLIWenzUyoih1j0+uXtrSRW48nNVLRUgoIQSmCUOAKqg8I+RyuclsglgxTZJiNm8o8tMnD+vV0rOvvprbFgCgUa3kslmWZWXJeP78+c76pcli9mh395133uEZBsTBYmEXCoVipfjlVw8y2Xw+W5hNBvZiLrKcqskze2YfB+bCXV5q+r63s7k1t6zOcOjFiSQKy42NUqnQ6bRde7515Xp/POq2T7MZYzEzWUBTltH0LEqRKhliRQQkHXf7IEnTNMnktFq9ABn86NGj9AxECTMxp1ldxIDQIOl1R4ViZuG4Z+fnv/d7P3r9et80zUwhmyK6tLYmiPzCmY2mk1K1tnPtxpPHjzUI3cjHGCdeyAtKfXn5y2fPeF33PK9ULiOeu37na+Px2HF9DQDIwKya6Rye4TRZWWv1B73Z3DIKufHcrLfqOPErRZ1g9/nFm9PQ0fVMmqQcxxUK+SAOBUF0Y7s7HN2++94Xn//85t3bxaVa2mkPh30gCPDDP/gHssjPZ5MkSSuVpeHYLJTyURjEvpPVjXa7mySkWMynaeosLE3TZE1mBZ7ipN/vC5wYEnL1zu37Xz1AOK7kMptra5jS89kiieL52fnOpY3ezEQI5Tg5DkMpm7UCL05SlmU5xAgi5/s+wwBV1sbTybVbt13XtswpTcnr1we1aqPZbJ63zzBINENnKZfEuFQoXpyd1GuVRqNxfNYhzFta2oIYN5v1heWdXZynmLAso0kioYmRKxXLpaPDUxxHtVLxvHP+jd/5HZwkT774EhIiK3x/NCzXW73xsFotFvJ5Fkn9fh+AtNfpkCT94Osfrm1sOI5nKDkAwONnT7OGJiHk2g4E3MHRYWOposqSORoqsjwyZ57noSSWNaU7HUGenZlW1shJCIEEa4Y2Ms1CrREl9OrlG4nnmf2ewKH2cFTI5q9sbj5//pRyKFfOB4GHOLZSqSiSurf7Wtd1wrKirMzHM8jQjY3V09NTWdEXjq3pKsHxYmbrqnrRPW60mt1Ov1FrqqLk+z6mxA8DhBBCKJcruK4LQSKKIk0Y13VLhdxsNosTyjOMBOBgPEhFFvD8UqPpuv766trnn38uy2KaJCutJkLo8ubWvXv3Nrcv45RAngcc2n31cjKZ8Ahtrq1OR0NN1oIIQ8S89/7XVFXlIPrJX/4fPZ+tNxoMgx4+fPjpx98xDINlwKv9vYte//s/+N3xqM8Q6s3cS2urJ50jVZY+/+lnF8cn+XL2bW08juO3OheOEyI/kGXRcSxWENOUpmmqKJLnOYIgRBHmOE5WJZ7n4zBiGRCF3nhicmI2xkRmWQho1pBYAUUk+fo3v/Hsq92JaTdWNmXd0DPa4eHBYj6Zm2NRgK1GYzyaEgKUjOraDo+RxPIMC6MkiiEBAAgMGyVY1OW5NVNkiVJKYxpGmJd4z3dUSaZJygOJ5/kUxQCkuqAIvKjohu04kiYCBs7n3tx2ohDLihg4Ns9ySZSoqjq3TcRCBCDLoDhOeEEAiAUApHHEcVxI4yRJFE4AhNp+gDhW4UXf98MwrrYa1++8U6xUB51hkhDE8FNrrhj66urqyclJ4HqyJPUHg7WdnZu3b7kL6/79+6uXNm7fvMUG6e6zp5Vm/fXRwfd+/HsPHjwoafndx49S3y+VShPbllWp2+1curQ+n88dx9tc3Xzy+CELQK1WszyvUCy2e31FFLKyJIriZOEGSTwedmrN2tS0dN2ACeEgzGW1ZrP5q3u/2bp6+cmTRzeuXalVqqPRZDSac5ywtrLaHvTa40FrpdWqVdpn5yzgLMvSdHm2MHv98ZWdy8POuSIK+Xyx0++FlG5uX4l8zDJw0D2DEFKWj5J0PB42q7WyVkwJfue9W6/2XucK1XKt/vLNqzgMykYm9r29N2+WV9dyRokwcO5OR6MBCVIWwCSNNE0Zj4etpeaoNyQUAlFGHOt6M5EXmJSvVquSxE+mI0lWAWLymSzGeDQbi4JcqzQgJY1G5fDksNpad13//OT00sba/sF+sVhElDk9PYUsKpWLhVxeFqXBYPD+hx8cHJ5MJiPXskLfxRhTSqvVqihwhiI8ePCAl3WOFxVZ3Ny+1Gn3OB7FceS73qA9qFbrk4l5aXMTiezu7sud7csvXr2pr61++9uf9i9Oi/n8y/3XtVotl8tsbW3BP/6X/+rNwd5iMVtZWWGRHBNQrpb6o8FsPFAlGQCmUCi5kXdycrRUa4V+oGgyAADjVJIkwzBkTe+0+/VmzVqYDCKR75vmLFeomeNJVldWVpZ++etfbW/vOGM7SQiryNli7uL0hGOgyLNxHKtG3gt8HCW2NZdlEUKazeem0ykH2Mlk4nnBJ598EpOUADoYj1aXlnVVOzw40BW12+3W6/WY0AjHAAFEycpSq93ub2xsXLS7CEFZEo6ODrLZ4vrGhhdhSMmk0/3x3/vR/ee7v/r1F+9ev24vrMOzI1FWhoNxa6mxtrJ8cXye14vvvffel09+vb6+PhnP4zguV0vXr9/c3zscjSaWZVnW/PatG67r+l6oaKptLwrFbK/XefHihSyrk4m51GwAhk492w0jWZAZCgBJECGGqgAGIk0NonhrfTt2XBJ4N69dHZn2Ym6lUUQgmbm2Zqj2fJEzMpwoIITCCKu6FuAEQia2vUql0h90Hj1+vLS8ZhiGHwcSzymC2ut149STVYXnRECgzLEQwvNuL8Qxz7OSJGmCNh6Pb71zq9fvj6cTTVFJ5FcqtV5vUiwWfcvKFzNIFt3AHwxGYRjKokQplUSREHL9yuX79+/7trO8vLy0suYE4Wg6XVpfDSI8X5gqx8eBL8uyoWpv9g4oBPlqudPppDhutVqZTGZqzZVsVhSkaztX0zDmEPvs2TPLsZdWl44O9gq5fEY2QJp6kUXT5Cd/8b9a9RphEpZlgih0A59BnGFkSUI5BiIGUEpjnCaU8DzvBT6FJEkSQGDWyMRxyCOWZVnXdT3sJTgVOQMxPMNBSomqcCnBjUpDkJXR1FxaueT7iedFnX7nd373uwGOh6Pe44f3OYbhU0jjhHJpJqPbpgMpEWWB0IQBKI4SlmVjmlJEEY/SNEmSFCYMhBBxDMZRihOW4ZIQ8ALLywyEgMZpMZdPEgIQo+Z0SZGHw7Gq6PbCwRiHke95HiIMAACykJAEQYogAyFK0jRKUp7n0zhSVTWkse/7AkSCIECOj6KIh4gVeMtzHNvjEP/pp98WJTmhYDyb44QUMllVVTuDvuXYvCQ2Gy1d0Qu5vLWYIY6tNOrFbO7izUng+eXl5sAcmY61sXnp0ZePAsfNG7plWXIms729aZmmbdsn52dbm9uhE1ycnNZKxUaj3hmPZtYioSCrqY1czjCM52+OQhyHzpzjuIRAQRDSKK5XKhzLHJ+ehAnOl0uEEEHkzk5OV1bWstkyQhwOwgjjmKFXrl+xZpPZZBp5sa7ru7tPty/vnJ61RYHjIFxZao1GE57nLzrn+WI5TcH6yqo1Gz14+HBtc5tAAABYW15hUuj5DiuxDELjyazeXBFF8fT0FKTRJ9/85ssnj549273xzl3KQILd58+f71y7WS5VT/beJGncH48YBmg879geRqyRz3W6JzzPZ5WiKIrrG2uOYzm+J/GCpqij0YgXOTcIl5bXdFXVBHY0HQ1nbqle5xCrSmIUBuPhKPbDYrXSn03u3n3313/zN5ooN1pLC9v+4MNvLqzZ/S++QBAUCqXZbCYJ3Hg40CRh7dIGZXgKYb/bPjk/e+dr7z96/LjeKJ+fnlUK5cD2bduVNLW23GBZ9s2r/aXWMhLEQqlIcKSr8vWda08fPV4sZrPZDP7WD3//4uLs6rXLiOG8IBmOJu+8/+5oMjo/O9JkhWV523OVjNrr9ZrF2o3rV88vLsIwdIOwUChEfpCmVBZUI5sp1wsIwedPHyOGs0x3Mplcu3HlrH2s6Jpt2xIjQYC0Qrbf77MQ3Lp+7eGD+9lMPlcqn5ydqrLm2AsBMZVaOYxjCCGOosALIcNU6o3RaNRYas1tq9/vt6r1nUubZ2dnAICT8zNN10VNM81Jv33xjQ8/GAwmkiRVqvUw9APfVRSJZUXHcSq1JgOBN58fnRwuotAPg8j27ty5s7S+fHp+YQ6nuqotrClJaK3QMDKa5VschwCGtXIlV84jThgOJ5Ikmab58uXuzdu3rly9enZ2EUXRyfmJORuLspAkCQQcA2G1Ws1ks/vHh1PTlAQlxQligcAhAQFJkjKFYpKQ5frS3vPnd25eqdVqDx49ty0vbxgAMS6Mx+OhP5nrsiLIkpoxhrOZZuiappUK5cVo6lhWiENJlgGDkiQJcdi9aBuKoSiKlhETknbaPVXWOEAVRUkhMxqPCUh5jlMYQRRFLwqdyMtXKuZkmpdkkeMdH1uW1ahXojiYOzZEDKCMF/jVcmU2m3mue/fOO4oi/fLzz1v1hmroB2+O/vBP/sgNwtcHB4pmLKxZ7LiFbKZUKkEC9/b2EEK1VtO2bUNXe71epVJ5uff6/Y8/zufzSRD7jgsopZQilvXCANKEhczJ68PxaLC0Wn/98sViPNtYXQmwZ86nkEWqpgVBpChKFMQEJ4BiSikFTEKBIAi2a2mGGkVY4ERVVUmCKUhJTII48iJXkuTQo5SCJEmMrCZLHACgXKx5fqTl8ubCUpVspVzLF7LnFxd/58c/PD09BgTvvXz14qvHgWPLhkhBqgqK7zoApcViMfSDJCGipIQ48SNfEDmGY2RBdOzAdRwAAM+zcRRQAjlWUTXZdueixNE4FTiRpUjS5bdxFUM1BE7EUWKak5QmGKdxiNMYswLrOBaHIIIMIQCxLMPxEMI0jgghMSSiyCMAkyTheBFjnOKEFwXIsRhj37R9N5AM46NPP41SsjBnjVJlOp3yqjx33Hy1GsdxSc/dun7j4uLCyGVH08nF6RkJomq1Wl1ucYpkBV6r1To7OgWE9jtt1dBt19ve3pQkaTQYdrtdABjP9hRJDF2n0axThCgAcUpOjw5pGGxubrKSNp2ZHAPOL06bzTUcx2mMKUlwGGKS8rLECXypUj4+PLhx40a73ZUV/ZNPvvXoy69EWbp++06ukN198jDw/eePnvm+L0mCpCqO6y8tLdkz0zTNlZWVmWm+efO6XK7OF87Vq1crxcx4Ol1eXXux9xpHcaVSASkNggBJUM0YOEwFXtZ1vd/vr64uL0zzy198zvKclM3JqpK480zWCAiD08SZzAzD8HEQ+YE9mciKRgUpTGJZYRVJllk9l806nkNpKklSLpdTBHF3d1c1VJyQjc2dyWQcOYt6qz6Y2a3V9fb5hcAi37PzRk4W5f5g8L2/+0NZkf7Nn/+5iLiPP/n0vN2+6HS+/e1vnxwcjAb9IIiy2SzLAJHnht12oVzO5Eu1Ruvp08d+GByenuE0UVUxm8mYw6kh628FMkJWtm17vd5KMfnlr/72nffe/cN/9PffvHrdLNXnU/P05GQ8HsOPfvD7KY4vb15qdztyppBQEASerEjnZ0e5XG7uuJ7nAUBWV1dxGLmuW6lUXNe1PV+WZeyHjVozVyiMJxPHtbMZlaZJbzC0Fn6pXIUgdX2HU+XA82QG/fAH3//qyePTs4uEgq2NrRQnoshXqqV8PnvvV3/r2B7GKcMwHIcoSDfWLh0cHPhJLAiCKIozc5HJZVVV7Z9erK+uIoHvDQd7+/uFahkyrDkdr9TrOAqaS2svX71K4vTyle12+xxCOl+429vb3/nWd/vdzuHea1GV6pdWu4P+o18/zBl6vpibjMaabLCIgzwysvpkNCWEyLLMEFrPl3JGxg6cAMeKoiuy9tlf/6y10qrUqpPp9Pj4WJAlI2tQkL71x1KC6vXGcDiYTCaGrMmiYs5nCCFO5lMS8zyDg1BihNXWSqVUlmXx7PyYMPDVm+NKsdIsVizXCXV2NBqZx+31xpLlLGIEqCIQQO9euRF64eHh4QcffCAryqNHjxqN1uOnT1KQpjE2FMN1XU4Es8V8ZXktTZJpvw8JTVm+XKmEke/ajgRQLp+pLbfaw74dphyDmm9/51Xt+Ow4DvzReFCrtigDKQDdXrvRaFxcXCw1W1ndOD05qZbK5+fniOcqlco//KM/fvHydUwIAfCi0wk8B8dRRlJzuTwmqRcGnmU1KlWJY73Q29jeevn6VSFfsiyrUqmFccRxXJTg8WSyubmJk+ji+DiYuyLPYoB/9cufrzVXAtdJQMyJvBv4hFABcALLhTgBgESxL4oix/GUwDiOAQAAEsRwFDIpTTIZ3fM8BNkwDCGksqpMJ64oSCyHZFkURAQoixjxe9//YafbIwj2ugMjm9k/PMhoerVYHfd7tXp5Z2fns7/6v7KmvD58EfqOxogSx/ICE5HYCjxBUkicaqqRzRmdzgXLI0XRXMdHDJPiJInCTFZfzG3CCL7vZvKa79mSIGmiSmNCISE8TCjJ54tJjFVOliTJi/2E0NFwghBHSQJJykBKaYpDjEnK8AIhhEdsnMaAgQlJeRYBAOI4EUURQpimaZhgkedVTo6C2LQXUZJubm9/51vfGncGxWLR8t3JwpIMI5PJvXqyWy6XG2tLmKT9ft8cT8b94ZUrV9cvbcxsa/v6VZ7jiB+dn5292T8MgiDB0cr62syxrly54k6tbqdPEQyiSBQgIFQTlXavK2u6pEr2YtJudy+t7Yii6IXOsxe7jdoKwzACYjKGhiAwTTPAuFgqkTQ1DD2nG47jdHrDQrE8n011w1he34iS8OL4GBIqseLx4eGdr93xgoAQeHFxkctlGo0GQtyb16+moz5OU1FSBVnK6apuqJbtbmxtdi/aluWUSxUv9BfurFKvabLhWHY5n7OsOWA5a7FgYlqsVI4u+pY9z+vyfG7OAp9SakgCQ1JZEXzf31zf7PSGfXNeqlYyujibmjvLl1VZsV271+9sb2+Loogxbrfbo+kojLAkqoVinhUYP/Q2t65FYTzqDwSBH4z6tVpN5uRKpRLEEcex+wd7rmc/evhkc3un3mxd2d6ZDIc7l7cOD46LxWIuo++9fhk4tmma+Urd9UMGkMFocvPuu+Z81u2cW7O5yIqVYlmW5acvd3/re9+iKaZuWCtU/u2//3ff/d739Kxyfn7OMlzOyCQxjqIIfuv3/zFNsWVOeZ4vNVbNxVzTtDDyEEPiOCQUZgv5ozcH5VIpSZJKpeR4rhcGqqIjgESGlSRpaM51XS/mcoNeJ2/ogEWmF3pBqCC0ubVxcHziRz5HKYSUYZGRye3vH2QyGUPLqKo8NEez+ZSDnCxJc3MRuN7KanMxm0OCcoWCmtXG01HWyByfntXqTZGXIMYH+/uZQhZwiEFciBMGoeGobwiiLIiiKMuaOhpO/MAVBK5QyBHC5HK5WqmuqcrDB/eNnLF+ZVPVM4N2f39/nxdYQKhjBwyDMhljYk7DFNfrdWthL6amwYpZTUsZ4ofB1ubl3nDAICSpSrvXZQXeNM219VWe5/v9rjmfiaKoq8b5+bksi616w3fDBBMn9JtLrdl0Mpubqi5kMxmVU4/3j5rN5traijU3c8XcvQeP1tcuCSlgRD7V+ePTE7LwdEGaLRb11SarK+PxWEhBYLucKGTzucCPoigyDANjDBEIgkiX9fb5uWqIvCwGfgQB4AGoV6rHnZ6sKDyHHGsus/x0Oo1o8v0f/+j53lGr3jw/eCOxPAFUz2Y8Zw4AiCPCcCwhSYRDz/N0VWMo0+10IKE8z6dpKqlKpVrlJbFYqkQp7fT6iOPWV5cH/Z5tLkRR9OMoTdNmuTLqdW9evSIIAieLe/v7eweHS8vLhWIuCrEgyYquSZL05s2eYRiDTvtSc7lcKvyn//IfCUmKuSIkaYBdAlLCspACHiB7ZquqLggc5MB8bsqCjBAbBjHkYBRFmqaFOEEIqaochqFlOQyAlCQUAIZXUgIojrNZQzfU2dxfXrtcLNUyqnbRvXDjsN6sWZaV1zPd4wscBRzH1uqV+7/56re/910v8e7du+eNF7Hr1uplDGIqs67rikjgAJvN6GHo4zRGHB+FWJakyHFEjjcXJseLFIkRiUWeoTRN4lRieZBQQeDs2Mtks4hjSUIlgAxN93HEikIQpwhx7sIOHJvQGEIqsEKU4AT8/woKwzCSIk5nJssyIi+4rv8WVsIY8zwfBWFWz2KMo5REUeT5jqHp17avAsDoRqa1urZ3fJzNF8+OT4uVUmWpVm82AE4DLxQk8Sc/+cm/+Gf/fDCZvDk9XlpackZTczg+brc//vjjzskJw7FQ4MbjqcErCKHxfPHhR+9HnmMtZgtzISmqqMgnp8d/9mf/+k//9E/LxfpoOMkUs3Ecy4qBIHP39p1Bv91rd7SMYeTyJycnaYor5TLPMJIkuX7k+f7KcgtxbGt17fT0WJOk6XBUMHKT0RgD/Gz3ebFckyTl7t13njx7GuO0kM8O2qdPnz5lJY0XhFxGUyWZ47hKpZLEOJPJKYra7nedwJIkqVap542MOR0/ffZMzWQlSZpN59evXGN4qdfrCCwdTUdeGGXyudl4RNNElXjPdkqlytz2PJwqhh4GznKjycZoMZsv7EW5UmzWG2+Jp06vK4q8ObdESeUlsVordPudVnPdsuzL29s7l7d6vQ4h4M2rA9t2q41qpVKWNcFxnPnM+urR4xu3bpvjiSKKjWbty/sPe73eR9/4erVYGA8H5nzGipKs6jtbl/77//iL7/zgBydnZ5FjMYAmMdF1PYzwwelhqVn+4O67YkSePnw8moxlQ+t0LkRR+O3f/u6rV6/eXq0ZCmF/PKkut2orSzHGLMtarjO3rTAMJUlyLXsyGNZrNZLgxPcvjk4Czx+Px6IoGppuSJLIC9lCniDYPj0r6Vns+EkYDWdTJwoQQt50rkG4Uq5Ish6kUFSzvh9+44P3lirFUlalAGOGNWrNW9/8yE0SlkeFonF8fFAsF8ejKcdwL589HXa7c3Osq8pkYo7GkyCKFUNNCS4V8kmMM6qhKGo2k5c1Xc9mIAXOdEaSpNVq8TwfBIHEC+2z8zdv3vzmy/scYkXEdfaPBienjmuFaWRa9tR1ApqkLHAsS+J4gRdlXW+uLhOWsUlsxoEfR7VG46LfYRCaz2ae52aK+Vy52FpblmXZ7I9IGJcKRUVREAEZSc2I/MX5sZSRM82SVMmGTCrLckbRcrlCrlQeeU5lZ4Mp5RcA8GoGAmFn+wqDwNLm6mgytPrTkpK9fefdfL1q1EqlWl0GSKKM49lrOxsUEhyFzUaDRUhVFHM6LpVKPM/PTTOXywEA3gpDHMeBlGxcWt/a2hoOBtZ8SikJE5xCoOvG44ePDFUDgOI0xiAZT2ej/mg2MkftgabKlKYsyyQJppSkJBmNB41Gg2XZ8XgqSdLq6mq+kL1586ZhGLmMcXnnKgvZxA++dufdUrnquL6hqGkUE0Icz33x6uX5+fnBy5fuwlrZ3MiUc7IqNRs1RZB8c5E4fi1biF1/a3V9rd7yzUXqhWUjl7oejSIZsiovMgnhGeT7vqJKAs+mSexZi3I+pwhiGkUMAgBCRVU5UYAQchyXRLHECwCAhKSGrOZ0g5cFwFFFVzKG5vt+Np/LV+qUEc5O24f7h61WwzD02HeTOIgS7857N/3EcQO7UszvvXg56U/v3vmgubp1+8OPY8gmkH3r/QcAhGE4Hgx5xMZxjHEEIXRtR2A5kuJKsUQpZVgUxbHlOHEcsyzL8yKFMIwTDnJRGA6HwzTFCY6c+SwJgySKAYsIA/OZbCGbK2RzqiSTJAUpieP47XqepuloNKKUsiwbxtHbfh7DIlEUIU5VUYrTxMMYiTwnsI1ymcH4/r0vXu0+9217f/dls1xdTE1F18r1qus7x/v7zx8+Pjs6vPv+3R/86Ac//9lfU4wJgr1B33fd995776PvfOv2++8FcdSq1YqqMR9NeFEUJCWfyzz76tG03RtedPRMhpfEyWSyMKf/+y/+5x//wR8kcby2sS5y/Ob65t3bd5Iw6nd7zsJ6K7jHJFV07e1gHp2cUAijBCuanC/mC4WsORl6zgIxpJA3AtfGsYeTYGm53m2fua7163u/anfPIQu7o34CaKXVWN26xOtKtlz2k5gVeJoSz3EW5uzk8Gip2cxnC5QQCXHPn+4Wi/VSraWohqhpakbtjrqd472MhBb2oNM9VkTOMqdaNre5czWOqMRrjuUM+n2O5S3LAgy86HWdwIccz8ki4rjz9oWiKAyAICEgISwDKvVaoVREFLQqDXMyTQkpt+rnnfZsPPnFTz9bTM1ut8twLGZov9sDCW42ao1a1TJn25c2m/Xqg9/c75xf8Ij/2V/+9LO//Cn2Yxxgz/EbtVrsRZqsXpx3JElJwjhwXIYCzdC3rl5e3lhLkkQSxF63rYrC+++/jzG+fv16tVz+8jf3puPR6vJSt33x/wCW3nVTmedoTwAAAABJRU5ErkJggg==", + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import io\n", + "\n", + "from PIL import Image\n", + "import requests\n", + "\n", + "response = requests.get(\"https://s3-us-west-2.amazonaws.com/air-example-data/AnimalDetection/JPEGImages/2007_000063.jpg\")\n", + "image = Image.open(io.BytesIO(response.content))\n", + "image" + ] + }, + { + "cell_type": "markdown", + "id": "b8ab2cf1", + "metadata": {}, + "source": [ + "Then, print the image's annotation:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "ee5e074a", + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\tVOC2012\n", + "\t2007_000063.jpg\n", + "\t\n", + "\t\tThe VOC2007 Database\n", + "\t\tPASCAL VOC2007\n", + "\t\tflickr\n", + "\t\n", + "\t\n", + "\t\t500\n", + "\t\t375\n", + "\t\t3\n", + "\t\n", + "\t1\n", + "\t\n", + "\t\tdog\n", + "\t\tUnspecified\n", + "\t\t0\n", + "\t\t0\n", + "\t\t\n", + "\t\t\t123\n", + "\t\t\t115\n", + "\t\t\t379\n", + "\t\t\t275\n", + "\t\t\n", + "\t\n", + "" + ] + } + ], + "source": [ + "!curl \"https://s3-us-west-2.amazonaws.com/air-example-data/AnimalDetection/Annotations/2007_000063.xml\"" + ] + }, + { + "cell_type": "markdown", + "id": "686f0885", + "metadata": {}, + "source": [ + "Notice how there's one object labeled \"dog\"\n", + "\n", + "```\n", + "dog\n", + "Unspecified\n", + "0\n", + "0\n", + "\n", + " 123\n", + " 115\n", + " 379\n", + " 275\n", + "\n", + "```\n", + "\n", + "[Ray Datasets](datasets) lets you read and preprocess data in parallel. Datasets doesn't\n", + "have built-in support for VOC-style annotations, so you'll need to define a custom\n", + "datasource.\n", + "\n", + "A Datasource is an object that reads data of a particular type. For example, Datasets\n", + "implements a Datasource that reads CSV files. Your datasource will parse labels and\n", + "bounding boxes from XML files. Later, you'll read the corresponding images.\n", + "\n", + "To implement the datasource, extend the built-in `FileBasedDatasource` class\n", + "and override the `_read_file` method." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "0a4b8820", + "metadata": {}, + "outputs": [], + "source": [ + "from typing import List, Tuple\n", + "\n", + "import xmltodict\n", + "import pandas as pd\n", + "import pyarrow as pa\n", + "\n", + "from ray.data.datasource import FileBasedDatasource\n", + "from ray.data.extensions import TensorArray\n", + "\n", + "\n", + "class VOCAnnotationDatasource(FileBasedDatasource):\n", + " def _read_file(self, f: pa.NativeFile, path: str, **reader_args) -> pd.DataFrame:\n", + " text = f.read().decode(\"utf-8\")\n", + " annotation = xmltodict.parse(text)[\"annotation\"]\n", + "\n", + " objects = annotation[\"object\"]\n", + " # If there's one object, `objects` is a `dict`; otherwise, it's a `list[dict]`.\n", + " if isinstance(objects, dict):\n", + " objects = [objects]\n", + "\n", + " boxes: List[Tuple] = []\n", + " for obj in objects:\n", + " x1 = float(obj[\"bndbox\"][\"xmin\"])\n", + " y1 = float(obj[\"bndbox\"][\"ymin\"])\n", + " x2 = float(obj[\"bndbox\"][\"xmax\"])\n", + " y2 = float(obj[\"bndbox\"][\"ymax\"])\n", + " boxes.append((x1, y1, x2, y2))\n", + "\n", + " labels: List[int] = [CLASS_TO_LABEL[obj[\"name\"]] for obj in objects]\n", + "\n", + " filename = annotation[\"filename\"]\n", + "\n", + " return pd.DataFrame(\n", + " {\n", + " \"boxes\": TensorArray([boxes]),\n", + " \"labels\": TensorArray([labels]),\n", + " \"filename\": [filename],\n", + " }\n", + " )\n", + "\n", + " def _rows_per_file(self):\n", + " return 1" + ] + }, + { + "cell_type": "markdown", + "id": "10d6ed44", + "metadata": {}, + "source": [ + "### Read annotations\n", + "\n", + "To load the annotations into a `Dataset`, call `ray.data.read_datasource` and pass\n", + "the custom datasource to the constructor. Ray will read the annotations in parallel." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "0a4717e2", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "find: ‘.git’: No such file or directory\n", + "2023-03-01 13:05:51,314\tINFO worker.py:1360 -- Connecting to existing Ray cluster at address: 10.0.26.109:6379...\n", + "2023-03-01 13:05:51,327\tINFO worker.py:1548 -- Connected to Ray cluster. View the dashboard at \u001b[1m\u001b[32mhttps://console.anyscale-staging.com/api/v2/sessions/ses_mf1limh36cs2yrh9wkf6h2a75k/services?redirect_to=dashboard \u001b[39m\u001b[22m\n", + "2023-03-01 13:05:52,269\tINFO packaging.py:330 -- Pushing file package 'gcs://_ray_pkg_00aff5a3a84ab6438be1961b97a5beaa.zip' (266.32MiB) to Ray cluster...\n", + "2023-03-01 13:05:58,529\tINFO packaging.py:343 -- Successfully pushed file package 'gcs://_ray_pkg_00aff5a3a84ab6438be1961b97a5beaa.zip'.\n" + ] + } + ], + "source": [ + "import os\n", + "import ray\n", + "\n", + "\n", + "annotations: ray.data.Dataset = ray.data.read_datasource(\n", + " VOCAnnotationDatasource(), paths=\"s3://anonymous@air-example-data/AnimalDetection/Annotations\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "db3d0ee6", + "metadata": {}, + "source": [ + "Look at the first two samples. `VOCAnnotationDatasource` should've correctly parsed\n", + "labels and bounding boxes." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "e0039edf", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'boxes': array([[123., 115., 379., 275.]]),\n", + " 'labels': 2,\n", + " 'filename': '2007_000063.jpg'},\n", + " {'boxes': array([[124., 68., 319., 310.]]),\n", + " 'labels': 1,\n", + " 'filename': '2007_000528.jpg'}]" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "annotations.take(2)" + ] + }, + { + "cell_type": "markdown", + "id": "5ff0097f", + "metadata": {}, + "source": [ + "### Load images into memory" + ] + }, + { + "cell_type": "markdown", + "id": "87846ae1", + "metadata": {}, + "source": [ + "Each row of `annotations` contains the filename of an image.\n", + "\n", + "Write a user-defined function that loads these images. For each annotation, your function will:\n", + "1. Open the image associated with the annotation.\n", + "2. Add the image to a new `\"image\"` column." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "494c71d6", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-03-01 13:06:08,005\tINFO bulk_executor.py:41 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[read->MapBatches(read_images)]\n", + "read->MapBatches(read_images): 100%|██████████| 128/128 [00:24<00:00, 5.25it/s]\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "016ad4fe729a4949bf6f59153b039ec7", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "VBox(children=(HTML(value='

Dataset

'), Tab(children=(HTML(value='
Dict[str, np.ndarray]:\n", + " images: List[np.ndarray] = []\n", + " for filename in batch[\"filename\"]:\n", + " url = os.path.join(\"https://s3-us-west-2.amazonaws.com/air-example-data/AnimalDetection/JPEGImages\", filename)\n", + " response = requests.get(url)\n", + " image = Image.open(io.BytesIO(response.content))\n", + " images.append(np.array(image))\n", + " batch[\"image\"] = np.array(images, dtype=object)\n", + " return batch\n", + "\n", + "\n", + "dataset = annotations.map_batches(read_images)\n", + "dataset" + ] + }, + { + "cell_type": "markdown", + "id": "e7cdc755", + "metadata": {}, + "source": [ + "### Split the dataset into train and test sets" + ] + }, + { + "cell_type": "markdown", + "id": "9cfddd49", + "metadata": {}, + "source": [ + "Once you've created a `Dataset`, split the dataset into train and test sets." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "f92ee5c1", + "metadata": {}, + "outputs": [], + "source": [ + "train_dataset, test_dataset = dataset.train_test_split(0.2)" + ] + }, + { + "cell_type": "markdown", + "id": "6b68209a", + "metadata": {}, + "source": [ + "## Define preprocessing logic" + ] + }, + { + "cell_type": "markdown", + "id": "9dbea4b4", + "metadata": {}, + "source": [ + "A `Preprocessor` is an object that defines preprocessing logic. It's the standard way\n", + "to preprocess data with Ray.\n", + "\n", + "Create two preprocessors: one to transpose and scale images (`ToTensor`), and another to\n", + "randomly augment images every epoch (`RandomHorizontalFlip`). You'll later pass these\n", + "preprocessors to a `Trainer`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "bbba448e", + "metadata": {}, + "outputs": [], + "source": [ + "from torchvision import transforms\n", + "\n", + "from ray.data.preprocessors import TorchVisionPreprocessor\n", + "\n", + "transform = transforms.ToTensor()\n", + "preprocessor = TorchVisionPreprocessor(columns=[\"image\"], transform=transform)\n", + "\n", + "per_epoch_transform = transforms.RandomHorizontalFlip(p=0.5)\n", + "per_epoch_preprocessor = TorchVisionPreprocessor(columns=[\"image\"], transform=per_epoch_transform)" + ] + }, + { + "cell_type": "markdown", + "id": "1c647be8", + "metadata": {}, + "source": [ + "## Fine-tune the object detection model\n", + "\n", + "### Define the training loop\n", + "\n", + "Write a function that trains `fasterrcnn_resnet50_fpn`. Your code will look like\n", + "standard Torch code with a few changes.\n", + "\n", + "Here are a few things to point out:\n", + "1. Distribute the model with `ray.train.torch.prepare_model`. Don't use `DistributedDataParallel`.\n", + "2. Pass your Dataset to the Trainer. The Trainer automatically shards the data across workers.\n", + "3. Iterate over data with `DatasetIterator.iter_batches`. Don't use a Torch `DataLoader`.\n", + "4. Pass preprocessors to the Trainer.\n", + "\n", + "In addition, report metrics and checkpoints with `session.report`. `session.report` tracks these metrics in Ray AIR's internal bookkeeping, allowing you to monitor training and analyze training runs after they've finished." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "44ec65fc", + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "from torchvision import models\n", + "\n", + "from ray.air import Checkpoint\n", + "from ray.air import session\n", + "\n", + "\n", + "def train_one_epoch(*, model, optimizer, batch_size, epoch):\n", + " model.train()\n", + "\n", + " lr_scheduler = None\n", + " if epoch == 0:\n", + " warmup_factor = 1.0 / 1000\n", + " lr_scheduler = torch.optim.lr_scheduler.LinearLR(\n", + " optimizer, start_factor=warmup_factor, total_iters=250\n", + " )\n", + "\n", + " device = ray.train.torch.get_device()\n", + " train_dataset_shard = session.get_dataset_shard(\"train\")\n", + "\n", + " batches = train_dataset_shard.iter_batches(batch_size=batch_size)\n", + " for batch in batches:\n", + " inputs = [torch.as_tensor(image).to(device) for image in batch[\"image\"]]\n", + " targets = [\n", + " {\n", + " \"boxes\": torch.as_tensor(boxes).to(device),\n", + " \"labels\": torch.as_tensor(labels).to(device),\n", + " }\n", + " for boxes, labels in zip(batch[\"boxes\"], batch[\"labels\"])\n", + " ]\n", + " loss_dict = model(inputs, targets)\n", + " losses = sum(loss for loss in loss_dict.values())\n", + "\n", + " optimizer.zero_grad()\n", + " losses.backward()\n", + " optimizer.step()\n", + "\n", + " if lr_scheduler is not None:\n", + " lr_scheduler.step()\n", + "\n", + " session.report(\n", + " {\n", + " \"losses\": losses.item(),\n", + " \"epoch\": epoch,\n", + " \"lr\": optimizer.param_groups[0][\"lr\"],\n", + " **{key: value.item() for key, value in loss_dict.items()},\n", + " }\n", + " )\n", + "\n", + "\n", + "def train_loop_per_worker(config):\n", + " # By default, `fasterrcnn_resnet50_fpn`'s backbone is pre-trained on ImageNet.\n", + " model = models.detection.fasterrcnn_resnet50_fpn(num_classes=3)\n", + " model = ray.train.torch.prepare_model(model)\n", + " parameters = [p for p in model.parameters() if p.requires_grad]\n", + " optimizer = torch.optim.SGD(\n", + " parameters,\n", + " lr=config[\"lr\"],\n", + " momentum=config[\"momentum\"],\n", + " weight_decay=config[\"weight_decay\"],\n", + " )\n", + " lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(\n", + " optimizer, milestones=config[\"lr_steps\"], gamma=config[\"lr_gamma\"]\n", + " )\n", + "\n", + " for epoch in range(0, config[\"epochs\"]):\n", + " train_one_epoch(\n", + " model=model,\n", + " optimizer=optimizer,\n", + " batch_size=config[\"batch_size\"],\n", + " epoch=epoch,\n", + " )\n", + " lr_scheduler.step()\n", + " checkpoint = Checkpoint.from_dict(\n", + " {\n", + " \"model\": model.module.state_dict(),\n", + " \"optimizer\": optimizer.state_dict(),\n", + " \"lr_scheduler\": lr_scheduler.state_dict(),\n", + " \"config\": config,\n", + " \"epoch\": epoch,\n", + " }\n", + " )\n", + " session.report({}, checkpoint=checkpoint)" + ] + }, + { + "cell_type": "markdown", + "id": "0d68c97c", + "metadata": {}, + "source": [ + "### Fine-tune the model" + ] + }, + { + "cell_type": "markdown", + "id": "eef58891", + "metadata": {}, + "source": [ + "Once you've defined the training loop, create a `TorchTrainer` and pass the training\n", + "loop to the constructor. Then, call `TorchTrainer.fit` to train the model." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "06a59e9b", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-03-01 13:06:39,486\tINFO instantiator.py:21 -- Created a temporary directory at /tmp/tmp1stz0z_r\n", + "2023-03-01 13:06:39,488\tINFO instantiator.py:76 -- Writing /tmp/tmp1stz0z_r/_remote_module_non_scriptable.py\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "
\n", + "
\n", + "

Tune Status

\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Current time:2023-03-01 13:08:45
Running for: 00:02:05.37
Memory: 50.5/480.2 GiB
\n", + "
\n", + "
\n", + "
\n", + "

System Info

\n", + " Using FIFO scheduling algorithm.
Resources requested: 0/64 CPUs, 0/8 GPUs, 0.0/324.83 GiB heap, 0.0/143.21 GiB objects (0.0/1.0 accelerator_type:V100)\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "

Trial Status

\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Trial name status loc iter total time (s)
TorchTrainer_f5aa9_00000TERMINATED10.0.26.109:175347 244 108.703
\n", + "
\n", + "
\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "(RayTrainWorker pid=175611) 2023-03-01 13:06:56,331\tINFO config.py:86 -- Setting up process group for: env:// [rank=0, world_size=4]\n", + "(TorchTrainer pid=175347) 2023-03-01 13:07:00,615\tINFO bulk_executor.py:41 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[TorchVisionPreprocessor] -> AllToAllOperator[randomize_block_order]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(autoscaler +1m25s) Tip: use `ray status` to view detailed cluster status. To disable these messages, set RAY_SCHEDULER_EVENTS=0.\n", + "(autoscaler +1m25s) Warning: The following resource request cannot be scheduled right now: {'CPU': 1.0}. This is likely due to all cluster resources being claimed by actors. Consider creating fewer actors or adding more nodes to this Ray cluster.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "(TorchTrainer pid=175347) /home/ray/anaconda3/lib/python3.9/site-packages/ray/train/_internal/dataset_iterator.py:64: UserWarning: session.get_dataset_shard returns a ray.data.DatasetIterator instead of a Dataset/DatasetPipeline as of Ray v2.3. Use iter_torch_batches(), to_tf(), or iter_batches() to iterate over one epoch. See https://docs.ray.io/en/latest/data/api/dataset_iterator.html for full DatasetIterator docs.\n", + "(TorchTrainer pid=175347) warnings.warn(\n", + "Stage 0: 0%| | 0/1 [00:00 TaskPoolMapOperator[TorchVisionPreprocessor]\n", + "(PipelineSplitExecutorCoordinator pid=191352) \n", + "Stage 0: : 2it [00:08, 4.31s/it] 2023-03-01 13:07:33,990\tINFO bulk_executor.py:41 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[TorchVisionPreprocessor]\n", + "(RayTrainWorker pid=175612) 2023-03-01 13:07:34,394\tWARNING plan.py:527 -- Warning: The Ray cluster currently does not have any available CPUs. The Dataset job will hang unless more CPUs are freed up. A common reason is that cluster resources are used by Actors or Tune trials; see the following link for more details: https://docs.ray.io/en/master/data/dataset-internals.html#datasets-and-tune\n", + "(PipelineSplitExecutorCoordinator pid=191352) \n", + "Stage 0: : 3it [00:13, 4.48s/it]2023-03-01 13:07:38,660\tINFO bulk_executor.py:41 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[TorchVisionPreprocessor]\n", + "(RayTrainWorker pid=175612) /tmp/ipykernel_160001/3839218723.py:23: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at ../torch/csrc/utils/tensor_numpy.cpp:199.)\n", + "(RayTrainWorker pid=175614) /tmp/ipykernel_160001/3839218723.py:26: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at ../torch/csrc/utils/tensor_numpy.cpp:199.)\n", + "(RayTrainWorker pid=175611) /tmp/ipykernel_160001/3839218723.py:26: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at ../torch/csrc/utils/tensor_numpy.cpp:199.)\n", + "(RayTrainWorker pid=175613) /tmp/ipykernel_160001/3839218723.py:23: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at ../torch/csrc/utils/tensor_numpy.cpp:199.)\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "

Trial Progress

\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Trial name date done experiment_taghostname iterations_since_restorenode_ip pidshould_checkpoint time_since_restore time_this_iter_s time_total_s timestamp training_iterationtrial_id
TorchTrainer_f5aa9_000002023-03-01_13-08-41True 0ip-10-0-26-109 24410.0.26.109175347True 108.703 4.2088 108.703 1677704918 244f5aa9_00000
\n", + "
\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "(RayTrainWorker pid=175612) 2023-03-01 13:07:41,980\tINFO distributed.py:1027 -- Reducer buckets have been rebuilt in this iteration.\n", + "(PipelineSplitExecutorCoordinator pid=191352) \n", + "Stage 0: : 4it [01:11, 25.77s/it]2023-03-01 13:08:37,068\tINFO bulk_executor.py:41 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[TorchVisionPreprocessor]\n", + "(RayTrainWorker pid=175614) 2023-03-01 13:08:37,464\tWARNING plan.py:527 -- Warning: The Ray cluster currently does not have any available CPUs. The Dataset job will hang unless more CPUs are freed up. A common reason is that cluster resources are used by Actors or Tune trials; see the following link for more details: https://docs.ray.io/en/master/data/dataset-internals.html#datasets-and-tune\n", + "2023-03-01 13:08:45,074\tINFO tune.py:825 -- Total run time: 125.51 seconds (125.36 seconds for the tuning loop).\n" + ] + } + ], + "source": [ + "from ray.air.config import DatasetConfig, ScalingConfig\n", + "from ray.train.torch import TorchTrainer\n", + "\n", + "\n", + "trainer = TorchTrainer(\n", + " train_loop_per_worker=train_loop_per_worker,\n", + " train_loop_config={\n", + " \"batch_size\": 2,\n", + " \"lr\": 0.02,\n", + " \"epochs\": 1, # You'd normally train for 26 epochs.\n", + " \"momentum\": 0.9,\n", + " \"weight_decay\": 1e-4,\n", + " \"lr_steps\": [16, 22],\n", + " \"lr_gamma\": 0.1,\n", + " },\n", + " scaling_config=ScalingConfig(num_workers=4, use_gpu=True),\n", + " datasets={\"train\": train_dataset},\n", + " dataset_config={\n", + " # Don't augment test images. Only apply `per_epoch_preprocessor` to the train\n", + " # set.\n", + " \"train\": DatasetConfig(\n", + " per_epoch_preprocessor=per_epoch_preprocessor\n", + " ),\n", + " },\n", + " preprocessor=preprocessor,\n", + ")\n", + "results = trainer.fit()" + ] + }, + { + "cell_type": "markdown", + "id": "224a1139", + "metadata": {}, + "source": [ + "## Evaluate the model on test data\n", + "\n", + "Now that you've fine-tuned the model, you'll evaluate it on the test data.\n", + "\n", + "### Generate predictions on the test data" + ] + }, + { + "cell_type": "markdown", + "id": "1fc9bac2", + "metadata": {}, + "source": [ + "`Predictors` let you perform scalable [batch prediction](batch-prediction) and\n", + "[online inference](air-serving-guide). To evaluate the model, you'll use\n", + "`BatchPredictor` to perform inference in a distributed fashion.\n", + "\n", + "Create a `BatchPredictor` and pass `TorchDetectionPredictor` to the constructor. Then,\n", + "call `BatchPredictor.predict` to detect objects in the test data." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "cc3cc662", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-03-01 13:08:48,113\tINFO batch_predictor.py:214 -- `num_gpus_per_worker` is set for `BatchPreditor`.Automatically enabling GPU prediction for this predictor. To disable set `use_gpu` to `False` in `BatchPredictor.predict`.\n", + "2023-03-01 13:08:48,945\tINFO bulk_executor.py:41 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[TorchVisionPreprocessor] -> ActorPoolMapOperator[MapBatches(ScoringWrapper)]\n", + "TorchVisionPreprocessor: 100%|██████████| 26/26 [00:17<00:00, 1.49it/s]\n", + "MapBatches(ScoringWrapper), 0 actors [26 locality hits, 0 misses]: 100%|██████████| 26/26 [00:32<00:00, 1.25s/it] \n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "468b32006b5f440dae152b288d84d5d3", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "VBox(children=(HTML(value='

Dataset

'), Tab(children=(HTML(value='
pip install "ray[rllib]" tensorflow - rllib train --algo DQN --env CartPole-v1 + rllib train --algo DQN --env CartPole-v1 --stop '{"training_iteration": 30}'
.. margin:: @@ -43,7 +43,7 @@ RLlib supports any Farama-Foundation Gymnasium environment, as well as a number It also supports a large number of algorithms (see :ref:`rllib-algorithms-doc`) to choose from. -Running the above will return one of the `checkpoints` that get generated during training, +Running the above will return one of the `checkpoints` that get generated during training after 30 training iterations, as well as a command that you can use to evaluate the trained algorithm. You can evaluate the trained algorithm with the following command (assuming the checkpoint path is called ``checkpoint``): diff --git a/doc/source/serve/doc_code/production_fruit_example.py b/doc/source/serve/doc_code/production_fruit_example.py index 4ff5618cdad96..987ac186a8b21 100644 --- a/doc/source/serve/doc_code/production_fruit_example.py +++ b/doc/source/serve/doc_code/production_fruit_example.py @@ -9,8 +9,7 @@ from ray.serve.http_adapters import json_request # These imports are used only for type hints: -from typing import Dict, List -from starlette.requests import Request +from typing import Dict @serve.deployment(num_replicas=2) @@ -88,10 +87,6 @@ def check_price(self, amount: float) -> float: return self.price * amount -async def json_resolver(request: Request) -> List: - return await request.json() - - with InputNode() as query: fruit, amount = query[0], query[1] diff --git a/doc/source/serve/tutorials/batch.md b/doc/source/serve/tutorials/batch.md index 24d3983cb4410..cf367893eff53 100644 --- a/doc/source/serve/tutorials/batch.md +++ b/doc/source/serve/tutorials/batch.md @@ -186,4 +186,31 @@ Finally, let's run the script. $ python tutorial_batch.py ``` -You should get a similar output like before! \ No newline at end of file +You should get a similar output like before! + +## Troubleshooting + +If you see the following error: + +```console +TypeError: Descriptors cannot not be created directly. + If this call came from a _pb2.py file, your generated code is out of date and must be regenerated with protoc >= 3.19.0. + If you cannot immediately regenerate your protos, some other possible workarounds are: + 1. Downgrade the protobuf package to 3.20.x or lower. + 2. Set PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python (but this will use pure-Python parsing and will be much slower). +``` + +You can downgrade the protobuf package to 3.20.x or lower in your Docker image, or tell Ray to do it at runtime by specifying a [runtime environment](runtime-environments): + +Open a new YAML file called `batch_env.yaml` for runtime environment. + +```yaml +pip: + - protobuf==3.20.3 +``` + +Then, run the following command to deploy the model with the runtime environment. + +```console +$ serve run --runtime-env batch_env.yaml tutorial_batch:generator +``` diff --git a/doc/source/serve/tutorials/serve-ml-models.md b/doc/source/serve/tutorials/serve-ml-models.md index 9a9e0c6f62e4f..5e63c4779c2d5 100644 --- a/doc/source/serve/tutorials/serve-ml-models.md +++ b/doc/source/serve/tutorials/serve-ml-models.md @@ -61,10 +61,39 @@ Now that we've defined our Serve deployment, let's prepare it so that it can be ::: Finally, we can deploy our model to Ray Serve through the terminal. + ```console $ serve run tutorial_tensorflow:mnist_model ``` +:::{note} +If you see the following error: + +```console +TypeError: Descriptors cannot not be created directly. + If this call came from a _pb2.py file, your generated code is out of date and must be regenerated with protoc >= 3.19.0. + If you cannot immediately regenerate your protos, some other possible workarounds are: + 1. Downgrade the protobuf package to 3.20.x or lower. + 2. Set PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python (but this will use pure-Python parsing and will be much slower). +``` + +You can downgrade the protobuf package to 3.20.x or lower in your Docker image, or tell Ray to do it at runtime by specifying a [runtime environment](runtime-environments): + +Open a new YAML file called `tf_env.yaml` for runtime environment. + +```yaml +pip: + - protobuf==3.20.3 +``` + +Then, run the following command to deploy the model with the runtime environment. + +```console +$ serve run --runtime-env tf_env.yaml tutorial_tensorflow:mnist_model +``` + +::: + Let's query it! While Serve is running, open a separate terminal window, and run the following in an interactive Python shell or a separate Python script: ```python diff --git a/doc/source/train/config_guide.rst b/doc/source/train/config_guide.rst index 0b6c0381ddcbf..b2a010024808a 100644 --- a/doc/source/train/config_guide.rst +++ b/doc/source/train/config_guide.rst @@ -52,6 +52,8 @@ are :ref:`not tunable `. :start-after: __failure_config_start__ :end-before: __failure_config_end__ +.. _train-config-sync: + Sync configurations in Train (``SyncConfig``) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/source/train/dl_guide.rst b/doc/source/train/dl_guide.rst index 3fa2e6d812a3f..fd07a15c45919 100644 --- a/doc/source/train/dl_guide.rst +++ b/doc/source/train/dl_guide.rst @@ -61,7 +61,7 @@ training. and :func:`ray.train.torch.prepare_data_loader` utilities below, and instead handle the logic directly inside your training function. - First, use the :func:~ray.train.torch.prepare_model` function to automatically move your model to the right device and wrap it in + First, use the :func:`~ray.train.torch.prepare_model` function to automatically move your model to the right device and wrap it in ``DistributedDataParallel`` .. code-block:: diff diff --git a/doc/source/train/getting-started.rst b/doc/source/train/getting-started.rst index d071b2c44ee83..113eb6ff607ed 100644 --- a/doc/source/train/getting-started.rst +++ b/doc/source/train/getting-started.rst @@ -107,6 +107,7 @@ Here are examples for some of the commonly used trainers: :language: python :start-after: __torch_single_run_begin__ :end-before: __torch_single_run_end__ + :dedent: Now let's convert this to a distributed multi-worker training function! @@ -128,6 +129,7 @@ Here are examples for some of the commonly used trainers: :language: python :start-after: __torch_trainer_begin__ :end-before: __torch_trainer_end__ + :dedent: See :ref:`train-porting-code` for a more comprehensive example. @@ -156,6 +158,7 @@ Here are examples for some of the commonly used trainers: :language: python :start-after: __tf_single_run_begin__ :end-before: __tf_single_run_end__ + :dedent: Now let's convert this to a distributed multi-worker training function! All you need to do is: @@ -177,6 +180,7 @@ Here are examples for some of the commonly used trainers: :language: python :start-after: __tf_trainer_begin__ :end-before: __tf_trainer_end__ + :dedent: See :ref:`train-porting-code` for a more comprehensive example. diff --git a/doc/source/tune/examples/tune-xgboost.ipynb b/doc/source/tune/examples/tune-xgboost.ipynb index edf640b0b30cf..cde9b8ce92e06 100644 --- a/doc/source/tune/examples/tune-xgboost.ipynb +++ b/doc/source/tune/examples/tune-xgboost.ipynb @@ -127,6 +127,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "ec2a13f8", "metadata": {}, @@ -226,7 +227,7 @@ "\n", "To address this fact, XGBoost uses a parameter called *Eta*, which is sometimes called\n", "the *learning rate*. Don't confuse this with learning rates from gradient descent!\n", - "The original [paper on stochastic gradient boosting](https://www.sciencedirect.com/science/article/abs/pii/S0167947301000652)\n", + "The original [paper on stochastic gradient boosting](https://jerryfriedman.su.domains/ftp/stobst.pdf)\n", "introduces this parameter like so:\n", "\n", "$$\n", @@ -1252,4 +1253,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} \ No newline at end of file +} diff --git a/doc/source/tune/faq.rst b/doc/source/tune/faq.rst index bb02b09ddb060..bbefeef2f4cd9 100644 --- a/doc/source/tune/faq.rst +++ b/doc/source/tune/faq.rst @@ -797,13 +797,11 @@ The reasons for this are: 3. Concurrent jobs are harder to debug. If a trial of job A fills the disk, trials from job B on the same node are impacted. In practice, it's hard to reason about these conditions from the logs if something goes wrong. -4. Some internal implementations in Ray Tune assume that you only have one job - running at a time. This can lead to conflicts. -The fourth reason is especially problematic when you run concurrent tuning jobs. For instance, -a symptom is when trials from job A use parameters specified in job B, leading to unexpected -results. +Previously, some internal implementations in Ray Tune assumed that you only have one job +running at a time. A symptom was when trials from job A used parameters specified in job B, +leading to unexpected results. Please refer to [this github issue](https://github.com/ray-project/ray/issues/30091#issuecomment-1431676976) -for more context and a workaround. +for more context and a workaround if you run into this issue. diff --git a/docker/ray-ml/Dockerfile b/docker/ray-ml/Dockerfile index 80085327a5563..18454405bd523 100644 --- a/docker/ray-ml/Dockerfile +++ b/docker/ray-ml/Dockerfile @@ -24,7 +24,7 @@ RUN sudo apt-get update \ && $HOME/anaconda3/bin/pip --no-cache-dir install -U \ -r requirements.txt \ # Then, keep requirements bounds as constraints and install remaining test dependencies - && $HOME/anaconda3/bin/pip --use-deprecated=legacy-resolver --no-cache-dir install -U \ + && $HOME/anaconda3/bin/pip --no-cache-dir install -U \ -c requirements.txt \ -r requirements_rllib.txt \ -r requirements_train.txt \ diff --git a/python/ray/_private/state_api_test_utils.py b/python/ray/_private/state_api_test_utils.py index 95bb75dc0601d..8ac0f91007360 100644 --- a/python/ray/_private/state_api_test_utils.py +++ b/python/ray/_private/state_api_test_utils.py @@ -9,6 +9,7 @@ import time import traceback from typing import Callable, Dict, List, Optional +from ray.experimental.state.api import list_tasks import ray from ray.actor import ActorHandle @@ -306,3 +307,15 @@ def periodic_invoke_state_apis_with_actor(*args, **kwargs) -> ActorHandle: print("State api actor is ready now.") actor.start.remote() return actor + + +def verify_failed_task(name: str, error_type: str) -> bool: + """ + Check if a task with 'name' has failed with the exact error type 'error_type' + """ + tasks = list_tasks(filters=[("name", "=", name)]) + assert len(tasks) == 1, tasks + t = tasks[0] + assert t["state"] == "FAILED", t + assert t["error_type"] == error_type, t + return True diff --git a/python/ray/_private/test_utils.py b/python/ray/_private/test_utils.py index 8e5ccccb9ce4d..dd78ab87ab575 100644 --- a/python/ray/_private/test_utils.py +++ b/python/ray/_private/test_utils.py @@ -1797,7 +1797,7 @@ def wandb_populate_run_location_hook(): def safe_write_to_results_json( - result: str, + result: dict, default_file_name: str = "/tmp/release_test_output.json", env_var: Optional[str] = "TEST_OUTPUT_JSON", ): @@ -1810,3 +1810,5 @@ def safe_write_to_results_json( with open(test_output_json_tmp, "wt") as f: json.dump(result, f) os.replace(test_output_json_tmp, test_output_json) + logger.info(f"Wrote results to {test_output_json}") + logger.info(json.dumps(result)) diff --git a/python/ray/_private/worker.py b/python/ray/_private/worker.py index 99eef572a0b3e..0465302bee2d8 100644 --- a/python/ray/_private/worker.py +++ b/python/ray/_private/worker.py @@ -3008,8 +3008,7 @@ def remote( invocation. The default value is 1. Pass "dynamic" to allow the task to decide how many return values to return during execution, and the caller will - receive an ObjectRef[ObjectRefGenerator] (note, this setting is - experimental). + receive an ObjectRef[ObjectRefGenerator]. See :ref:`dynamic generators ` for more details. num_cpus: The quantity of CPU resources to reserve for this task or for the lifetime of the actor. diff --git a/python/ray/_raylet.pyx b/python/ray/_raylet.pyx index 416bf4d5a16c3..e931cb6845621 100644 --- a/python/ray/_raylet.pyx +++ b/python/ray/_raylet.pyx @@ -137,7 +137,6 @@ import ray.cloudpickle as ray_pickle from ray.core.generated.common_pb2 import ActorDiedErrorContext from ray._private.async_compat import sync_to_async, get_new_event_loop from ray._private.client_mode_hook import disable_client_hook -from ray._private.signature import DUMMY_TYPE import ray._private.gcs_utils as gcs_utils import ray._private.memory_monitor as memory_monitor import ray._private.profiling as profiling @@ -170,13 +169,6 @@ current_task_id_lock = threading.Lock() job_config_initialized = False job_config_initialization_lock = threading.Lock() -# The cached serialized dummy arg b`__RAY_DUMMY__`. -cdef dummy_type_serialized_arg = None -# The type of DUMMY_TYPE. -cdef dummy_type_type = type(DUMMY_TYPE) -# The value of DUMMY_TYPE, cdef DUMMY_TYPE to avoid global lookup. -cdef dummy_type_value = DUMMY_TYPE - class ObjectRefGenerator: def __init__(self, refs): @@ -464,35 +456,20 @@ cdef prepare_args_internal( unique_ptr[CTaskArg](new CTaskArgByReference( c_arg, c_owner_address, - (arg).call_site_data))) # Avoid calling Python function + arg.call_site()))) else: - # The type check is because some custom types may not implement __eq__ - # well. So, we only handle the args which type and value are exactly match - # the DUMMY_TYPE. - # TODO(fyrestone): Maybe we can remove the DUMMY_TYPE or make the - # DUMMY_TYPE None. - # https://github.com/ray-project/ray/pull/32478/ - if type(arg) is dummy_type_type and arg == dummy_type_value: - global dummy_type_serialized_arg - if dummy_type_serialized_arg is None: - # Cache the serialized dummy arg. - dummy_type_serialized_arg = serialized_arg = \ - worker.get_serialization_context().serialize(arg) - else: - serialized_arg = dummy_type_serialized_arg - else: - try: - serialized_arg = worker.get_serialization_context( - ).serialize(arg) - except TypeError as e: - msg = ( - "Could not serialize the argument " - f"{repr(arg)} for a task or actor " - f"{function_descriptor.repr}. Check " - "https://docs.ray.io/en/master/ray-core/objects/serialization.html#troubleshooting " # noqa - "for more information.") - raise TypeError(msg) from e + try: + serialized_arg = worker.get_serialization_context( + ).serialize(arg) + except TypeError as e: + msg = ( + "Could not serialize the argument " + f"{repr(arg)} for a task or actor " + f"{function_descriptor.repr}. Check " + "https://docs.ray.io/en/master/ray-core/objects/serialization.html#troubleshooting " # noqa + "for more information.") + raise TypeError(msg) from e metadata = serialized_arg.metadata if language != Language.PYTHON: metadata_fields = metadata.split(b",") @@ -1984,19 +1961,18 @@ cdef class CoreWorker: self.python_scheduling_strategy_to_c( scheduling_strategy, &c_scheduling_strategy) - if retry_exception_allowlist: - try: - serialized_retry_exception_allowlist = ray_pickle.dumps( - retry_exception_allowlist, - ) - except TypeError as e: - msg = ( - "Could not serialize the retry exception allowlist" - f"{retry_exception_allowlist} for task {function_descriptor.repr}. " - "Check " - "https://docs.ray.io/en/master/ray-core/objects/serialization.html#troubleshooting " # noqa - "for more information.") - raise TypeError(msg) from e + try: + serialized_retry_exception_allowlist = ray_pickle.dumps( + retry_exception_allowlist, + ) + except TypeError as e: + msg = ( + "Could not serialize the retry exception allowlist" + f"{retry_exception_allowlist} for task {function_descriptor.repr}. " + "Check " + "https://docs.ray.io/en/master/ray-core/objects/serialization.html#troubleshooting " # noqa + "for more information.") + raise TypeError(msg) from e with self.profile_event(b"submit_task"): prepare_resources(resources, &c_resources) @@ -2672,10 +2648,7 @@ cdef class CoreWorker: eventloop, async_thread = self.get_event_loop( function_descriptor, specified_cgname) coroutine = func(*args, **kwargs) - if threading.get_ident() == async_thread.ident: - future = asyncio.ensure_future(coroutine, eventloop) - else: - future = asyncio.run_coroutine_threadsafe(coroutine, eventloop) + future = asyncio.run_coroutine_threadsafe(coroutine, eventloop) future.add_done_callback(lambda _: event.Notify()) with nogil: (CCoreWorkerProcess.GetCoreWorker() diff --git a/python/ray/air/checkpoint.py b/python/ray/air/checkpoint.py index 17e1c425186d8..f103ec2b6ad6b 100644 --- a/python/ray/air/checkpoint.py +++ b/python/ray/air/checkpoint.py @@ -211,6 +211,7 @@ def __init__( self._data_dict: Optional[Dict[str, Any]] = data_dict self._uri: Optional[str] = uri self._override_preprocessor: Optional["Preprocessor"] = None + self._override_preprocessor_set = False self._uuid = uuid.uuid4() @@ -396,7 +397,7 @@ def to_dict(self) -> dict: checkpoint_data[_METADATA_KEY] = self._metadata # If override_preprocessor is specified, then set that in the output dict. - if self._override_preprocessor: + if self._override_preprocessor_set: checkpoint_data[PREPROCESSOR_KEY] = self._override_preprocessor return checkpoint_data @@ -426,16 +427,15 @@ def from_directory(cls, path: Union[str, os.PathLike]) -> "Checkpoint": return checkpoint - # TODO: Deprecate `from_checkpoint`. For context, see #29058. @classmethod + @DeveloperAPI def from_checkpoint(cls, other: "Checkpoint") -> "Checkpoint": - """Create a checkpoint from a generic :py:class:`Checkpoint`. + """Create a checkpoint from a generic :class:`Checkpoint`. This method can be used to create a framework-specific checkpoint from a - generic :py:class:`Checkpoint` object. + generic :class:`Checkpoint` object. Examples: - >>> result = TorchTrainer.fit(...) # doctest: +SKIP >>> checkpoint = TorchCheckpoint.from_checkpoint(result.checkpoint) # doctest: +SKIP # noqa: E501 >>> model = checkpoint.get_model() # doctest: +SKIP @@ -541,7 +541,7 @@ def _to_directory(self, path: str, move_instead_of_copy: bool = False) -> None: self._save_checkpoint_metadata_in_directory(path) - if self._override_preprocessor: + if self._override_preprocessor_set and self._override_preprocessor: save_preprocessor_to_dir(self._override_preprocessor, path) def _to_directory_safe(self, path: str, move_instead_of_copy: bool = False) -> None: @@ -764,7 +764,7 @@ def __fspath__(self): def get_preprocessor(self) -> Optional["Preprocessor"]: """Return the saved preprocessor, if one exists.""" - if self._override_preprocessor: + if self._override_preprocessor_set: return self._override_preprocessor # The preprocessor will either be stored in an in-memory dict or @@ -787,10 +787,11 @@ def get_preprocessor(self) -> Optional["Preprocessor"]: return preprocessor - def set_preprocessor(self, preprocessor: "Preprocessor"): + def set_preprocessor(self, preprocessor: Optional["Preprocessor"]): """Saves the provided preprocessor to this Checkpoint.""" self._override_preprocessor = preprocessor + self._override_preprocessor_set = True @classmethod def _get_checkpoint_type( diff --git a/python/ray/air/examples/dreambooth/README.rst b/python/ray/air/examples/dreambooth/README.rst new file mode 100644 index 0000000000000..90ff959e68bff --- /dev/null +++ b/python/ray/air/examples/dreambooth/README.rst @@ -0,0 +1,134 @@ +Fine-tuning DreamBooth with Ray AIR +=================================== + +.. + ATTN: This should be kept in sync with release/air_examples/dreambooth/dreambooth_run.sh + +.. + section_intro + + +This example shows how to fine-tune a DreamBooth model using Ray AIR. + +Because of the large model sizes, you'll need 2 A10G GPUs per worker. + +The example can leverage data-parallel training to speed up training time. Of course, this will +require more GPUs. + +The demo tunes both the text_encoder and unet parts of Stable Diffusion, and utilizes the prior preserving loss function. + + +.. image:: images/dreambooth_example.png + :target: images/dreambooth_example.png + :alt: DreamBooth example + + +The full code repository can be found here: `https://github.com/ray-project/ray/blob/master/python/ray/air/examples/dreambooth/ `_ + +How it works +------------ + +For an up-to-date description of how the code works, +`please refer to the example in our documentation `_. + +Run the example +--------------- + +.. + section_run_example + +First, we download the pre-trained stable diffusion model as a starting point. + +We will then train this model with a few images of our subject. + +To achieve this, we choose a non-word as an identifier, e.g. ``unqtkn``. When fine-tuning the model with our subject, we will teach it that the prompt is ``A photo of a unqtkn ``. + +After fine-tuning we can run inference with this specific prompt. For instance: ``A photo of a unqtkn `` will create an image of our subject. + +Step 0: Preparation +^^^^^^^^^^^^^^^^^^^ + +Clone the Ray repository, go to the example directory, and install dependencies. + +.. code-block:: bash + + git clone https://github.com/ray-project/ray.git + cd ray/python/ray/air/examples/dreambooth + pip install -Ur requirements.txt + +Prepare some directories and environment variables. + +.. code-block:: bash + + export DATA_PREFIX="./" + export ORIG_MODEL_NAME="CompVis/stable-diffusion-v1-4" + export ORIG_MODEL_HASH="249dd2d739844dea6a0bc7fc27b3c1d014720b28" + export ORIG_MODEL_DIR="$DATA_PREFIX/model-orig" + export ORIG_MODEL_PATH="$ORIG_MODEL_DIR/models--${ORIG_MODEL_NAME/\//--}/snapshots/$ORIG_MODEL_HASH" + export TUNED_MODEL_DIR="$DATA_PREFIX/model-tuned" + export IMAGES_REG_DIR="$DATA_PREFIX/images-reg" + export IMAGES_OWN_DIR="$DATA_PREFIX/images-own" + export IMAGES_NEW_DIR="$DATA_PREFIX/images-new" + + export CLASS_NAME="toy car" + + mkdir -p $ORIG_MODEL_DIR $TUNED_MODEL_DIR $IMAGES_REG_DIR $IMAGES_OWN_DIR $IMAGES_NEW_DIR + +Copy some images for fine-tuning into ``$IMAGES_OWN_DIR``. + +Step 1: Download the pre-trained model +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Download and cache a pre-trained Stable-Diffusion model locally. +Default model and version are ``CompVis/stable-diffusion-v1-4`` +at git hash ``3857c45b7d4e78b3ba0f39d4d7f50a2a05aa23d4``. + +.. code-block:: + + python cache_model.py --model_dir=$ORIG_MODEL_DIR --model_name=$ORIG_MODEL_NAME --revision=$ORIG_MODEL_HASH + +Note that actual model files will be downloaded into +``\\snapshots\\`` directory. + +Step 2: Create the regularization images +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Create a regularization image set for a class of subjects: + +.. code-block:: + + python run_model.py \ + --model_dir=$ORIG_MODEL_PATH \ + --output_dir=$IMAGES_REG_DIR \ + --prompts="photo of a $CLASS_NAME" \ + --num_samples_per_prompt=200 + +Step 3: Fine-tune the model +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Save a few (4 to 5) images of the subject being fine-tuned +in a local directory. Then launch the training job with: + +.. code-block:: + + python train.py \ + --model_dir=$ORIG_MODEL_PATH \ + --output_dir=$TUNED_MODEL_DIR \ + --instance_images_dir=$IMAGES_OWN_DIR \ + --instance_prompt="a photo of unqtkn $CLASS_NAME" \ + --class_images_dir=$IMAGES_REG_DIR \ + --class_prompt="a photo of a $CLASS_NAME" + +Step 4: Generate images of our subject +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Try your model with the same commandline as Step 2, but point +to your own model this time! + +.. code-block:: + + python run_model.py \ + --model_dir=$TUNED_MODEL_DIR \ + --output_dir=$IMAGES_NEW_DIR \ + --prompts="photo of a unqtkn $CLASS_NAME" \ + --num_samples_per_prompt=20 diff --git a/python/ray/air/examples/dreambooth/cache_model.py b/python/ray/air/examples/dreambooth/cache_model.py new file mode 100644 index 0000000000000..2d8d62e43e371 --- /dev/null +++ b/python/ray/air/examples/dreambooth/cache_model.py @@ -0,0 +1,20 @@ +# Cache model files to a local directory + +import os + +from huggingface_hub import snapshot_download + +from flags import cache_model_flags + + +def cache(args): + os.makedirs(args.model_dir, exist_ok=True) + + snapshot_download( + repo_id=args.model_name, revision=args.revision, cache_dir=args.model_dir + ) + + +if __name__ == "__main__": + args = cache_model_flags().parse_args() + cache(args) diff --git a/python/ray/air/examples/dreambooth/dataset.py b/python/ray/air/examples/dreambooth/dataset.py new file mode 100644 index 0000000000000..c9f078b42bcea --- /dev/null +++ b/python/ray/air/examples/dreambooth/dataset.py @@ -0,0 +1,92 @@ +import pandas as pd +import torch + +from ray.data import read_images +from ray.data.preprocessors import TorchVisionPreprocessor +from torchvision import transforms +from transformers import AutoTokenizer + + +def get_train_dataset(args, image_resolution=512): + """Build a Ray Dataset for fine-tuning DreamBooth model.""" + # Load images into Ray Dataset + instance_dataset = read_images(args.instance_images_dir) + class_dataset = read_images(args.class_images_dir) + + # We now duplicate the instance images multiple times to make the + # two sets contain exactly the same number of images. + # This is so we can zip them up during training to compute the + # prior preserving loss in one pass. + dup_times = class_dataset.count() // instance_dataset.count() + instance_dataset = instance_dataset.map_batches( + lambda df: pd.concat([df] * dup_times) + ) + + # Load tokenizer for tokenizing the image prompts. + tokenizer = AutoTokenizer.from_pretrained( + pretrained_model_name_or_path=args.model_dir, + subfolder="tokenizer", + ) + + def _tokenize(prompt): + return tokenizer( + prompt, + truncation=True, + padding="max_length", + max_length=tokenizer.model_max_length, + return_tensors="pt", + ).input_ids.numpy() + + # Get the token ids for both prompts. + class_prompt_ids = _tokenize(args.class_prompt)[0] + instance_prompt_ids = _tokenize(args.instance_prompt)[0] + + # Image preprocessing. + # Instance and class images used by this example are in sizes 700x700 + # and 512x512 respectively. + # Depending on the sizes of actual training images, there may need to be a + # transforms.Resize() step as well. + transform = transforms.Compose( + [ + transforms.ToTensor(), + transforms.RandomCrop(image_resolution), + transforms.Normalize([0.5], [0.5]), + ] + ) + preprocessor = TorchVisionPreprocessor(["image"], transform=transform) + + instance_dataset = preprocessor.transform(instance_dataset).add_column( + "prompt_ids", lambda df: [instance_prompt_ids] * len(df) + ) + class_dataset = preprocessor.transform(class_dataset).add_column( + "prompt_ids", lambda df: [class_prompt_ids] * len(df) + ) + # --- + + # Now, zip the images up. + final_size = min(instance_dataset.count(), class_dataset.count()) + train_dataset = ( + instance_dataset.limit(final_size) + .repartition(final_size) + .zip(class_dataset.limit(final_size).repartition(final_size)) + ) + + return train_dataset.random_shuffle() + + +def collate(batch, device, dtype): + """Build Torch training batch.""" + # Layout of the batch is that instance image data (pixels, prompt ids) occupy + # the top half of the batch. And class image data occupy the bottom half + # of the batch. + # During training, a batch will be chunked into 2 sub-batches for prior + # preserving loss calculation. + images = torch.squeeze(torch.stack([batch["image"], batch["image_1"]])) + images = images.to(memory_format=torch.contiguous_format).float() + + prompt_ids = torch.cat([batch["prompt_ids"], batch["prompt_ids_1"]], dim=0) + + return { + "prompt_ids": prompt_ids.to(device), # token ids should stay int. + "images": images.to(device, dtype=dtype), + } diff --git a/python/ray/air/examples/dreambooth/flags.py b/python/ray/air/examples/dreambooth/flags.py new file mode 100644 index 0000000000000..55c1d23930fba --- /dev/null +++ b/python/ray/air/examples/dreambooth/flags.py @@ -0,0 +1,136 @@ +import argparse + + +def train_arguments(): + """Commandline arguments for running DreamBooth training script.""" + parser = argparse.ArgumentParser() + + parser.add_argument( + "--model_dir", + type=str, + default=None, + required=True, + help="Path to a pretrained huggingface Stable Diffusion model.", + ) + parser.add_argument( + "--output_dir", + type=str, + default=None, + required=True, + help="Directory where trained models are saved.", + ) + parser.add_argument( + "--instance_images_dir", + type=str, + default=None, + required=True, + help=( + "Directory where a few images of the instance to be fine tuned " + "into the model are saved." + ), + ) + parser.add_argument( + "--instance_prompt", + type=str, + default=None, + required=True, + help=("Prompt for creating the instance images."), + ) + parser.add_argument( + "--class_images_dir", + type=str, + default=None, + required=True, + help=( + "Directory where images of similar objects for preserving " + "model priors are saved." + ), + ) + parser.add_argument( + "--class_prompt", + type=str, + default=None, + required=True, + help=("Prompt for creating the class images."), + ) + parser.add_argument( + "--train_batch_size", type=int, default=1, help="Train batch size." + ) + parser.add_argument("--lr", type=float, default=5e-6, help="Train learning rate.") + parser.add_argument( + "--num_epochs", type=int, default=4, help="Number of epochs to train." + ) + parser.add_argument( + "--prior_loss_weight", + type=float, + default=1.0, + help="The weight for prior preservation loss.", + ) + parser.add_argument( + "--max_grad_norm", type=float, default=1.0, help="Maximum gradient norm." + ) + parser.add_argument("--num_workers", type=int, default=2, help="Number of workers.") + + return parser + + +def cache_model_flags(): + """Commandline arguments for running local model caching script.""" + parser = argparse.ArgumentParser() + + parser.add_argument( + "--model_dir", + type=str, + default=None, + required=True, + help="Directory to write the cached model files.", + ) + parser.add_argument( + "--model_name", + type=str, + default="CompVis/stable-diffusion-v1-4", + help="Name of the huggingface model.", + ) + parser.add_argument( + "--revision", + type=str, + default="3857c45b7d4e78b3ba0f39d4d7f50a2a05aa23d4", + help="Revision of the huggingface model repo to cache.", + ) + + return parser + + +def run_model_flags(): + """Commandline arguments for running a tuned DreamBooth model.""" + parser = argparse.ArgumentParser() + + parser.add_argument( + "--model_dir", + type=str, + default=None, + required=True, + help="Directory of the tuned model files.", + ) + parser.add_argument( + "--output_dir", + type=str, + default=None, + required=True, + help="Directory to save the generated images.", + ) + parser.add_argument( + "--prompts", + type=str, + default=None, + required=True, + help="Comma separated prompt strings for generating the images.", + ) + parser.add_argument( + "--num_samples_per_prompt", + type=int, + default=1, + help="Number of images to generate for each prompt.", + ) + + return parser diff --git a/python/ray/air/examples/dreambooth/images/dreambooth_example.png b/python/ray/air/examples/dreambooth/images/dreambooth_example.png new file mode 100644 index 0000000000000..4909d8ed13b1a Binary files /dev/null and b/python/ray/air/examples/dreambooth/images/dreambooth_example.png differ diff --git a/python/ray/air/examples/dreambooth/images/dreambooth_training.png b/python/ray/air/examples/dreambooth/images/dreambooth_training.png new file mode 100644 index 0000000000000..8a759bcd51da9 Binary files /dev/null and b/python/ray/air/examples/dreambooth/images/dreambooth_training.png differ diff --git a/python/ray/air/examples/dreambooth/images/unqtkn/1.jpg b/python/ray/air/examples/dreambooth/images/unqtkn/1.jpg new file mode 100644 index 0000000000000..1d629ecb3b4c5 Binary files /dev/null and b/python/ray/air/examples/dreambooth/images/unqtkn/1.jpg differ diff --git a/python/ray/air/examples/dreambooth/images/unqtkn/2.jpg b/python/ray/air/examples/dreambooth/images/unqtkn/2.jpg new file mode 100644 index 0000000000000..017556bc8d4e4 Binary files /dev/null and b/python/ray/air/examples/dreambooth/images/unqtkn/2.jpg differ diff --git a/python/ray/air/examples/dreambooth/images/unqtkn/3.jpg b/python/ray/air/examples/dreambooth/images/unqtkn/3.jpg new file mode 100644 index 0000000000000..ecfe0a1903c82 Binary files /dev/null and b/python/ray/air/examples/dreambooth/images/unqtkn/3.jpg differ diff --git a/python/ray/air/examples/dreambooth/images/unqtkn/4.jpg b/python/ray/air/examples/dreambooth/images/unqtkn/4.jpg new file mode 100644 index 0000000000000..b425f40aa0dae Binary files /dev/null and b/python/ray/air/examples/dreambooth/images/unqtkn/4.jpg differ diff --git a/python/ray/air/examples/dreambooth/images/unqtkn/5.jpg b/python/ray/air/examples/dreambooth/images/unqtkn/5.jpg new file mode 100644 index 0000000000000..95ebf0001f420 Binary files /dev/null and b/python/ray/air/examples/dreambooth/images/unqtkn/5.jpg differ diff --git a/python/ray/air/examples/dreambooth/requirements.txt b/python/ray/air/examples/dreambooth/requirements.txt new file mode 100644 index 0000000000000..4d031ccc7c4fc --- /dev/null +++ b/python/ray/air/examples/dreambooth/requirements.txt @@ -0,0 +1,8 @@ +accelerate==0.15.0 +bitsandbytes +diffusers==0.11.1 +flax==0.6.4 +huggingface_hub +numpy==1.21 +torchvision +transformers>=4.25.1 diff --git a/python/ray/air/examples/dreambooth/run_model.py b/python/ray/air/examples/dreambooth/run_model.py new file mode 100644 index 0000000000000..e8d8a5d6da8ba --- /dev/null +++ b/python/ray/air/examples/dreambooth/run_model.py @@ -0,0 +1,33 @@ +import hashlib +from os import path + +from diffusers import DiffusionPipeline +import torch + +from flags import run_model_flags + + +def run(args): + print(f"Loading model from {args.model_dir}") + pipeline = DiffusionPipeline.from_pretrained( + args.model_dir, torch_dtype=torch.float16 + ) + pipeline.set_progress_bar_config(disable=True) + if torch.cuda.is_available(): + pipeline.to("cuda") + + prompts = args.prompts.split(",") + + # Generate 1 image to reduce memory consumption. + for prompt in prompts: + for i in range(args.num_samples_per_prompt): + for image in pipeline(prompt).images: + hash_image = hashlib.sha1(image.tobytes()).hexdigest() + image_filename = path.join(args.output_dir, f"{i}-{hash_image}.jpg") + image.save(image_filename) + print(f"Saved {image_filename}") + + +if __name__ == "__main__": + args = run_model_flags().parse_args() + run(args) diff --git a/python/ray/air/examples/dreambooth/train.py b/python/ray/air/examples/dreambooth/train.py new file mode 100644 index 0000000000000..643ab66cf2b7b --- /dev/null +++ b/python/ray/air/examples/dreambooth/train.py @@ -0,0 +1,228 @@ +import itertools + +from diffusers import ( + AutoencoderKL, + DDPMScheduler, + DiffusionPipeline, + UNet2DConditionModel, +) +from diffusers.utils.import_utils import is_xformers_available +from ray.air import session, ScalingConfig +from ray.train.torch import TorchTrainer +import torch +import torch.nn.functional as F +from torch.nn.parallel import DistributedDataParallel +from torch.nn.utils import clip_grad_norm_ +from transformers import CLIPTextModel + +from dataset import collate, get_train_dataset +from flags import train_arguments + + +def prior_preserving_loss(model_pred, target, weight): + # Chunk the noise and model_pred into two parts and compute + # the loss on each part separately. + model_pred, model_pred_prior = torch.chunk(model_pred, 2, dim=0) + target, target_prior = torch.chunk(target, 2, dim=0) + + # Compute instance loss + loss = F.mse_loss(model_pred.float(), target.float(), reduction="mean") + + # Compute prior loss + prior_loss = F.mse_loss( + model_pred_prior.float(), target_prior.float(), reduction="mean" + ) + + # Add the prior loss to the instance loss. + return loss + weight * prior_loss + + +def get_target(scheduler, noise, latents, timesteps): + """Get the target for loss depending on the prediction type.""" + pred_type = scheduler.config.prediction_type + if pred_type == "epsilon": + return noise + if pred_type == "v_prediction": + return scheduler.get_velocity(latents, noise, timesteps) + raise ValueError(f"Unknown prediction type {pred_type}") + + +def load_models(config, cuda): + """Load pre-trained Stable Diffusion models.""" + # Load all models in bfloat16 to save GRAM. + # For models that are only used for inferencing, + # full precision is also not required. + dtype = torch.bfloat16 + + text_encoder = CLIPTextModel.from_pretrained( + args.model_dir, + subfolder="text_encoder", + torch_dtype=dtype, + ) + text_encoder.to(cuda[1]) + text_encoder.train() + + noise_scheduler = DDPMScheduler.from_pretrained( + config["model_dir"], + subfolder="scheduler", + torch_dtype=dtype, + ) + + # VAE is only used for inference, keeping weights in full precision is not required. + vae = AutoencoderKL.from_pretrained( + config["model_dir"], + subfolder="vae", + torch_dtype=dtype, + ) + # We are not training VAE part of the model. + vae.requires_grad_(False) + vae.to(cuda[1]) + + # Convert unet to bf16 to save GRAM. + unet = UNet2DConditionModel.from_pretrained( + config["model_dir"], + subfolder="unet", + torch_dtype=dtype, + ) + if is_xformers_available(): + unet.enable_xformers_memory_efficient_attention() + # UNET is the largest component, occupying first GPU by itself. + unet.to(cuda[0]) + unet.train() + + torch.cuda.empty_cache() + + return text_encoder, noise_scheduler, vae, unet + + +def get_cuda_devices(): + devices = [f"cuda:{i}" for i in range(torch.cuda.device_count())] + local_rank = session.get_local_rank() + assert len(devices) >= 2, "Require at least 2 GPU devices to work." + return devices[(local_rank * 2) : ((local_rank * 2) + 2)] + + +def train_fn(config): + cuda = get_cuda_devices() + + # Load pre-trained models. + text_encoder, noise_scheduler, vae, unet = load_models(config, cuda) + + # Wrap in DDP + text_encoder = DistributedDataParallel( + text_encoder, device_ids=[cuda[1]], output_device=cuda[1] + ) + unet = DistributedDataParallel(unet, device_ids=[cuda[0]], output_device=cuda[0]) + + # Use the regular AdamW optimizer to work with bfloat16 weights. + optimizer = torch.optim.AdamW( + itertools.chain(text_encoder.parameters(), unet.parameters()), + lr=config["lr"], + ) + + train_dataset = session.get_dataset_shard("train") + + # Train! + num_train_epochs = config["num_epochs"] + + print(f"Running {num_train_epochs} epochs.") + + global_step = 0 + for epoch in range(num_train_epochs): + for step, batch in enumerate( + train_dataset.iter_torch_batches( + batch_size=config["train_batch_size"], device=cuda[1] + ) + ): + # Load batch on GPU 2 because VAE and text encoder are there. + batch = collate(batch, cuda[1], torch.bfloat16) + + optimizer.zero_grad() + + # Convert images to latent space + latents = vae.encode(batch["images"]).latent_dist.sample() * 0.18215 + + # Sample noise that we'll add to the latents + noise = torch.randn_like(latents) + bsz = latents.shape[0] + # Sample a random timestep for each image + timesteps = torch.randint( + 0, + noise_scheduler.config.num_train_timesteps, + (bsz,), + device=latents.device, + ) + timesteps = timesteps.long() + + # Add noise to the latents according to the noise magnitude at each timestep + # (this is the forward diffusion process) + noisy_latents = noise_scheduler.add_noise(latents, noise, timesteps) + + # Get the text embedding for conditioning + encoder_hidden_states = text_encoder(batch["prompt_ids"])[0] + + # Predict the noise residual. We need to move all data bits to GPU 1. + model_pred = unet( + noisy_latents.to(cuda[0]), + timesteps.to(cuda[0]), + encoder_hidden_states.to(cuda[0]), + ).sample + target = get_target(noise_scheduler, noise, latents, timesteps).to(cuda[0]) + + # Now, move model prediction to GPU 2 for loss calculation. + loss = prior_preserving_loss( + model_pred, target, config["prior_loss_weight"] + ) + loss.backward() + + # Gradient clipping before optimizer stepping. + clip_grad_norm_( + itertools.chain(text_encoder.parameters(), unet.parameters()), + config["max_grad_norm"], + ) + + optimizer.step() # Step all optimizers. + + global_step += 1 + results = { + "step": global_step, + "loss": loss.detach().item(), + } + session.report(results) + + # Create pipeline using the trained modules and save it. + if session.get_world_rank() == 0: + pipeline = DiffusionPipeline.from_pretrained( + config["model_dir"], + text_encoder=text_encoder.module, + unet=unet.module, + ) + pipeline.save_pretrained(config["output_dir"]) + + +if __name__ == "__main__": + args = train_arguments().parse_args() + + # Build training dataset. + train_dataset = get_train_dataset(args) + + print(f"Loaded training dataset (size: {train_dataset.count()})") + + # Train with Ray AIR TorchTrainer. + trainer = TorchTrainer( + train_fn, + train_loop_config=vars(args), + scaling_config=ScalingConfig( + use_gpu=True, + num_workers=args.num_workers, + resources_per_worker={ + "GPU": 2, + }, + ), + datasets={ + "train": train_dataset, + }, + ) + result = trainer.fit() + + print(result) diff --git a/python/ray/air/tests/execution/test_tracked_actor.py b/python/ray/air/tests/execution/test_tracked_actor.py index 924ee88f7f258..f3eadaed6450b 100644 --- a/python/ray/air/tests/execution/test_tracked_actor.py +++ b/python/ray/air/tests/execution/test_tracked_actor.py @@ -1,4 +1,5 @@ from collections import Counter +import gc from typing import Any, Optional, Type import pytest @@ -39,6 +40,14 @@ def ray_start_4_cpus(): ray.shutdown() +@pytest.fixture +def cleanup(): + # Garbage collect at the start + # This ensures that all resources are freed up for the upcoming test. + gc.collect() + yield + + class Actor: def __init__(self, **kwargs): self.kwargs = kwargs @@ -176,7 +185,7 @@ def stop_callback(tracked_actor): "resource_manager_cls", [FixedResourceManager, PlacementGroupResourceManager] ) @pytest.mark.parametrize("where", ["init", "fn"]) -def test_actor_fail(ray_start_4_cpus, resource_manager_cls, where): +def test_actor_fail(ray_start_4_cpus, cleanup, resource_manager_cls, where): """Test that actor failures are handled properly. - Start actor that either fails on init or in a task (RayActorError) diff --git a/python/ray/air/tests/test_checkpoints.py b/python/ray/air/tests/test_checkpoints.py index feecfc0d432d4..0b6aced414b6a 100644 --- a/python/ray/air/tests/test_checkpoints.py +++ b/python/ray/air/tests/test_checkpoints.py @@ -736,6 +736,11 @@ def testDictCheckpointSetPreprocessor(self): preprocessor = checkpoint.get_preprocessor() assert preprocessor.multiplier == 1 + # Check that we can set it to None + checkpoint.set_preprocessor(None) + preprocessor = checkpoint.get_preprocessor() + assert preprocessor is None + def testDictCheckpointSetPreprocessorAsDir(self): preprocessor = DummyPreprocessor(1) data = {"metric": 5} diff --git a/python/ray/data/__init__.py b/python/ray/data/__init__.py index 9a114f84490a2..f4894f0a3d985 100644 --- a/python/ray/data/__init__.py +++ b/python/ray/data/__init__.py @@ -1,3 +1,11 @@ +import sys + +# Short term workaround for https://github.com/ray-project/ray/issues/32435 +# Datasets currently has a hard dependency on pandas, so it doesn't need to be delayed. +# ray.data import is still eager for all ray imports for Python 3.6: +if sys.version_info >= (3, 7): + import pandas # noqa + from ray.data._internal.compute import ActorPoolStrategy from ray.data._internal.progress_bar import set_progress_bars from ray.data.dataset import Dataset diff --git a/python/ray/data/_internal/block_batching.py b/python/ray/data/_internal/block_batching.py index febee5a06b072..477d739a225cb 100644 --- a/python/ray/data/_internal/block_batching.py +++ b/python/ray/data/_internal/block_batching.py @@ -240,12 +240,31 @@ def _resolve_blocks( An iterator over resolved blocks. """ + hit = 0 + miss = 0 + unknown = 0 for block_ref in block_ref_iter: if block_ref is not None: stats_timer = stats.iter_get_s.timer() if stats else nullcontext() + # Count the number of blocks that we hit locally or miss (so have to + # fetch from remote node). This is to measure the effectiveness of + # prefetch. + loc = ray.experimental.get_object_locations([block_ref]) + nodes = loc[block_ref]["node_ids"] + if nodes: + current = ray.get_runtime_context().get_node_id() + if current in nodes: + hit += 1 + else: + miss += 1 + else: + unknown += 1 with stats_timer: block = ray.get(block_ref) yield block + stats.iter_blocks_local = hit + stats.iter_blocks_remote = miss + stats.iter_unknown_location = unknown def _prefetch_blocks( diff --git a/python/ray/data/_internal/bulk_dataset_iterator.py b/python/ray/data/_internal/dataset_iterator_impl.py similarity index 98% rename from python/ray/data/_internal/bulk_dataset_iterator.py rename to python/ray/data/_internal/dataset_iterator_impl.py index d7d19caaa906a..edcc1b6daae46 100644 --- a/python/ray/data/_internal/bulk_dataset_iterator.py +++ b/python/ray/data/_internal/dataset_iterator_impl.py @@ -11,7 +11,7 @@ from ray.data import Dataset -class BulkDatasetIterator(DatasetIterator): +class DatasetIteratorImpl(DatasetIterator): def __init__( self, base_dataset: "Dataset", diff --git a/python/ray/data/_internal/execution/bulk_executor.py b/python/ray/data/_internal/execution/bulk_executor.py index 53ca0e59b8023..e4e522b97dd8b 100644 --- a/python/ray/data/_internal/execution/bulk_executor.py +++ b/python/ray/data/_internal/execution/bulk_executor.py @@ -5,6 +5,7 @@ from ray.data._internal.execution.interfaces import ( Executor, ExecutionOptions, + OutputIterator, RefBundle, PhysicalOperator, ) @@ -81,7 +82,7 @@ def execute_recursive(op: PhysicalOperator) -> List[RefBundle]: ) return output - return execute_recursive(dag) + return OutputIterator(execute_recursive(dag)) def get_stats(self) -> DatasetStats: return self._stats diff --git a/python/ray/data/_internal/execution/interfaces.py b/python/ray/data/_internal/execution/interfaces.py index 2aee8c531eb93..6b52c90df210a 100644 --- a/python/ray/data/_internal/execution/interfaces.py +++ b/python/ray/data/_internal/execution/interfaces.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import Dict, List, Optional, Iterable, Tuple, Callable +from typing import Dict, List, Optional, Iterable, Iterator, Tuple, Callable, Union import ray from ray.data._internal.logical.interfaces import Operator @@ -9,6 +9,9 @@ from ray.data.context import DatasetContext from ray.types import ObjectRef +# Node id string returned by `ray.get_runtime_context().get_node_id()`. +NodeIdStr = str + @dataclass class RefBundle: @@ -38,6 +41,9 @@ class RefBundle: # output splits. It is otherwise None. output_split_idx: Optional[int] = None + # Cached location, used for get_cached_location(). + _cached_location: Optional[NodeIdStr] = None + def __post_init__(self): for b in self.blocks: assert isinstance(b, tuple), b @@ -74,6 +80,28 @@ def destroy_if_owned(self) -> int: trace_deallocation(b[0], "RefBundle.destroy_if_owned", free=should_free) return self.size_bytes() if should_free else 0 + def get_cached_location(self) -> Optional[NodeIdStr]: + """Return a location for this bundle's data, if possible. + + Caches the resolved location so multiple calls to this are efficient. + """ + if self._cached_location is None: + # Only consider the first block in the bundle for now. TODO(ekl) consider + # taking into account other blocks. + ref = self.blocks[0][0] + # This call is pretty fast for owned objects (~5k/s), so we don't need to + # batch it for now. + locs = ray.experimental.get_object_locations([ref]) + nodes = locs[ref]["node_ids"] + if nodes: + self._cached_location = nodes[0] + else: + self._cached_location = "" + if self._cached_location: + return self._cached_location + else: + return None # Return None if cached location is "". + def __eq__(self, other) -> bool: return self is other @@ -167,14 +195,16 @@ class ExecutionOptions: resource_limits: ExecutionResources = ExecutionResources() # Set this to prefer running tasks on the same node as the output - # node (node driving the execution). - locality_with_output: bool = False + # node (node driving the execution). It can also be set to a list of node ids + # to spread the outputs across those nodes. + locality_with_output: Union[bool, List[NodeIdStr]] = False # Set this to preserve the ordering between blocks processed by operators under the # streaming executor. The bulk executor always preserves order. preserve_order: bool = False - # Whether to enable locality-aware task dispatch to actors (on by default). + # Whether to enable locality-aware task dispatch to actors (on by default). This + # applies to both ActorPoolStrategy map and streaming_split operations. actor_locality_enabled: bool = True @@ -334,6 +364,15 @@ def get_work_refs(self) -> List[ray.ObjectRef]: """ return [] + def throttling_disabled(self) -> bool: + """Whether to disable resource throttling for this operator. + + This should return True for operators that only manipulate bundle metadata + (e.g., the OutputSplitter operator). This hints to the execution engine that + these operators should not be throttled based on resource usage. + """ + return False + def num_active_work_refs(self) -> int: """Return the number of active work refs. @@ -390,6 +429,34 @@ def incremental_resource_usage(self) -> ExecutionResources: return ExecutionResources() +class OutputIterator(Iterator[RefBundle]): + """Iterator used to access the output of an Executor execution. + + This is a blocking iterator. Datasets guarantees that all its iterators are + thread-safe (i.e., multiple threads can block on them at the same time). + """ + + def __init__(self, base: Iterable[RefBundle]): + self._it = iter(base) + + def get_next(self, output_split_idx: Optional[int] = None) -> RefBundle: + """Can be used to pull outputs by a specified output index. + + This is used to support the streaming_split() API, where the output of a + streaming execution is to be consumed by multiple processes. + + Args: + output_split_idx: The output split index to get results for. This arg is + only allowed for iterators created by `Dataset.streaming_split()`. + """ + if output_split_idx is not None: + raise NotImplementedError() + return next(self._it) + + def __next__(self) -> RefBundle: + return self.get_next() + + class Executor: """Abstract class for executors, which implement physical operator execution. @@ -404,7 +471,7 @@ def __init__(self, options: ExecutionOptions): def execute( self, dag: PhysicalOperator, initial_stats: Optional[DatasetStats] = None - ) -> Iterable[RefBundle]: + ) -> OutputIterator: """Start execution. Args: diff --git a/python/ray/data/_internal/execution/legacy_compat.py b/python/ray/data/_internal/execution/legacy_compat.py index 9d01e88b0b188..ddacb80510305 100644 --- a/python/ray/data/_internal/execution/legacy_compat.py +++ b/python/ray/data/_internal/execution/legacy_compat.py @@ -41,21 +41,43 @@ def execute_to_legacy_block_iterator( allow_clear_input_blocks: bool, dataset_uuid: str, ) -> Iterator[ObjectRef[Block]]: - """Execute a plan with the new executor and return a block iterator. + """Same as execute_to_legacy_bundle_iterator but returning blocks.""" + bundle_iter = execute_to_legacy_bundle_iterator( + executor, plan, allow_clear_input_blocks, dataset_uuid + ) + for bundle in bundle_iter: + for block, _ in bundle.blocks: + yield block + + +def execute_to_legacy_bundle_iterator( + executor: Executor, + plan: ExecutionPlan, + allow_clear_input_blocks: bool, + dataset_uuid: str, + dag_rewrite=None, +) -> Iterator[RefBundle]: + """Execute a plan with the new executor and return a bundle iterator. Args: executor: The executor to use. plan: The legacy plan to execute. allow_clear_input_blocks: Whether the executor may consider clearing blocks. dataset_uuid: UUID of the dataset for this execution. + dag_rewrite: Callback that can be used to mutate the DAG prior to execution. + This is currently used as a legacy hack to inject the OutputSplit operator + for `Dataset.streaming_split()`. Returns: - The output as a block iterator. + The output as a bundle iterator. """ + if DatasetContext.get_current().optimizer_enabled: dag, stats = get_execution_plan(plan._logical_plan).dag, None else: dag, stats = _to_operator_dag(plan, allow_clear_input_blocks) + if dag_rewrite: + dag = dag_rewrite(dag) # Enforce to preserve ordering if the plan has stages required to do so, such as # Zip and Sort. @@ -64,10 +86,7 @@ def execute_to_legacy_block_iterator( executor._options.preserve_order = True bundle_iter = executor.execute(dag, initial_stats=stats) - - for bundle in bundle_iter: - for block, _ in bundle.blocks: - yield block + return bundle_iter def execute_to_legacy_block_list( diff --git a/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py b/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py index cd91bdf9740da..286f16c1f228a 100644 --- a/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py +++ b/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py @@ -12,6 +12,7 @@ ExecutionOptions, PhysicalOperator, TaskContext, + NodeIdStr, ) from ray.data._internal.execution.operators.map_operator import ( MapOperator, @@ -21,9 +22,6 @@ from ray.types import ObjectRef from ray._raylet import ObjectRefGenerator -# Type alias for a node id. -NodeIdStr = str - # Higher values here are better for prefetching and locality. It's ok for this to be # fairly high since streaming backpressure prevents us from overloading actors. DEFAULT_MAX_TASKS_IN_FLIGHT = 4 @@ -656,14 +654,4 @@ def _get_location(self, bundle: RefBundle) -> Optional[NodeIdStr]: Returns: A node id associated with the bundle, or None if unknown. """ - # Only consider the first block in the bundle for now. TODO(ekl) consider - # taking into account other blocks. - ref = bundle.blocks[0][0] - # This call is pretty fast for owned objects (~5k/s), so we don't need to - # batch it for now. - locs = ray.experimental.get_object_locations([ref]) - nodes = locs[ref]["node_ids"] - if nodes: - return nodes[0] - else: - return None + return bundle.get_cached_location() diff --git a/python/ray/data/_internal/execution/operators/map_operator.py b/python/ray/data/_internal/execution/operators/map_operator.py index 3a55725a234a2..57fb4e7d0b4a1 100644 --- a/python/ray/data/_internal/execution/operators/map_operator.py +++ b/python/ray/data/_internal/execution/operators/map_operator.py @@ -1,4 +1,5 @@ from abc import ABC, abstractmethod +import copy from dataclasses import dataclass import itertools from typing import List, Iterator, Any, Dict, Optional, Union @@ -46,6 +47,7 @@ def __init__( self._transform_fn = transform_fn self._ray_remote_args = _canonicalize_ray_remote_args(ray_remote_args or {}) + self._ray_remote_args_factory = None # Bundles block references up to the min_rows_per_bundle target. self._block_ref_bundler = _BlockRefBundler(min_rows_per_bundle) @@ -132,14 +134,30 @@ def start(self, options: "ExecutionOptions"): self._output_queue = _OrderedOutputQueue() else: self._output_queue = _UnorderedOutputQueue() + if options.locality_with_output: - # Try to schedule tasks locally. - self._ray_remote_args[ - "scheduling_strategy" - ] = NodeAffinitySchedulingStrategy( - ray.get_runtime_context().get_node_id(), - soft=True, - ) + if isinstance(options.locality_with_output, list): + locs = options.locality_with_output + else: + locs = [ray.get_runtime_context().get_node_id()] + + class RoundRobinAssign: + def __init__(self, locs): + self.locs = locs + self.i = 0 + + def __call__(self, args): + args = copy.deepcopy(args) + args["scheduling_strategy"] = NodeAffinitySchedulingStrategy( + self.locs[self.i], + soft=True, + ) + self.i += 1 + self.i %= len(self.locs) + return args + + self._ray_remote_args_factory = RoundRobinAssign(locs) + # Put the function def in the object store to avoid repeated serialization # in case it's large (i.e., closure captures large objects). self._transform_fn_ref = ray.put(self._transform_fn) @@ -159,6 +177,11 @@ def add_input(self, refs: RefBundle, input_index: int): bundle = self._block_ref_bundler.get_next_bundle() self._add_bundled_input(bundle) + def _get_runtime_ray_remote_args(self) -> Dict[str, Any]: + if self._ray_remote_args_factory: + return self._ray_remote_args_factory(self._ray_remote_args) + return self._ray_remote_args + @abstractmethod def _add_bundled_input(self, refs: RefBundle): """Add a pre-bundled upstream output to this operator. diff --git a/python/ray/data/_internal/execution/operators/output_splitter.py b/python/ray/data/_internal/execution/operators/output_splitter.py index 41971bb21cb4c..b75fcce0e09f9 100644 --- a/python/ray/data/_internal/execution/operators/output_splitter.py +++ b/python/ray/data/_internal/execution/operators/output_splitter.py @@ -1,5 +1,5 @@ import math -from typing import List +from typing import List, Dict, Optional from ray.data.block import Block, BlockMetadata, BlockAccessor from ray.data._internal.remote_fn import cached_remote_fn @@ -7,7 +7,9 @@ from ray.data._internal.execution.interfaces import ( RefBundle, PhysicalOperator, + ExecutionOptions, ExecutionResources, + NodeIdStr, ) from ray.types import ObjectRef @@ -17,10 +19,12 @@ class OutputSplitter(PhysicalOperator): The output bundles of this operator will have a `bundle.output_split_idx` attr set to an integer from [0..n-1]. This operator tries to divide the rows evenly - across output splits. + across output splits. If the `equal` option is set, the operator will furthermore + guarantee an exact split of rows across outputs, truncating the Dataset as needed. - If the `equal` option is set, the operator will furthermore guarantee an exact - split of rows across outputs, truncating the Dataset as needed. + Implementation wise, this operator keeps an internal buffer of bundles. The buffer + has a minimum size calculated to enable a good locality hit rate, as well as ensure + we can satisfy the `equal` requirement. OutputSplitter does not provide any ordering guarantees. """ @@ -30,6 +34,7 @@ def __init__( input_op: PhysicalOperator, n: int, equal: bool, + locality_hints: Optional[List[NodeIdStr]] = None, ): super().__init__(f"split({n}, equal={equal})", [input_op]) self._equal = equal @@ -40,6 +45,40 @@ def __init__( # The number of rows output to each output split so far. self._num_output: List[int] = [0 for _ in range(n)] + if locality_hints is not None: + if n != len(locality_hints): + raise ValueError( + "Locality hints list must have length `n`: " + f"len({locality_hints}) != {n}" + ) + self._locality_hints = locality_hints + if locality_hints: + # To optimize locality, we should buffer a certain number of elements + # internally before dispatch to allow the locality algorithm a good chance + # of selecting a preferred location. We use a small multiple of `n` since + # it's reasonable to buffer a couple blocks per consumer. + self._min_buffer_size = 2 * n + else: + self._min_buffer_size = 0 + self._locality_hits = 0 + self._locality_misses = 0 + + def start(self, options: ExecutionOptions) -> None: + super().start(options) + # Force disable locality optimization. + if not options.actor_locality_enabled: + self._locality_hints = None + self._min_buffer_size = 0 + + def throttling_disabled(self) -> bool: + """Disables resource-based throttling. + + It doesn't make sense to throttle the inputs to this operator, since all that + would do is lower the buffer size and prevent us from emitting outputs / + reduce the locality hit rate. + """ + return True + def has_next(self) -> bool: return len(self._output_queue) > 0 @@ -47,6 +86,9 @@ def get_next(self) -> RefBundle: return self._output_queue.pop() def get_stats(self) -> StatsDict: + return {"split": []} # TODO(ekl) add split metrics? + + def get_metrics(self) -> Dict[str, int]: stats = {} for i, num in enumerate(self._num_output): stats[f"num_output_{i}"] = num @@ -59,9 +101,10 @@ def add_input(self, bundle, input_index) -> None: self._dispatch_bundles() def inputs_done(self) -> None: + super().inputs_done() if not self._equal: - # There shouldn't be any buffered data if we're not in equal split mode. - assert not self._buffer + self._dispatch_bundles(dispatch_all=True) + assert not self._buffer, "Should have dispatched all bundles." return # Otherwise: @@ -97,21 +140,31 @@ def current_resource_usage(self) -> ExecutionResources: ) def progress_str(self) -> str: - if self._equal: - return f"{len(self._buffer)} buffered" - assert not self._buffer - return "" + if self._locality_hints: + return ( + f"[{self._locality_hits} locality hits, {self._locality_misses} misses]" + ) + else: + return "[locality disabled]" - def _dispatch_bundles(self) -> None: + def _dispatch_bundles(self, dispatch_all: bool = False) -> None: # Dispatch all dispatchable bundles from the internal buffer. # This may not dispatch all bundles when equal=True. - while self._buffer: + while self._buffer and ( + dispatch_all or len(self._buffer) >= self._min_buffer_size + ): target_index = self._select_output_index() target_bundle = self._pop_bundle_to_dispatch(target_index) if self._can_safely_dispatch(target_index, target_bundle.num_rows()): target_bundle.output_split_idx = target_index self._num_output[target_index] += target_bundle.num_rows() self._output_queue.append(target_bundle) + if self._locality_hints: + preferred_loc = self._locality_hints[target_index] + if self._get_location(target_bundle) == preferred_loc: + self._locality_hits += 1 + else: + self._locality_misses += 1 else: # Put it back and abort. self._buffer.insert(0, target_bundle) @@ -123,7 +176,12 @@ def _select_output_index(self) -> int: return i def _pop_bundle_to_dispatch(self, target_index: int) -> RefBundle: - # TODO implement locality aware bundle selection. + if self._locality_hints: + preferred_loc = self._locality_hints[target_index] + for bundle in self._buffer: + if self._get_location(bundle) == preferred_loc: + self._buffer.remove(bundle) + return bundle return self._buffer.pop(0) def _can_safely_dispatch(self, target_index: int, nrow: int) -> bool: @@ -160,6 +218,16 @@ def _split_from_buffer(self, nrow: int) -> List[RefBundle]: assert sum(b.num_rows() for b in output) == nrow, (acc, nrow) return output + def _get_location(self, bundle: RefBundle) -> Optional[NodeIdStr]: + """Ask Ray for the node id of the given bundle. + + This method may be overriden for testing. + + Returns: + A node id associated with the bundle, or None if unknown. + """ + return bundle.get_cached_location() + def _split(bundle: RefBundle, left_size: int) -> (RefBundle, RefBundle): left_blocks, left_meta = [], [] diff --git a/python/ray/data/_internal/execution/operators/task_pool_map_operator.py b/python/ray/data/_internal/execution/operators/task_pool_map_operator.py index abeec33e1904c..89d51d7857ac6 100644 --- a/python/ray/data/_internal/execution/operators/task_pool_map_operator.py +++ b/python/ray/data/_internal/execution/operators/task_pool_map_operator.py @@ -52,9 +52,9 @@ def _add_bundled_input(self, bundle: RefBundle): map_task = cached_remote_fn(_map_task, num_returns="dynamic") input_blocks = [block for block, _ in bundle.blocks] ctx = TaskContext(task_idx=self._next_task_idx) - ref = map_task.options(**self._ray_remote_args, name=self.name).remote( - self._transform_fn_ref, ctx, *input_blocks - ) + ref = map_task.options( + **self._get_runtime_ray_remote_args(), name=self.name + ).remote(self._transform_fn_ref, ctx, *input_blocks) self._next_task_idx += 1 task = _TaskState(bundle) self._tasks[ref] = task diff --git a/python/ray/data/_internal/execution/operators/zip_operator.py b/python/ray/data/_internal/execution/operators/zip_operator.py new file mode 100644 index 0000000000000..3238948ff3e42 --- /dev/null +++ b/python/ray/data/_internal/execution/operators/zip_operator.py @@ -0,0 +1,252 @@ +import itertools +from typing import Callable, List, Optional, Tuple + +import ray +from ray.data._internal.delegating_block_builder import DelegatingBlockBuilder +from ray.data._internal.remote_fn import cached_remote_fn +from ray.data._internal.split import _split_at_indices +from ray.data._internal.stats import StatsDict +from ray.data._internal.execution.interfaces import ( + RefBundle, + PhysicalOperator, +) +from ray.data.block import ( + Block, + BlockAccessor, + BlockExecStats, + BlockMetadata, + BlockPartition, +) + + +class ZipOperator(PhysicalOperator): + """An operator that zips its inputs together. + + NOTE: the implementation is bulk for now, which materializes all its inputs in + object store, before starting execution. Should re-implement it as a streaming + operator in the future. + """ + + def __init__( + self, + left_input_op: PhysicalOperator, + right_input_op: PhysicalOperator, + ): + """Create a ZipOperator. + + Args: + left_input_ops: The input operator at left hand side. + right_input_op: The input operator at right hand side. + """ + self._left_buffer: List[RefBundle] = [] + self._right_buffer: List[RefBundle] = [] + self._output_buffer: List[RefBundle] = [] + self._stats: StatsDict = {} + super().__init__("Zip", [left_input_op, right_input_op]) + + def num_outputs_total(self) -> Optional[int]: + left_num_outputs = self.input_dependencies[0].num_outputs_total() + right_num_outputs = self.input_dependencies[1].num_outputs_total() + if left_num_outputs is not None and right_num_outputs is not None: + return max(left_num_outputs, right_num_outputs) + elif left_num_outputs is not None: + return left_num_outputs + else: + return right_num_outputs + + def add_input(self, refs: RefBundle, input_index: int) -> None: + assert not self.completed() + assert input_index == 0 or input_index == 1, input_index + if input_index == 0: + self._left_buffer.append(refs) + else: + self._right_buffer.append(refs) + + def inputs_done(self) -> None: + self._output_buffer, self._stats = self._zip( + self._left_buffer, self._right_buffer + ) + self._left_buffer.clear() + self._right_buffer.clear() + super().inputs_done() + + def has_next(self) -> bool: + return len(self._output_buffer) > 0 + + def get_next(self) -> RefBundle: + return self._output_buffer.pop(0) + + def get_stats(self) -> StatsDict: + return self._stats + + def get_transformation_fn(self) -> Callable: + return self._zip + + def _zip( + self, left_input: List[RefBundle], right_input: List[RefBundle] + ) -> Tuple[List[RefBundle], StatsDict]: + """Zip the RefBundles from `left_input` and `right_input` together. + + Zip is done in 2 steps: aligning blocks, and zipping blocks from + both sides. + + Aligning blocks (optional): check the blocks from `left_input` and + `right_input` are aligned or not, i.e. if having different number of blocks, or + having different number of rows in some blocks. If not aligned, repartition the + smaller input with `_split_at_indices` to align with larger input. + + Zipping blocks: after blocks from both sides are aligned, zip + blocks from both sides together in parallel. + """ + left_blocks_with_metadata = [] + for bundle in left_input: + for block, meta in bundle.blocks: + left_blocks_with_metadata.append((block, meta)) + right_blocks_with_metadata = [] + for bundle in right_input: + for block, meta in bundle.blocks: + right_blocks_with_metadata.append((block, meta)) + + left_block_rows, left_block_bytes = self._calculate_blocks_rows_and_bytes( + left_blocks_with_metadata + ) + right_block_rows, right_block_bytes = self._calculate_blocks_rows_and_bytes( + right_blocks_with_metadata + ) + + # Check that both sides have the same number of rows. + # TODO(Clark): Support different number of rows via user-directed + # dropping/padding. + total_left_rows = sum(left_block_rows) + total_right_rows = sum(right_block_rows) + if total_left_rows != total_right_rows: + raise ValueError( + "Cannot zip datasets of different number of rows: " + f"{total_left_rows}, {total_right_rows}" + ) + + # Whether the left and right input sides are inverted + input_side_inverted = False + if sum(right_block_bytes) > sum(left_block_bytes): + # Make sure that right side is smaller, so we minimize splitting + # work when aligning both sides. + # TODO(Clark): Improve this heuristic for minimizing splitting work, + # e.g. by generating the splitting plans for each route (via + # _generate_per_block_split_indices) and choosing the plan that splits + # the least cumulative bytes. + left_blocks_with_metadata, right_blocks_with_metadata = ( + right_blocks_with_metadata, + left_blocks_with_metadata, + ) + left_block_rows, right_block_rows = right_block_rows, left_block_rows + input_side_inverted = True + + # Get the split indices that will align both sides. + indices = list(itertools.accumulate(left_block_rows)) + indices.pop(-1) + + # Split other at the alignment indices, such that for every block from + # left side, we have a list of blocks from right side that have the same + # cumulative number of rows as that left block. + # NOTE: _split_at_indices has a no-op fastpath if the blocks are already + # aligned. + aligned_right_blocks_with_metadata = _split_at_indices( + right_blocks_with_metadata, + indices, + block_rows=right_block_rows, + ) + del right_blocks_with_metadata + + left_blocks = [b for b, _ in left_blocks_with_metadata] + right_blocks_list = aligned_right_blocks_with_metadata[0] + del left_blocks_with_metadata, aligned_right_blocks_with_metadata + + zip_one_block = cached_remote_fn(_zip_one_block, num_returns=2) + + output_blocks = [] + output_metadata = [] + for left_block, right_blocks in zip(left_blocks, right_blocks_list): + # For each block from left side, zip it together with 1 or more blocks from + # right side. We're guaranteed to have that left_block has the same number + # of rows as right_blocks has cumulatively. + res, meta = zip_one_block.remote( + left_block, *right_blocks, inverted=input_side_inverted + ) + output_blocks.append(res) + output_metadata.append(meta) + + # Early release memory. + del left_blocks, right_blocks_list + + # TODO(ekl) it might be nice to have a progress bar here. + output_metadata = ray.get(output_metadata) + output_refs = [] + input_owned = all(b.owns_blocks for b in left_input) + for block, meta in zip(output_blocks, output_metadata): + output_refs.append( + RefBundle( + [ + ( + block, + meta, + ) + ], + owns_blocks=input_owned, + ) + ) + stats = {self._name: output_metadata} + + # Clean up inputs. + for ref in left_input: + ref.destroy_if_owned() + for ref in right_input: + ref.destroy_if_owned() + + return output_refs, stats + + def _calculate_blocks_rows_and_bytes( + self, + blocks_with_metadata: BlockPartition, + ) -> Tuple[List[int], List[int]]: + """Calculate the number of rows and size in bytes for a list of blocks with + metadata. + """ + get_num_rows_and_bytes = cached_remote_fn(_get_num_rows_and_bytes) + block_rows = [] + block_bytes = [] + for block, metadata in blocks_with_metadata: + if metadata.num_rows is None or metadata.size_bytes is None: + # Need to fetch number of rows or size in bytes, so just fetch both. + num_rows, size_bytes = ray.get(get_num_rows_and_bytes.remote(block)) + # Cache on the block metadata. + metadata.num_rows = num_rows + metadata.size_bytes = size_bytes + block_rows.append(metadata.num_rows) + block_bytes.append(metadata.size_bytes) + return block_rows, block_bytes + + +def _zip_one_block( + block: Block, *other_blocks: Block, inverted: bool = False +) -> Tuple[Block, BlockMetadata]: + """Zip together `block` with `other_blocks`.""" + stats = BlockExecStats.builder() + # Concatenate other blocks. + # TODO(Clark): Extend BlockAccessor.zip() to work with N other blocks, + # so we don't need to do this concatenation. + builder = DelegatingBlockBuilder() + for other_block in other_blocks: + builder.add_block(other_block) + other_block = builder.build() + if inverted: + # Swap blocks if ordering was inverted during block alignment splitting. + block, other_block = other_block, block + # Zip block and other blocks. + result = BlockAccessor.for_block(block).zip(other_block) + br = BlockAccessor.for_block(result) + return result, br.get_metadata(input_files=[], exec_stats=stats.build()) + + +def _get_num_rows_and_bytes(block: Block) -> Tuple[int, int]: + block = BlockAccessor.for_block(block) + return block.num_rows(), block.size_bytes() diff --git a/python/ray/data/_internal/execution/streaming_executor.py b/python/ray/data/_internal/execution/streaming_executor.py index b22f14c435fe7..68b587fb8ceeb 100644 --- a/python/ray/data/_internal/execution/streaming_executor.py +++ b/python/ray/data/_internal/execution/streaming_executor.py @@ -10,6 +10,7 @@ Executor, ExecutionOptions, ExecutionResources, + OutputIterator, RefBundle, PhysicalOperator, ) @@ -83,18 +84,30 @@ def execute( self._output_node: OpState = self._topology[dag] self.start() - # Drain items from the runner thread until completion. - try: - item = self._output_node.get_output_blocking() - while item is not None: - if isinstance(item, Exception): - raise item - else: - self._output_info.update(1) - yield item - item = self._output_node.get_output_blocking() - finally: - self.shutdown() + class StreamIterator(OutputIterator): + def __init__(self, outer: Executor): + self._outer = outer + + def get_next(self, output_split_idx: Optional[int] = None) -> RefBundle: + try: + item = self._outer._output_node.get_output_blocking( + output_split_idx + ) + # Translate the special sentinel values for MaybeRefBundle into + # exceptions. + if item is None: + raise StopIteration + elif isinstance(item, Exception): + raise item + else: + # Otherwise return a concrete RefBundle. + self._outer._output_info.update(1) + return item + except Exception: + self._outer.shutdown() + raise + + return StreamIterator(self) def shutdown(self): with self._shutdown_lock: diff --git a/python/ray/data/_internal/execution/streaming_executor_state.py b/python/ray/data/_internal/execution/streaming_executor_state.py index ed43f5f5d346b..34530ed145a66 100644 --- a/python/ray/data/_internal/execution/streaming_executor_state.py +++ b/python/ray/data/_internal/execution/streaming_executor_state.py @@ -144,7 +144,7 @@ def dispatch_next_task(self) -> None: return assert False, "Nothing to dispatch" - def get_output_blocking(self) -> MaybeRefBundle: + def get_output_blocking(self, output_split_idx: Optional[int]) -> MaybeRefBundle: """Get an item from this node's output queue, blocking as needed. Returns: @@ -152,9 +152,27 @@ def get_output_blocking(self) -> MaybeRefBundle: """ while True: try: - return self.outqueue.popleft() + # Non-split output case. + if output_split_idx is None: + return self.outqueue.popleft() + + # Scan the queue and look for outputs tagged for the given index. + for i in range(len(self.outqueue)): + bundle = self.outqueue[i] + if bundle is None or isinstance(bundle, Exception): + # End of stream for this index! Note that we + # do not remove the None, so that it can act + # as the termination signal for all indices. + return bundle + elif bundle.output_split_idx == output_split_idx: + self.outqueue.remove(bundle) + return bundle + + # Didn't find any outputs matching this index, repeat the loop until + # we find one or hit a None. except IndexError: - time.sleep(0.01) + pass + time.sleep(0.01) def outqueue_memory_usage(self) -> int: """Return the object store memory of this operator's outqueue. @@ -306,9 +324,14 @@ def select_operator_to_run( if not ops: return None - # Equally penalize outqueue length and num bundles processing for backpressure. + # Run metadata-only operators first. After that, equally penalize outqueue length + # and num bundles processing for backpressure. return min( - ops, key=lambda op: len(topology[op].outqueue) + topology[op].num_processing() + ops, + key=lambda op: ( + not op.throttling_disabled(), + len(topology[op].outqueue) + topology[op].num_processing(), + ), ) @@ -335,6 +358,10 @@ def _execution_allowed( Returns: Whether the op is allowed to run. """ + + if op.throttling_disabled(): + return True + assert isinstance(global_usage, TopologyResourceUsage), global_usage # To avoid starvation problems when dealing with fractional resource types, # convert all quantities to integer (0 or 1) for deciding admissibility. This diff --git a/python/ray/data/_internal/logical/operators/map_operator.py b/python/ray/data/_internal/logical/operators/map_operator.py index 413cfabbe6457..7fad1807ba22f 100644 --- a/python/ray/data/_internal/logical/operators/map_operator.py +++ b/python/ray/data/_internal/logical/operators/map_operator.py @@ -8,7 +8,6 @@ ) from ray.data.block import BatchUDF, RowUDF from ray.data.context import DEFAULT_BATCH_SIZE -from ray.data.datasource import Datasource if sys.version_info >= (3, 8): @@ -142,26 +141,6 @@ def __init__( ) -class Write(AbstractUDFMap): - """Logical operator for write.""" - - def __init__( - self, - input_op: LogicalOperator, - datasource: Datasource, - ray_remote_args: Optional[Dict[str, Any]] = None, - **write_args, - ): - super().__init__( - "Write", - input_op, - fn=lambda x: x, - ray_remote_args=ray_remote_args, - ) - self._datasource = datasource - self._write_args = write_args - - class Filter(AbstractUDFMap): """Logical operator for filter.""" diff --git a/python/ray/data/_internal/logical/operators/n_ary_operator.py b/python/ray/data/_internal/logical/operators/n_ary_operator.py new file mode 100644 index 0000000000000..145979f5ee3b5 --- /dev/null +++ b/python/ray/data/_internal/logical/operators/n_ary_operator.py @@ -0,0 +1,17 @@ +from ray.data._internal.logical.interfaces import LogicalOperator + + +class Zip(LogicalOperator): + """Logical operator for zip.""" + + def __init__( + self, + left_input_op: LogicalOperator, + right_input_op: LogicalOperator, + ): + """ + Args: + left_input_ops: The input operator at left hand side. + right_input_op: The input operator at right hand side. + """ + super().__init__("Zip", [left_input_op, right_input_op]) diff --git a/python/ray/data/_internal/logical/optimizers.py b/python/ray/data/_internal/logical/optimizers.py index f1c0d83ff16bb..5d37da3952e0d 100644 --- a/python/ray/data/_internal/logical/optimizers.py +++ b/python/ray/data/_internal/logical/optimizers.py @@ -10,6 +10,7 @@ OperatorFusionRule, ReorderRandomizeBlocksRule, ) +from ray.data._internal.logical.util import record_operators_usage from ray.data._internal.planner.planner import Planner @@ -37,6 +38,8 @@ def get_execution_plan(logical_plan: LogicalPlan) -> PhysicalPlan: (2) planning: convert logical to physical operators. (3) physical optimization: optimize physical operators. """ + # Record usage of logical operators. + record_operators_usage(logical_plan.dag) logical_plan = LogicalOptimizer().optimize(logical_plan) physical_plan = Planner().plan(logical_plan) diff --git a/python/ray/data/_internal/logical/util.py b/python/ray/data/_internal/logical/util.py new file mode 100644 index 0000000000000..3a32f3b6c55f4 --- /dev/null +++ b/python/ray/data/_internal/logical/util.py @@ -0,0 +1,87 @@ +from typing import Dict +import json +import threading + +from ray._private.usage.usage_lib import TagKey, record_extra_usage_tag +from ray.data._internal.logical.interfaces import LogicalOperator +from ray.data._internal.logical.operators.read_operator import Read +from ray.data._internal.logical.operators.write_operator import Write + +# The dictionary for the operator name and count. +_recorded_operators = dict() +_recorded_operators_lock = threading.Lock() + +# The white list of operator names allowed to be recorded. +_op_name_white_list = [ + # Read + "ReadRange", + "ReadMongo", + "ReadParquet", + "ReadParquetBulk", + "ReadImage", + "ReadJSON", + "ReadCSV", + "ReadText", + "ReadNumpy", + "ReadTFRecord", + "ReadBinary", + "ReadCustom", + # Write + "WriteParquet", + "WriteJSON", + "WriteCSV", + "WriteTFRecord", + "WriteNumpy", + "WriteMongo", + "WriteCustom", + # Map + "MapBatches", + "MapRows", + "Filter", + "FlatMap", + # All-to-all + "RandomizeBlocks", + "RandomShuffle", + "Repartition", + "Sort", + "Aggregate", +] + + +def record_operators_usage(op: LogicalOperator): + """Record logical operator usage with Ray telemetry.""" + ops_dict = dict() + _collect_operators_to_dict(op, ops_dict) + ops_json_str = "" + with _recorded_operators_lock: + for op, count in ops_dict.items(): + _recorded_operators.setdefault(op, 0) + _recorded_operators[op] += count + ops_json_str = json.dumps(_recorded_operators) + + record_extra_usage_tag(TagKey.DATA_LOGICAL_OPS, ops_json_str) + + +def _collect_operators_to_dict(op: LogicalOperator, ops_dict: Dict[str, int]): + """Collect the logical operator name and count into `ops_dict`.""" + for child in op.input_dependencies: + _collect_operators_to_dict(child, ops_dict) + + op_name = op.name + + # Check read and write operator, and anonymize user-defined data source. + if isinstance(op, Read): + op_name = f"Read{op._datasource.get_name()}" + if op_name not in _op_name_white_list: + op_name = "ReadCustom" + elif isinstance(op, Write): + op_name = f"Write{op._datasource.get_name()}" + if op_name not in _op_name_white_list: + op_name = "WriteCustom" + + # Anonymize any operator name if not in white list. + if op_name not in _op_name_white_list: + op_name = "Unknown" + + ops_dict.setdefault(op_name, 0) + ops_dict[op_name] += 1 diff --git a/python/ray/data/_internal/plan.py b/python/ray/data/_internal/plan.py index 7b50da8129a9b..4976a63ab89bb 100644 --- a/python/ray/data/_internal/plan.py +++ b/python/ray/data/_internal/plan.py @@ -238,6 +238,51 @@ def get_plan_as_string(self) -> str: num_blocks, count, schema_str ) + # If the resulting string representation fits in one line, use it directly. + SCHEMA_LINE_CHAR_LIMIT = 80 + MIN_FIELD_LENGTH = 10 + INDENT_STR = " " * 3 + trailing_space = " " * (max(num_stages, 0) * 3) + if len(dataset_str) > SCHEMA_LINE_CHAR_LIMIT: + # If the resulting string representation exceeds the line char limit, + # first try breaking up each `Dataset` parameter into its own line + # and check if each line fits within the line limit. We check the + # `schema` param's length, since this is likely the longest string. + schema_str_on_new_line = f"{trailing_space}{INDENT_STR}schema={schema_str}" + if len(schema_str_on_new_line) > SCHEMA_LINE_CHAR_LIMIT: + # If the schema cannot fit on a single line, break up each field + # into its own line. + schema_str = [] + for n, t in zip(schema.names, schema.types): + if hasattr(t, "__name__"): + t = t.__name__ + col_str = f"{trailing_space}{INDENT_STR * 2}{n}: {t}" + # If the field line exceeds the char limit, abbreviate + # the field name to fit while maintaining the full type + if len(col_str) > SCHEMA_LINE_CHAR_LIMIT: + shortened_suffix = f"...: {str(t)}" + # Show at least 10 characters of the field name, even if + # we have already hit the line limit with the type. + chars_left_for_col_name = max( + SCHEMA_LINE_CHAR_LIMIT - len(shortened_suffix), + MIN_FIELD_LENGTH, + ) + col_str = ( + f"{col_str[:chars_left_for_col_name]}{shortened_suffix}" + ) + schema_str.append(col_str) + schema_str = ",\n".join(schema_str) + schema_str = ( + "{\n" + schema_str + f"\n{trailing_space}{INDENT_STR}" + "}" + ) + dataset_str = ( + f"Dataset(" + f"\n{trailing_space}{INDENT_STR}num_blocks={num_blocks}," + f"\n{trailing_space}{INDENT_STR}num_rows={count}," + f"\n{trailing_space}{INDENT_STR}schema={schema_str}" + f"\n{trailing_space})" + ) + if num_stages == 0: plan_str = dataset_str else: diff --git a/python/ray/data/_internal/planner/plan_udf_map_op.py b/python/ray/data/_internal/planner/plan_udf_map_op.py index e02ab405ed10e..49a47ae76ffe8 100644 --- a/python/ray/data/_internal/planner/plan_udf_map_op.py +++ b/python/ray/data/_internal/planner/plan_udf_map_op.py @@ -14,13 +14,11 @@ FlatMap, MapBatches, MapRows, - Write, ) from ray.data._internal.planner.filter import generate_filter_fn from ray.data._internal.planner.flat_map import generate_flat_map_fn from ray.data._internal.planner.map_batches import generate_map_batches_fn from ray.data._internal.planner.map_rows import generate_map_rows_fn -from ray.data._internal.planner.write import generate_write_fn from ray.data.block import Block, CallableClass @@ -45,8 +43,6 @@ def _plan_udf_map_op( transform_fn = generate_flat_map_fn() elif isinstance(op, Filter): transform_fn = generate_filter_fn() - elif isinstance(op, Write): - transform_fn = generate_write_fn(op._datasource, **op._write_args) else: raise ValueError(f"Found unknown logical operator during planning: {op}") diff --git a/python/ray/data/_internal/planner/planner.py b/python/ray/data/_internal/planner/planner.py index af60ac1a4019b..8a27e968c614f 100644 --- a/python/ray/data/_internal/planner/planner.py +++ b/python/ray/data/_internal/planner/planner.py @@ -1,12 +1,14 @@ from typing import Dict from ray.data._internal.execution.interfaces import PhysicalOperator +from ray.data._internal.execution.operators.zip_operator import ZipOperator from ray.data._internal.logical.interfaces import ( LogicalOperator, LogicalPlan, PhysicalPlan, ) from ray.data._internal.logical.operators.all_to_all_operator import AbstractAllToAll +from ray.data._internal.logical.operators.n_ary_operator import Zip from ray.data._internal.logical.operators.read_operator import Read from ray.data._internal.logical.operators.write_operator import Write from ray.data._internal.logical.operators.map_operator import AbstractUDFMap @@ -49,6 +51,9 @@ def _plan(self, logical_op: LogicalOperator) -> PhysicalOperator: elif isinstance(logical_op, AbstractAllToAll): assert len(physical_children) == 1 physical_op = _plan_all_to_all_op(logical_op, physical_children[0]) + elif isinstance(logical_op, Zip): + assert len(physical_children) == 2 + physical_op = ZipOperator(physical_children[0], physical_children[1]) else: raise ValueError( f"Found unknown logical operator during planning: {logical_op}" diff --git a/python/ray/data/_internal/stats.py b/python/ray/data/_internal/stats.py index c4a490d56351f..270916b2c77e3 100644 --- a/python/ray/data/_internal/stats.py +++ b/python/ray/data/_internal/stats.py @@ -223,6 +223,16 @@ def __init__( self.iter_total_s: Timer = Timer() self.extra_metrics = {} + # Block fetch stats during iteration. + # These are stats about locations of blocks when the iterator is trying to + # consume them. The iteration performance will be affected depending on + # whether the block is in the local object store of the node where the + # iterator is running. + # This serves as an indicator of block prefetching effectiveness. + self.iter_blocks_local: int = 0 + self.iter_blocks_remote: int = 0 + self.iter_unknown_location: int = 0 + @property def stats_actor(self): return _get_or_create_stats_actor() @@ -279,6 +289,9 @@ def to_summary(self) -> "DatasetStatsSummary": self.iter_format_batch_s, self.iter_user_s, self.iter_total_s, + self.iter_blocks_local, + self.iter_blocks_remote, + self.iter_unknown_location, ) stats_summary_parents = [] if self.parents is not None: @@ -616,6 +629,12 @@ class IterStatsSummary: user_time: Timer # Total time taken by Dataset iterator, in seconds total_time: Timer + # Num of blocks that are in local object store + iter_blocks_local: int + # Num of blocks that are in remote node and have to fetch locally + iter_blocks_remote: int + # Num of blocks with unknown locations + iter_unknown_location: int def __str__(self) -> str: out = "" @@ -629,6 +648,11 @@ def __str__(self) -> str: out += "\nDataset iterator time breakdown:\n" out += "* In ray.wait(): {}\n".format(fmt(self.wait_time.get())) out += "* In ray.get(): {}\n".format(fmt(self.get_time.get())) + out += "* Num blocks local: {}\n".format(self.iter_blocks_local) + out += "* Num blocks remote: {}\n".format(self.iter_blocks_remote) + out += "* Num blocks unknown location: {}\n".format( + self.iter_unknown_location + ) out += "* In next_batch(): {}\n".format(fmt(self.next_time.get())) out += "* In format_batch(): {}\n".format(fmt(self.format_time.get())) out += "* In user code: {}\n".format(fmt(self.user_time.get())) diff --git a/python/ray/data/_internal/stream_split_dataset_iterator.py b/python/ray/data/_internal/stream_split_dataset_iterator.py new file mode 100644 index 0000000000000..c7249d0c4db8f --- /dev/null +++ b/python/ray/data/_internal/stream_split_dataset_iterator.py @@ -0,0 +1,199 @@ +import copy +import logging +import sys +import threading +from typing import ( + List, + Dict, + Optional, + Iterator, + Callable, + Any, + Union, + TYPE_CHECKING, +) + +import ray + +from ray.data.dataset_iterator import DatasetIterator +from ray.data.block import Block, DataBatch +from ray.data.context import DatasetContext +from ray.data._internal.execution.streaming_executor import StreamingExecutor +from ray.data._internal.execution.legacy_compat import ( + execute_to_legacy_bundle_iterator, +) +from ray.data._internal.block_batching import batch_block_refs +from ray.data._internal.execution.operators.output_splitter import OutputSplitter +from ray.data._internal.execution.interfaces import NodeIdStr, RefBundle +from ray.types import ObjectRef +from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy + +if TYPE_CHECKING: + import pyarrow + from ray.data import Dataset + +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + +logger = logging.getLogger(__name__) + + +class StreamSplitDatasetIterator(DatasetIterator): + """Implements a collection of iterators over a shared data stream.""" + + @staticmethod + def create( + base_dataset: "Dataset", + n: int, + equal: bool, + locality_hints: Optional[List[NodeIdStr]], + ) -> List["StreamSplitDatasetIterator"]: + """Create a split iterator from the given base Dataset and options. + + See also: `Dataset.streaming_split`. + """ + ctx = DatasetContext.get_current() + + # To avoid deadlock, the concurrency on this actor must be set to at least `n`. + coord_actor = SplitCoordinator.options( + max_concurrency=n, + scheduling_strategy=NodeAffinitySchedulingStrategy( + ray.get_runtime_context().get_node_id(), soft=False + ), + ).remote(ctx, base_dataset, n, equal, locality_hints) + + return [ + StreamSplitDatasetIterator(base_dataset, coord_actor, i) for i in range(n) + ] + + def __init__( + self, + base_dataset: "Dataset", + coord_actor: ray.actor.ActorHandle, + output_split_idx: int, + ): + self._base_dataset = base_dataset + self._coord_actor = coord_actor + self._output_split_idx = output_split_idx + + def iter_batches( + self, + *, + prefetch_blocks: int = 0, + batch_size: int = 256, + batch_format: Literal["default", "numpy", "pandas"] = "default", + drop_last: bool = False, + local_shuffle_buffer_size: Optional[int] = None, + local_shuffle_seed: Optional[int] = None, + _collate_fn: Optional[Callable[[DataBatch], Any]] = None, + ) -> Iterator[DataBatch]: + """Implements DatasetIterator.""" + + def gen_blocks() -> Iterator[ObjectRef[Block]]: + future: ObjectRef[ + Optional[ObjectRef[Block]] + ] = self._coord_actor.get.remote(self._output_split_idx) + while True: + block_ref: Optional[ObjectRef[Block]] = ray.get(future) + if not block_ref: + break + else: + future = self._coord_actor.get.remote(self._output_split_idx) + yield block_ref + + yield from batch_block_refs( + gen_blocks(), + stats=None, + prefetch_blocks=prefetch_blocks, + batch_size=batch_size, + batch_format=batch_format, + drop_last=drop_last, + collate_fn=_collate_fn, + shuffle_buffer_min_size=local_shuffle_buffer_size, + shuffle_seed=local_shuffle_seed, + ) + + def stats(self) -> str: + """Implements DatasetIterator.""" + return self._base_dataset.stats() + + def schema(self) -> Union[type, "pyarrow.lib.Schema"]: + """Implements DatasetIterator.""" + return self._base_dataset.schema() + + +@ray.remote(num_cpus=0) +class SplitCoordinator: + """Coordinator actor for routing blocks to output splits. + + This actor runs a streaming executor locally on its main thread. Clients can + retrieve results via actor calls running on other threads. + """ + + def __init__( + self, + ctx: DatasetContext, + dataset: "Dataset", + n: int, + equal: bool, + locality_hints: Optional[List[NodeIdStr]], + ): + # Automatically set locality with output to the specified location hints. + if locality_hints: + ctx.execution_options.locality_with_output = locality_hints + logger.info(f"Auto configuring locality_with_output={locality_hints}") + + DatasetContext._set_current(ctx) + self._base_dataset = dataset + self._n = n + self._equal = equal + self._locality_hints = locality_hints + self._finished = False + self._lock = threading.RLock() + # Guarded by self._lock. + self._next_bundle: Dict[int, RefBundle] = {} + + executor = StreamingExecutor(copy.deepcopy(ctx.execution_options)) + + def add_split_op(dag): + return OutputSplitter(dag, n, equal, locality_hints) + + self._output_iterator = execute_to_legacy_bundle_iterator( + executor, + dataset._plan, + True, + dataset._plan._dataset_uuid, + dag_rewrite=add_split_op, + ) + + def get(self, output_split_idx: int) -> Optional[ObjectRef[Block]]: + """Blocking get operation. + + This is intended to be called concurrently from multiple clients. + """ + try: + # Ensure there is at least one bundle. + with self._lock: + if output_split_idx in self._next_bundle: + next_bundle = self._next_bundle[output_split_idx] + else: + next_bundle = None + + # Fetch next bundle if needed. + if next_bundle is None: + # This is a BLOCKING call, so do it outside the lock. + next_bundle = self._output_iterator.get_next(output_split_idx) + + block = next_bundle.blocks.pop()[0] + + # Accumulate any remaining blocks in next_bundle map as needed. + with self._lock: + self._next_bundle[output_split_idx] = next_bundle + if not next_bundle.blocks: + del self._next_bundle[output_split_idx] + + return block + except StopIteration: + return None diff --git a/python/ray/data/_internal/util.py b/python/ray/data/_internal/util.py index f947e44fd763f..8274423deaff8 100644 --- a/python/ray/data/_internal/util.py +++ b/python/ray/data/_internal/util.py @@ -343,7 +343,7 @@ def _consumption_api( """ base = ( " will trigger execution of the lazy transformations performed on " - "this dataset, and will block until execution completes." + "this dataset." ) if delegate: message = delegate + base diff --git a/python/ray/data/dataset.py b/python/ray/data/dataset.py index d29fe31169938..58314b05d4ce9 100644 --- a/python/ray/data/dataset.py +++ b/python/ray/data/dataset.py @@ -35,6 +35,7 @@ Repartition, Sort, ) +from ray.data._internal.logical.operators.n_ary_operator import Zip from ray.data._internal.logical.optimizers import LogicalPlan from ray.data._internal.logical.operators.map_operator import ( Filter, @@ -50,7 +51,8 @@ from ray.data._internal.planner.write import generate_write_fn from ray.data.dataset_iterator import DatasetIterator from ray.data._internal.block_list import BlockList -from ray.data._internal.bulk_dataset_iterator import BulkDatasetIterator +from ray.data._internal.dataset_iterator_impl import DatasetIteratorImpl +from ray.data._internal.stream_split_dataset_iterator import StreamSplitDatasetIterator from ray.data._internal.compute import ( ActorPoolStrategy, CallableClass, @@ -146,7 +148,7 @@ from ray.data.dataset_pipeline import DatasetPipeline from ray.data.grouped_dataset import GroupedDataset - from ray.data._internal.execution.interfaces import Executor + from ray.data._internal.execution.interfaces import Executor, NodeIdStr from ray.data._internal.torch_iterable_dataset import TorchTensorBatchType @@ -286,7 +288,7 @@ def map( ... [{"value": i} for i in range(1000)]) >>> ds.map(lambda record: {"v2": record["value"] * 2}) Map - +- Dataset(num_blocks=..., num_rows=1000, schema={value: int64}) + +- Dataset(num_blocks=200, num_rows=1000, schema={value: int64}) >>> # Define a callable class that persists state across >>> # function invocations for efficiency. >>> init_model = ... # doctest: +SKIP @@ -801,7 +803,11 @@ def select_columns( >>> ds = ds.select_columns(cols=["col1", "col2"]) >>> ds MapBatches() - +- Dataset(num_blocks=10, num_rows=10, schema={col1: int64, col2: int64, col3: int64}) + +- Dataset( + num_blocks=10, + num_rows=10, + schema={col1: int64, col2: int64, col3: int64} + ) Time complexity: O(dataset size / parallelism) @@ -1140,6 +1146,44 @@ def process_batch(batch): return self.map_batches(process_batch) + @ConsumptionAPI + def streaming_split( + self, + n: int, + *, + equal: bool = False, + locality_hints: Optional[List["NodeIdStr"]] = None, + ) -> List[DatasetIterator]: + """Returns ``n`` :class:`~ray.data.DatasetIterator`s that can be used to read + disjoint subsets of the dataset in parallel. + + This method is the recommended way to consume Datasets from multiple processes + (e.g., for distributed training). It requires streaming execution mode. + + The returned iterators are Ray-serializable and can be freely passed to any + Ray task or actor. + + Examples: + >>> import ray + >>> ds = ray.data.range(1000000) + >>> it1, it2 = ds.streaming_split(2, equal=True) + >>> list(it1.iter_batches()) # doctest: +SKIP + >>> list(it2.iter_batches()) # doctest: +SKIP + + Args: + n: Number of output iterators to return. + equal: If True, each output iterator will see an exactly equal number + of rows, dropping data if necessary. If False, some iterators may see + slightly more or less rows than other, but no data will be dropped. + locality_hints: Specify the node ids corresponding to each iterator + location. Datasets will try to minimize data movement based on the + iterator output locations. This list must have length ``n``. + + Returns: + The output iterator splits. + """ + return StreamSplitDatasetIterator.create(self, n, equal, locality_hints) + @ConsumptionAPI def split( self, n: int, *, equal: bool = False, locality_hints: Optional[List[Any]] = None @@ -1160,7 +1204,8 @@ def split( Time complexity: O(1) - See also: ``Dataset.split_at_indices``, ``Dataset.split_proportionately`` + See also: ``Dataset.split_at_indices``, ``Dataset.split_proportionately``, + and ``Dataset.streaming_split``. Args: n: Number of child datasets to return. @@ -1361,7 +1406,8 @@ def split_at_indices(self, indices: List[int]) -> List["Dataset[T]"]: Time complexity: O(num splits) - See also: ``Dataset.split``, ``Dataset.split_proportionately`` + See also: ``Dataset.split_at_indices``, ``Dataset.split_proportionately``, + and ``Dataset.streaming_split``. Args: indices: List of sorted integers which indicate where the dataset @@ -2029,7 +2075,7 @@ def sort( ... [{"value": i} for i in range(1000)]) >>> ds.sort("value", descending=True) Sort - +- Dataset(num_blocks=..., num_rows=1000, schema={value: int64}) + +- Dataset(num_blocks=200, num_rows=1000, schema={value: int64}) >>> # Sort by a key function. >>> ds.sort(lambda record: record["value"]) # doctest: +SKIP @@ -2097,7 +2143,13 @@ def zip(self, other: "Dataset[U]") -> "Dataset[(T, U)]": """ plan = self._plan.with_stage(ZipStage(other)) - return Dataset(plan, self._epoch, self._lazy) + + logical_plan = self._logical_plan + other_logical_plan = other._logical_plan + if logical_plan is not None and other_logical_plan is not None: + op = Zip(logical_plan.dag, other_logical_plan.dag) + logical_plan = LogicalPlan(op) + return Dataset(plan, self._epoch, self._lazy, logical_plan) @ConsumptionAPI def limit(self, limit: int) -> "Dataset[T]": @@ -2819,7 +2871,7 @@ def iterator(self) -> DatasetIterator: It is recommended to use ``DatasetIterator`` methods over directly calling methods such as ``iter_batches()``. """ - return BulkDatasetIterator(self) + return DatasetIteratorImpl(self) @ConsumptionAPI def iter_rows(self, *, prefetch_blocks: int = 0) -> Iterator[Union[T, TableRow]]: @@ -3221,7 +3273,17 @@ def to_tf( >>> import ray >>> ds = ray.data.read_csv("s3://anonymous@air-example-data/iris.csv") >>> ds - Dataset(num_blocks=1, num_rows=150, schema={sepal length (cm): double, sepal width (cm): double, petal length (cm): double, petal width (cm): double, target: int64}) + Dataset( + num_blocks=1, + num_rows=150, + schema={ + sepal length (cm): double, + sepal width (cm): double, + petal length (cm): double, + petal width (cm): double, + target: int64 + } + ) If your model accepts a single tensor as input, specify a single feature column. @@ -3242,7 +3304,17 @@ def to_tf( >>> ds = preprocessor.transform(ds) >>> ds Concatenator - +- Dataset(num_blocks=1, num_rows=150, schema={sepal length (cm): double, sepal width (cm): double, petal length (cm): double, petal width (cm): double, target: int64}) + +- Dataset( + num_blocks=1, + num_rows=150, + schema={ + sepal length (cm): double, + sepal width (cm): double, + petal length (cm): double, + petal width (cm): double, + target: int64 + } + ) >>> ds.to_tf("features", "target") # doctest: +SKIP <_OptionsDataset element_spec=(TensorSpec(shape=(None, 4), dtype=tf.float64, name='features'), TensorSpec(shape=(None,), dtype=tf.int64, name='target'))> diff --git a/python/ray/data/dataset_iterator.py b/python/ray/data/dataset_iterator.py index 47397fc728eb5..ea000b3f713e0 100644 --- a/python/ray/data/dataset_iterator.py +++ b/python/ray/data/dataset_iterator.py @@ -497,7 +497,17 @@ def to_tf( ... "s3://anonymous@air-example-data/iris.csv" ... ) >>> it = ds.iterator(); it - DatasetIterator(Dataset(num_blocks=1, num_rows=150, schema={sepal length (cm): double, sepal width (cm): double, petal length (cm): double, petal width (cm): double, target: int64})) + DatasetIterator(Dataset( + num_blocks=1, + num_rows=150, + schema={ + sepal length (cm): double, + sepal width (cm): double, + petal length (cm): double, + petal width (cm): double, + target: int64 + } + )) If your model accepts a single tensor as input, specify a single feature column. @@ -518,7 +528,17 @@ def to_tf( >>> it = preprocessor.transform(ds).iterator() >>> it DatasetIterator(Concatenator - +- Dataset(num_blocks=1, num_rows=150, schema={sepal length (cm): double, sepal width (cm): double, petal length (cm): double, petal width (cm): double, target: int64})) + +- Dataset( + num_blocks=1, + num_rows=150, + schema={ + sepal length (cm): double, + sepal width (cm): double, + petal length (cm): double, + petal width (cm): double, + target: int64 + } + )) >>> it.to_tf("features", "target") # doctest: +SKIP <_OptionsDataset element_spec=(TensorSpec(shape=(None, 4), dtype=tf.float64, name='features'), TensorSpec(shape=(None,), dtype=tf.int64, name='target'))> diff --git a/python/ray/data/datasource/file_based_datasource.py b/python/ray/data/datasource/file_based_datasource.py index 2badee7158f6f..1d13ad2016218 100644 --- a/python/ray/data/datasource/file_based_datasource.py +++ b/python/ray/data/datasource/file_based_datasource.py @@ -1,3 +1,4 @@ +import itertools import logging import pathlib import posixpath @@ -13,12 +14,17 @@ Optional, Tuple, Union, + TypeVar, ) +import numpy as np + from ray.data._internal.arrow_block import ArrowRow from ray.data._internal.delegating_block_builder import DelegatingBlockBuilder from ray.data._internal.execution.interfaces import TaskContext from ray.data._internal.output_buffer import BlockOutputBuffer +from ray.data._internal.progress_bar import ProgressBar +from ray.data._internal.remote_fn import cached_remote_fn from ray.data._internal.util import _check_pyarrow_version, _resolve_custom_scheme from ray.data.block import Block, BlockAccessor from ray.data.context import DatasetContext @@ -45,6 +51,13 @@ logger = logging.getLogger(__name__) +# We should parallelize file size fetch operations beyond this threshold. +FILE_SIZE_FETCH_PARALLELIZATION_THRESHOLD = 16 + +# 16 file size fetches from S3 takes ~1.5 seconds with Arrow's S3FileSystem. +PATHS_PER_FILE_SIZE_FETCH_TASK = 16 + + @DeveloperAPI class BlockWritePathProvider: """Abstract callable that provides concrete output paths when writing @@ -288,9 +301,7 @@ def write( def write_block(write_path: str, block: Block): logger.debug(f"Writing {write_path} file.") - fs = filesystem - if isinstance(fs, _S3FileSystemWrapper): - fs = fs.unwrap() + fs = _unwrap_s3_serialization_workaround(filesystem) if _block_udf is not None: block = _block_udf(block) @@ -373,8 +384,9 @@ def __init__( self._block_udf = _block_udf self._reader_args = reader_args paths, self._filesystem = _resolve_paths_and_filesystem(paths, filesystem) - self._paths, self._file_sizes = meta_provider.expand_paths( - paths, self._filesystem + self._paths, self._file_sizes = map( + list, + zip(*meta_provider.expand_paths(paths, self._filesystem, partitioning)), ) if self._partition_filter is not None: # Use partition filter to skip files which are not needed. @@ -418,8 +430,7 @@ def read_files( fs: Union["pyarrow.fs.FileSystem", _S3FileSystemWrapper], ) -> Iterable[Block]: logger.debug(f"Reading {len(read_paths)} files.") - if isinstance(fs, _S3FileSystemWrapper): - fs = fs.unwrap() + fs = _unwrap_s3_serialization_workaround(filesystem) ctx = DatasetContext.get_current() output_buffer = BlockOutputBuffer( block_udf=_block_udf, target_max_block_size=ctx.target_max_block_size @@ -672,48 +683,6 @@ def _resolve_paths_and_filesystem( return resolved_paths, filesystem -def _expand_directory( - path: str, - filesystem: "pyarrow.fs.FileSystem", - exclude_prefixes: Optional[List[str]] = None, -) -> List[str]: - """ - Expand the provided directory path to a list of file paths. - - Args: - path: The directory path to expand. - filesystem: The filesystem implementation that should be used for - reading these files. - exclude_prefixes: The file relative path prefixes that should be - excluded from the returned file set. Default excluded prefixes are - "." and "_". - - Returns: - A list of file paths contained in the provided directory. - """ - if exclude_prefixes is None: - exclude_prefixes = [".", "_"] - - from pyarrow.fs import FileSelector - - selector = FileSelector(path, recursive=True) - files = filesystem.get_file_info(selector) - base_path = selector.base_dir - filtered_paths = [] - for file_ in files: - if not file_.is_file: - continue - file_path = file_.path - if not file_path.startswith(base_path): - continue - relative = file_path[len(base_path) :] - if any(relative.startswith(prefix) for prefix in exclude_prefixes): - continue - filtered_paths.append((file_path, file_)) - # We sort the paths to guarantee a stable order. - return zip(*sorted(filtered_paths, key=lambda x: x[0])) - - def _is_url(path) -> bool: return urllib.parse.urlparse(path).scheme != "" @@ -752,6 +721,15 @@ def _wrap_s3_serialization_workaround(filesystem: "pyarrow.fs.FileSystem"): return filesystem +def _unwrap_s3_serialization_workaround( + filesystem: Union["pyarrow.fs.FileSystem", "_S3FileSystemWrapper"] +): + if isinstance(filesystem, _S3FileSystemWrapper): + return filesystem.unwrap() + else: + return filesystem + + class _S3FileSystemWrapper: def __init__(self, fs: "pyarrow.fs.S3FileSystem"): self._fs = fs @@ -792,3 +770,31 @@ def _resolve_kwargs( kwarg_overrides = kwargs_fn() kwargs.update(kwarg_overrides) return kwargs + + +Uri = TypeVar("Uri") +Meta = TypeVar("Meta") + + +def _fetch_metadata_parallel( + uris: List[Uri], + fetch_func: Callable[[List[Uri]], List[Meta]], + desired_uris_per_task: int, + **ray_remote_args, +) -> Iterator[Meta]: + """Fetch file metadata in parallel using Ray tasks.""" + remote_fetch_func = cached_remote_fn(fetch_func, num_cpus=0.5) + if ray_remote_args: + remote_fetch_func = remote_fetch_func.options(**ray_remote_args) + # Choose a parallelism that results in a # of metadata fetches per task that + # dominates the Ray task overhead while ensuring good parallelism. + # Always launch at least 2 parallel fetch tasks. + parallelism = max(len(uris) // desired_uris_per_task, 2) + metadata_fetch_bar = ProgressBar("Metadata Fetch Progress", total=parallelism) + fetch_tasks = [] + for uri_chunk in np.array_split(uris, parallelism): + if len(uri_chunk) == 0: + continue + fetch_tasks.append(remote_fetch_func.remote(uri_chunk)) + results = metadata_fetch_bar.fetch_until_complete(fetch_tasks) + yield from itertools.chain.from_iterable(results) diff --git a/python/ray/data/datasource/file_meta_provider.py b/python/ray/data/datasource/file_meta_provider.py index f1fc90027b0b8..68a6bc14f1170 100644 --- a/python/ray/data/datasource/file_meta_provider.py +++ b/python/ray/data/datasource/file_meta_provider.py @@ -1,18 +1,23 @@ +import itertools import logging +import pathlib +import os import re from typing import ( List, Optional, Union, - TYPE_CHECKING, + Iterator, Tuple, Any, + TYPE_CHECKING, ) if TYPE_CHECKING: import pyarrow from ray.data.block import BlockMetadata +from ray.data.datasource.partitioning import Partitioning from ray.util.annotations import DeveloperAPI logger = logging.getLogger(__name__) @@ -97,7 +102,8 @@ def expand_paths( self, paths: List[str], filesystem: Optional["pyarrow.fs.FileSystem"], - ) -> Tuple[List[str], List[Optional[int]]]: + partitioning: Optional[Partitioning] = None, + ) -> Iterator[Tuple[str, int]]: """Expands all paths into concrete file paths by walking directories. Also returns a sidecar of file sizes. @@ -112,11 +118,9 @@ def expand_paths( expanding all paths and reading their files. Returns: - A tuple whose first item contains the list of file paths discovered, - and whose second item contains the size of each file. `None` may be - returned if a file size is either unknown or will be fetched later - by `_get_block_metadata()`, but the length of both lists must be - equal. + An iterator of (file_path, file_size) pairs. None may be returned for the + file size if it is either unknown or will be fetched later by + `_get_block_metadata()`, but the length of both lists must be equal. """ raise NotImplementedError @@ -154,10 +158,8 @@ def expand_paths( self, paths: List[str], filesystem: "pyarrow.fs.FileSystem", - ) -> Tuple[List[str], List[Optional[int]]]: - from pyarrow.fs import FileType - from ray.data.datasource.file_based_datasource import _expand_directory - + partitioning: Optional[Partitioning] = None, + ) -> Iterator[Tuple[str, int]]: if len(paths) > 1: logger.warning( f"Expanding {len(paths)} path(s). This may be a HIGH LATENCY " @@ -165,24 +167,8 @@ def expand_paths( f"all point to files and never directories, try rerunning this read " f"with `meta_provider=FastFileMetadataProvider()`." ) - expanded_paths = [] - file_infos = [] - for path in paths: - try: - file_info = filesystem.get_file_info(path) - except OSError as e: - _handle_read_os_error(e, path) - if file_info.type == FileType.Directory: - paths, file_infos_ = _expand_directory(path, filesystem) - expanded_paths.extend(paths) - file_infos.extend(file_infos_) - elif file_info.type == FileType.File: - expanded_paths.append(path) - file_infos.append(file_info) - else: - raise FileNotFoundError(path) - file_sizes = [file_info.size for file_info in file_infos] - return expanded_paths, file_sizes + + yield from _expand_paths(paths, filesystem, partitioning) @DeveloperAPI @@ -201,15 +187,15 @@ def expand_paths( self, paths: List[str], filesystem: "pyarrow.fs.FileSystem", - ) -> Tuple[List[str], List[Optional[int]]]: + partitioning: Optional[Partitioning] = None, + ) -> Iterator[Tuple[str, int]]: logger.warning( f"Skipping expansion of {len(paths)} path(s). If your paths contain " f"directories or if file size collection is required, try rerunning this " f"read with `meta_provider=DefaultFileMetadataProvider()`." ) - import numpy as np - return paths, np.empty(len(paths), dtype=object) + yield from zip(paths, itertools.repeat(None, len(paths))) @DeveloperAPI @@ -322,12 +308,25 @@ def prefetch_file_metadata( ) -> Optional[List["pyarrow.parquet.FileMetaData"]]: from ray.data.datasource.parquet_datasource import ( PARALLELIZE_META_FETCH_THRESHOLD, - _fetch_metadata_remotely, + PIECES_PER_META_FETCH, + _SerializedPiece, + _fetch_metadata_serialization_wrapper, _fetch_metadata, ) + from ray.data.datasource.file_based_datasource import _fetch_metadata_parallel if len(pieces) > PARALLELIZE_META_FETCH_THRESHOLD: - return _fetch_metadata_remotely(pieces, **ray_remote_args) + # Wrap Parquet fragments in serialization workaround. + pieces = [_SerializedPiece(piece) for piece in pieces] + # Fetch Parquet metadata in parallel using Ray tasks. + return list( + _fetch_metadata_parallel( + pieces, + _fetch_metadata_serialization_wrapper, + PIECES_PER_META_FETCH, + **ray_remote_args, + ) + ) else: return _fetch_metadata(pieces) @@ -365,3 +364,161 @@ def _handle_read_os_error(error: OSError, paths: Union[str, List[str]]) -> str: ) else: raise error + + +def _expand_paths( + paths: List[str], + filesystem: "pyarrow.fs.FileSystem", + partitioning: Optional[Partitioning], +) -> Iterator[Tuple[str, int]]: + """Get the file sizes for all provided file paths.""" + from pyarrow.fs import LocalFileSystem + from ray.data.datasource.file_based_datasource import ( + FILE_SIZE_FETCH_PARALLELIZATION_THRESHOLD, + _unwrap_protocol, + ) + + # We break down our processing paths into a few key cases: + # 1. If len(paths) < threshold, fetch the file info for the individual files/paths + # serially. + # 2. If all paths are contained under the same parent directory (or base directory, + # if using partitioning), fetch all file infos at this prefix and filter to the + # provided paths on the client; this should be a single file info request. + # 3. If more than threshold requests required, parallelize them via Ray tasks. + + # 1. Small # of paths case. + if ( + len(paths) < FILE_SIZE_FETCH_PARALLELIZATION_THRESHOLD + # Local file systems are very fast to hit. + or isinstance(filesystem, LocalFileSystem) + ): + yield from _get_file_infos_serial(paths, filesystem) + else: + # 2. Common path prefix case. + # Get longest common path of all paths. + common_path = os.path.commonpath(paths) + # If parent directory (or base directory, if using partitioning) is common to + # all paths, fetch all file infos at that prefix and filter the response to the + # provided paths. + if ( + partitioning is not None + and common_path == _unwrap_protocol(partitioning.base_dir) + ) or all(str(pathlib.Path(path).parent) == common_path for path in paths): + yield from _get_file_infos_common_path_prefix( + paths, common_path, filesystem + ) + # 3. Parallelization case. + else: + # Parallelize requests via Ray tasks. + yield from _get_file_infos_parallel(paths, filesystem) + + +def _get_file_infos_serial( + paths: List[str], + filesystem: "pyarrow.fs.FileSystem", +) -> Iterator[Tuple[str, int]]: + for path in paths: + yield from _get_file_infos(path, filesystem) + + +def _get_file_infos_common_path_prefix( + paths: List[str], + common_path: str, + filesystem: "pyarrow.fs.FileSystem", +) -> Iterator[Tuple[str, int]]: + path_to_size = {path: None for path in paths} + for path, file_size in _get_file_infos(common_path, filesystem): + if path in path_to_size: + path_to_size[path] = file_size + # Dictionaries are insertion-ordered, so this path + size pairs should be + # yielded in the order of the original paths arg. + for path, size in path_to_size.items(): + assert size is not None + yield path, size + + +def _get_file_infos_parallel( + paths: List[str], + filesystem: "pyarrow.fs.FileSystem", +) -> Iterator[Tuple[str, int]]: + from ray.data.datasource.file_based_datasource import ( + PATHS_PER_FILE_SIZE_FETCH_TASK, + _wrap_s3_serialization_workaround, + _unwrap_s3_serialization_workaround, + _fetch_metadata_parallel, + ) + + # Capture the filesystem in the fetcher func closure, but wrap it in our + # serialization workaround to make sure that the pickle roundtrip works as expected. + filesystem = _wrap_s3_serialization_workaround(filesystem) + + def _file_infos_fetcher(paths: List[str]) -> List[Tuple[str, int]]: + fs = _unwrap_s3_serialization_workaround(filesystem) + return list( + itertools.chain.from_iterable(_get_file_infos(path, fs) for path in paths) + ) + + yield from _fetch_metadata_parallel( + paths, _file_infos_fetcher, PATHS_PER_FILE_SIZE_FETCH_TASK + ) + + +def _get_file_infos( + path: str, + filesystem: "pyarrow.fs.FileSystem", +) -> Iterator[Tuple[str, int]]: + """Get the file info for all files at or under the provided path.""" + from pyarrow.fs import FileType + + try: + file_info = filesystem.get_file_info(path) + except OSError as e: + _handle_read_os_error(e, path) + if file_info.type == FileType.Directory: + yield from _expand_directory(path, filesystem) + elif file_info.type == FileType.File: + yield path, file_info.size + else: + raise FileNotFoundError(path) + + +def _expand_directory( + path: str, + filesystem: "pyarrow.fs.FileSystem", + exclude_prefixes: Optional[List[str]] = None, +) -> Iterator[Tuple[str, int]]: + """ + Expand the provided directory path to a list of file paths. + + Args: + path: The directory path to expand. + filesystem: The filesystem implementation that should be used for + reading these files. + exclude_prefixes: The file relative path prefixes that should be + excluded from the returned file set. Default excluded prefixes are + "." and "_". + + Returns: + An iterator of (file_path, file_size) tuples. + """ + if exclude_prefixes is None: + exclude_prefixes = [".", "_"] + + from pyarrow.fs import FileSelector + + selector = FileSelector(path, recursive=True) + files = filesystem.get_file_info(selector) + base_path = selector.base_dir + out = [] + for file_ in files: + if not file_.is_file: + continue + file_path = file_.path + if not file_path.startswith(base_path): + continue + relative = file_path[len(base_path) :] + if any(relative.startswith(prefix) for prefix in exclude_prefixes): + continue + out.append((file_path, file_.size)) + # We sort the paths to guarantee a stable order. + yield from sorted(out) diff --git a/python/ray/data/datasource/parquet_base_datasource.py b/python/ray/data/datasource/parquet_base_datasource.py index 69fa32df95f49..0956e1acd6f8c 100644 --- a/python/ray/data/datasource/parquet_base_datasource.py +++ b/python/ray/data/datasource/parquet_base_datasource.py @@ -24,7 +24,7 @@ class ParquetBaseDatasource(FileBasedDatasource): def get_name(self): """Return a human-readable name for this datasource. This will be used as the names of the read tasks. - Note: overrides the base `Datasource` method. + Note: overrides the base `FileBasedDatasource` method. """ return "ParquetBulk" diff --git a/python/ray/data/datasource/parquet_datasource.py b/python/ray/data/datasource/parquet_datasource.py index 674410a1c29b2..ef2a42246163d 100644 --- a/python/ray/data/datasource/parquet_datasource.py +++ b/python/ray/data/datasource/parquet_datasource.py @@ -1,4 +1,3 @@ -import itertools import logging from typing import TYPE_CHECKING, Callable, Iterator, List, Optional, Union @@ -18,7 +17,6 @@ _handle_read_os_error, ) from ray.data.datasource.parquet_base_datasource import ParquetBaseDatasource -from ray.types import ObjectRef from ray.util.annotations import PublicAPI import ray.cloudpickle as cloudpickle @@ -161,6 +159,13 @@ class ParquetDatasource(ParquetBaseDatasource): [{"a": 1, "b": "foo"}, ...] """ + def get_name(self): + """Return a human-readable name for this datasource. + This will be used as the names of the read tasks. + Note: overrides the base `ParquetBaseDatasource` method. + """ + return "Parquet" + def create_reader(self, **kwargs): return _ParquetDatasourceReader(**kwargs) @@ -403,29 +408,8 @@ def _read_pieces( yield output_buffer.next() -def _fetch_metadata_remotely( - pieces: List["pyarrow._dataset.ParquetFileFragment"], - **ray_remote_args, -) -> List[ObjectRef["pyarrow.parquet.FileMetaData"]]: - - remote_fetch_metadata = cached_remote_fn(_fetch_metadata_serialization_wrapper) - metas = [] - parallelism = min(len(pieces) // PIECES_PER_META_FETCH, 100) - meta_fetch_bar = ProgressBar("Metadata Fetch Progress", total=parallelism) - for pcs in np.array_split(pieces, parallelism): - if len(pcs) == 0: - continue - metas.append( - remote_fetch_metadata.options(**ray_remote_args).remote( - [_SerializedPiece(p) for p in pcs] - ) - ) - metas = meta_fetch_bar.fetch_until_complete(metas) - return list(itertools.chain.from_iterable(metas)) - - def _fetch_metadata_serialization_wrapper( - pieces: str, + pieces: _SerializedPiece, ) -> List["pyarrow.parquet.FileMetaData"]: pieces: List[ diff --git a/python/ray/data/read_api.py b/python/ray/data/read_api.py index bc25d2a92fb8d..7484d7c16e514 100644 --- a/python/ray/data/read_api.py +++ b/python/ray/data/read_api.py @@ -492,7 +492,17 @@ def read_parquet( ... ("variety", pa.string())] >>> ray.data.read_parquet("example://iris.parquet", ... schema=pa.schema(fields)) - Dataset(num_blocks=..., num_rows=150, schema={sepal.length: double, ...}) + Dataset( + num_blocks=1, + num_rows=150, + schema={ + sepal.length: double, + sepal.width: double, + petal.length: double, + petal.width: double, + variety: string + } + ) For further arguments you can pass to pyarrow as a keyword argument, see https://arrow.apache.org/docs/python/generated/pyarrow.dataset.Scanner.html#pyarrow.dataset.Scanner.from_fragment @@ -1268,6 +1278,14 @@ def from_pandas( if isinstance(dfs, pd.DataFrame): dfs = [dfs] + + from ray.air.util.data_batch_conversion import ( + _cast_ndarray_columns_to_tensor_extension, + ) + + context = DatasetContext.get_current() + if context.enable_tensor_extension_casting: + dfs = [_cast_ndarray_columns_to_tensor_extension(df.copy()) for df in dfs] return from_pandas_refs([ray.put(df) for df in dfs]) diff --git a/python/ray/data/tests/conftest.py b/python/ray/data/tests/conftest.py index 30e935ca0f995..ba67ec79ab704 100644 --- a/python/ray/data/tests/conftest.py +++ b/python/ray/data/tests/conftest.py @@ -185,10 +185,12 @@ def _write_partitioned_df( partition_keys, partition_path_encoder, file_writer_fn, + file_name_suffix="_1", ): import urllib.parse df_partitions = [df for _, df in df.groupby(partition_keys, as_index=False)] + paths = [] for df_partition in df_partitions: partition_values = [] for key in partition_keys: @@ -197,12 +199,15 @@ def _write_partitioned_df( partition_path_encoder.scheme.resolved_filesystem.create_dir(path) base_dir = partition_path_encoder.scheme.base_dir parsed_base_dir = urllib.parse.urlparse(base_dir) + file_name = f"test_{file_name_suffix}.tmp" if parsed_base_dir.scheme: # replace the protocol removed by the partition path generator - path = posixpath.join(f"{parsed_base_dir.scheme}://{path}", "test.tmp") + path = posixpath.join(f"{parsed_base_dir.scheme}://{path}", file_name) else: - path = os.path.join(path, "test.tmp") + path = os.path.join(path, file_name) file_writer_fn(df_partition, path) + paths.append(path) + return paths yield _write_partitioned_df @@ -246,21 +251,35 @@ def _assert_base_partitioned_ds( assert ds.schema() is not None actual_input_files = ds.input_files() assert len(actual_input_files) == num_input_files, actual_input_files - assert ( - str(ds) == f"Dataset(num_blocks={num_input_files}, num_rows={num_rows}, " - f"schema={schema})" - ), ds - assert ( - repr(ds) == f"Dataset(num_blocks={num_input_files}, num_rows={num_rows}, " - f"schema={schema})" - ), ds + + # For Datasets with long string representations, the format will include + # whitespace and newline characters, which is difficult to generalize + # without implementing the formatting logic again (from + # `ExecutionPlan.get_plan_as_string()`). Therefore, we remove whitespace + # characters to test the string contents regardless of the string repr length. + def _remove_whitespace(ds_str): + for c in ["\n", " ", " "]: + ds_str = ds_str.replace(c, "") + return ds_str + + assert "Dataset(num_blocks={},num_rows={},schema={})".format( + num_input_files, + num_rows, + _remove_whitespace(schema), + ) == _remove_whitespace(str(ds)), ds + assert "Dataset(num_blocks={},num_rows={},schema={})".format( + num_input_files, + num_rows, + _remove_whitespace(schema), + ) == _remove_whitespace(repr(ds)), ds + if num_computed is not None: assert ( ds._plan.execute()._num_computed() == num_computed ), f"{ds._plan.execute()._num_computed()} != {num_computed}" # Force a data read. - values = ds_take_transform_fn(ds.take()) + values = ds_take_transform_fn(ds.take_all()) if num_computed is not None: assert ( ds._plan.execute()._num_computed() == num_computed diff --git a/python/ray/data/tests/test_dataset.py b/python/ray/data/tests/test_dataset.py deleted file mode 100644 index aab3e64ef9211..0000000000000 --- a/python/ray/data/tests/test_dataset.py +++ /dev/null @@ -1,4620 +0,0 @@ -import itertools -import logging -import math -import os -import random -import signal -import time -from typing import Iterator -from unittest.mock import patch - -import numpy as np -import pandas as pd -import pyarrow as pa -import pyarrow.parquet as pq -import pytest - -import ray -from ray._private.test_utils import wait_for_condition -from ray.data._internal.dataset_logger import DatasetLogger -from ray.data._internal.stats import _StatsActor -from ray.data._internal.arrow_block import ArrowRow -from ray.data._internal.block_builder import BlockBuilder -from ray.data._internal.lazy_block_list import LazyBlockList -from ray.data._internal.pandas_block import PandasRow -from ray.data.aggregate import AggregateFn, Count, Max, Mean, Min, Std, Sum -from ray.data.block import BlockAccessor, BlockMetadata -from ray.data.context import DatasetContext -from ray.data.dataset import Dataset, _sliding_window -from ray.data.datasource.datasource import Datasource, ReadTask -from ray.data.datasource.csv_datasource import CSVDatasource -from ray.data.extensions.tensor_extension import ( - ArrowTensorArray, - ArrowTensorType, - TensorArray, - TensorDtype, -) -from ray.data.row import TableRow -from ray.data.tests.conftest import * # noqa -from ray.tests.conftest import * # noqa -from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy - - -def maybe_pipeline(ds, enabled): - if enabled: - return ds.window(blocks_per_window=1) - else: - return ds - - -class SlowCSVDatasource(CSVDatasource): - def _read_stream(self, f: "pa.NativeFile", path: str, **reader_args): - for block in CSVDatasource._read_stream(self, f, path, **reader_args): - time.sleep(3) - yield block - - -# Tests that we don't block on exponential rampup when doing bulk reads. -# https://github.com/ray-project/ray/issues/20625 -@pytest.mark.parametrize("block_split", [False, True]) -def test_bulk_lazy_eval_split_mode(shutdown_only, block_split, tmp_path): - # Defensively shutdown Ray for the first test here to make sure there - # is no existing Ray cluster. - ray.shutdown() - - ray.init(num_cpus=8) - ctx = ray.data.context.DatasetContext.get_current() - - try: - original = ctx.block_splitting_enabled - - ray.data.range(8, parallelism=8).write_csv(str(tmp_path)) - if not block_split: - # Setting infinite block size effectively disables block splitting. - ctx.target_max_block_size = float("inf") - ds = ray.data.read_datasource( - SlowCSVDatasource(), parallelism=8, paths=str(tmp_path) - ) - - start = time.time() - ds.map(lambda x: x) - delta = time.time() - start - - print("full read time", delta) - # Should run in ~3 seconds. It takes >9 seconds if bulk read is broken. - assert delta < 8, delta - finally: - ctx.block_splitting_enabled = original - - -@pytest.mark.parametrize("pipelined", [False, True]) -def test_basic_actors(shutdown_only, pipelined): - ray.init(num_cpus=6) - n = 5 - ds = ray.data.range(n) - ds = maybe_pipeline(ds, pipelined) - assert sorted(ds.map(lambda x: x + 1, compute="actors").take()) == list( - range(1, n + 1) - ) - - # Should still work even if num actors > num cpus. - ds = ray.data.range(n) - ds = maybe_pipeline(ds, pipelined) - assert sorted( - ds.map(lambda x: x + 1, compute=ray.data.ActorPoolStrategy(4, 4)).take() - ) == list(range(1, n + 1)) - - # Test setting custom max inflight tasks. - ds = ray.data.range(10, parallelism=5) - ds = maybe_pipeline(ds, pipelined) - assert sorted( - ds.map( - lambda x: x + 1, - compute=ray.data.ActorPoolStrategy(max_tasks_in_flight_per_actor=3), - ).take() - ) == list(range(1, 11)) - - # Test invalid max tasks inflight arg. - with pytest.raises(ValueError): - ray.data.range(10).map( - lambda x: x, - compute=ray.data.ActorPoolStrategy(max_tasks_in_flight_per_actor=0), - ) - - # Test min no more than max check. - with pytest.raises(ValueError): - ray.data.range(10).map(lambda x: x, compute=ray.data.ActorPoolStrategy(8, 4)) - - -@pytest.mark.parametrize("pipelined", [False, True]) -def test_avoid_placement_group_capture(shutdown_only, pipelined): - ray.init(num_cpus=2) - - @ray.remote - def run(): - ds0 = ray.data.range(5) - ds = maybe_pipeline(ds0, pipelined) - assert sorted(ds.map(lambda x: x + 1).take()) == [1, 2, 3, 4, 5] - ds = maybe_pipeline(ds0, pipelined) - assert ds.count() == 5 - ds = maybe_pipeline(ds0, pipelined) - assert sorted(ds.iter_rows()) == [0, 1, 2, 3, 4] - - pg = ray.util.placement_group([{"CPU": 1}]) - ray.get( - run.options( - scheduling_strategy=PlacementGroupSchedulingStrategy( - placement_group=pg, placement_group_capture_child_tasks=True - ) - ).remote() - ) - - -def test_callable_classes(shutdown_only): - ray.init(num_cpus=2) - ds = ray.data.range(10, parallelism=10) - - class StatefulFn: - def __init__(self): - self.num_reuses = 0 - - def __call__(self, x): - r = self.num_reuses - self.num_reuses += 1 - return r - - # Need to specify compute explicitly. - with pytest.raises(ValueError): - ds.map(StatefulFn).take() - - # Need to specify actor compute strategy. - with pytest.raises(ValueError): - ds.map(StatefulFn, compute="tasks").take() - - # Need to specify compute explicitly. - with pytest.raises(ValueError): - ds.flat_map(StatefulFn).take() - - # Need to specify actor compute strategy. - with pytest.raises(ValueError): - ds.flat_map(StatefulFn, compute="tasks") - - # Need to specify compute explicitly. - with pytest.raises(ValueError): - ds.filter(StatefulFn).take() - - # Need to specify actor compute strategy. - with pytest.raises(ValueError): - ds.filter(StatefulFn, compute="tasks") - - # map - actor_reuse = ds.map(StatefulFn, compute="actors").take() - assert sorted(actor_reuse) == list(range(10)), actor_reuse - - class StatefulFn: - def __init__(self): - self.num_reuses = 0 - - def __call__(self, x): - r = self.num_reuses - self.num_reuses += 1 - return [r] - - # flat map - actor_reuse = ds.flat_map(StatefulFn, compute="actors").take() - assert sorted(actor_reuse) == list(range(10)), actor_reuse - - # map batches - actor_reuse = ds.map_batches(StatefulFn, batch_size=1, compute="actors").take() - assert sorted(actor_reuse) == list(range(10)), actor_reuse - - class StatefulFn: - def __init__(self): - self.num_reuses = 0 - - def __call__(self, x): - r = self.num_reuses - self.num_reuses += 1 - return r > 0 - - # filter - actor_reuse = ds.filter(StatefulFn, compute="actors").take() - assert len(actor_reuse) == 9, actor_reuse - - -def test_transform_failure(shutdown_only): - ray.init(num_cpus=2) - ds = ray.data.from_items([0, 10], parallelism=2) - - def mapper(x): - time.sleep(x) - raise ValueError("oops") - return x - - with pytest.raises(ray.exceptions.RayTaskError): - ds.map(mapper).fully_executed() - - -def test_dataset_lineage_serialization(shutdown_only): - ray.init() - ds = ray.data.range(10) - ds = ds.map(lambda x: x + 1) - ds = ds.map(lambda x: x + 1) - ds = ds.random_shuffle() - epoch = ds._get_epoch() - uuid = ds._get_uuid() - plan_uuid = ds._plan._dataset_uuid - - serialized_ds = ds.serialize_lineage() - # Confirm that the original Dataset was properly copied before clearing/mutating. - in_blocks = ds._plan._in_blocks - # Should not raise. - in_blocks._check_if_cleared() - assert isinstance(in_blocks, LazyBlockList) - assert in_blocks._block_partition_refs[0] is None - - ray.shutdown() - ray.init() - - ds = Dataset.deserialize_lineage(serialized_ds) - # Check Dataset state. - assert ds._get_epoch() == epoch - assert ds._get_uuid() == uuid - assert ds._plan._dataset_uuid == plan_uuid - # Check Dataset content. - assert ds.count() == 10 - assert sorted(ds.take()) == list(range(2, 12)) - - -def test_dataset_lineage_serialization_unsupported(shutdown_only): - ray.init() - # In-memory data sources not supported. - ds = ray.data.from_items(list(range(10))) - ds = ds.map(lambda x: x + 1) - ds = ds.map(lambda x: x + 1) - - with pytest.raises(ValueError): - ds.serialize_lineage() - - # In-memory data source unions not supported. - ds = ray.data.from_items(list(range(10))) - ds1 = ray.data.from_items(list(range(10, 20))) - ds2 = ds.union(ds1) - - with pytest.raises(ValueError): - ds2.serialize_lineage() - - # Post-lazy-read unions not supported. - ds = ray.data.range(10).map(lambda x: x + 1) - ds1 = ray.data.range(20).map(lambda x: 2 * x) - ds2 = ds.union(ds1) - - with pytest.raises(ValueError): - ds2.serialize_lineage() - - # Lazy read unions supported. - ds = ray.data.range(10) - ds1 = ray.data.range(20) - ds2 = ds.union(ds1) - - serialized_ds = ds2.serialize_lineage() - ds3 = Dataset.deserialize_lineage(serialized_ds) - assert ds3.take(30) == list(range(10)) + list(range(20)) - - # Zips not supported. - ds = ray.data.from_items(list(range(10))) - ds1 = ray.data.from_items(list(range(10, 20))) - ds2 = ds.zip(ds1) - - with pytest.raises(ValueError): - ds2.serialize_lineage() - - -@pytest.mark.parametrize("pipelined", [False, True]) -def test_basic(ray_start_regular_shared, pipelined): - ds0 = ray.data.range(5) - ds = maybe_pipeline(ds0, pipelined) - assert sorted(ds.map(lambda x: x + 1).take()) == [1, 2, 3, 4, 5] - ds = maybe_pipeline(ds0, pipelined) - assert ds.count() == 5 - ds = maybe_pipeline(ds0, pipelined) - assert sorted(ds.iter_rows()) == [0, 1, 2, 3, 4] - - -def test_flat_map_generator(ray_start_regular_shared): - ds = ray.data.range(3) - - def map_generator(item: int) -> Iterator[int]: - for _ in range(2): - yield item + 1 - - assert sorted(ds.flat_map(map_generator).take()) == [1, 1, 2, 2, 3, 3] - - -def test_zip(ray_start_regular_shared): - ds1 = ray.data.range(5, parallelism=5) - ds2 = ray.data.range(5, parallelism=5).map(lambda x: x + 1) - ds = ds1.zip(ds2) - assert ds.schema() == tuple - assert ds.take() == [(0, 1), (1, 2), (2, 3), (3, 4), (4, 5)] - with pytest.raises(ValueError): - ds.zip(ray.data.range(3)).fully_executed() - - -@pytest.mark.parametrize( - "num_blocks1,num_blocks2", - list(itertools.combinations_with_replacement(range(1, 12), 2)), -) -def test_zip_different_num_blocks_combinations( - ray_start_regular_shared, num_blocks1, num_blocks2 -): - n = 12 - ds1 = ray.data.range(n, parallelism=num_blocks1) - ds2 = ray.data.range(n, parallelism=num_blocks2).map(lambda x: x + 1) - ds = ds1.zip(ds2) - assert ds.schema() == tuple - assert ds.take() == list(zip(range(n), range(1, n + 1))) - - -@pytest.mark.parametrize( - "num_cols1,num_cols2,should_invert", - [ - (1, 1, False), - (4, 1, False), - (1, 4, True), - (1, 10, True), - (10, 10, False), - ], -) -def test_zip_different_num_blocks_split_smallest( - ray_start_regular_shared, - num_cols1, - num_cols2, - should_invert, -): - n = 12 - num_blocks1 = 4 - num_blocks2 = 2 - ds1 = ray.data.from_items( - [{str(i): i for i in range(num_cols1)}] * n, parallelism=num_blocks1 - ) - ds2 = ray.data.from_items( - [{str(i): i for i in range(num_cols1, num_cols1 + num_cols2)}] * n, - parallelism=num_blocks2, - ) - ds = ds1.zip(ds2).fully_executed() - num_blocks = ds._plan._snapshot_blocks.executed_num_blocks() - assert ds.take() == [{str(i): i for i in range(num_cols1 + num_cols2)}] * n - if should_invert: - assert num_blocks == num_blocks2 - else: - assert num_blocks == num_blocks1 - - -def test_zip_pandas(ray_start_regular_shared): - ds1 = ray.data.from_pandas(pd.DataFrame({"col1": [1, 2], "col2": [4, 5]})) - ds2 = ray.data.from_pandas(pd.DataFrame({"col3": ["a", "b"], "col4": ["d", "e"]})) - ds = ds1.zip(ds2) - assert ds.count() == 2 - assert "{col1: int64, col2: int64, col3: object, col4: object}" in str(ds) - result = [r.as_pydict() for r in ds.take()] - assert result[0] == {"col1": 1, "col2": 4, "col3": "a", "col4": "d"} - - ds3 = ray.data.from_pandas(pd.DataFrame({"col2": ["a", "b"], "col4": ["d", "e"]})) - ds = ds1.zip(ds3) - assert ds.count() == 2 - assert "{col1: int64, col2: int64, col2_1: object, col4: object}" in str(ds) - result = [r.as_pydict() for r in ds.take()] - assert result[0] == {"col1": 1, "col2": 4, "col2_1": "a", "col4": "d"} - - -def test_zip_arrow(ray_start_regular_shared): - ds1 = ray.data.range_table(5).map(lambda r: {"id": r["value"]}) - ds2 = ray.data.range_table(5).map( - lambda r: {"a": r["value"] + 1, "b": r["value"] + 2} - ) - ds = ds1.zip(ds2) - assert ds.count() == 5 - assert "{id: int64, a: int64, b: int64}" in str(ds) - result = [r.as_pydict() for r in ds.take()] - assert result[0] == {"id": 0, "a": 1, "b": 2} - - # Test duplicate column names. - ds = ds1.zip(ds1).zip(ds1) - assert ds.count() == 5 - assert "{id: int64, id_1: int64, id_2: int64}" in str(ds) - result = [r.as_pydict() for r in ds.take()] - assert result[0] == {"id": 0, "id_1": 0, "id_2": 0} - - -def test_arrow_block_select(): - df = pd.DataFrame({"one": [10, 11, 12], "two": [11, 12, 13], "three": [14, 15, 16]}) - table = pa.Table.from_pandas(df) - block_accessor = BlockAccessor.for_block(table) - - block = block_accessor.select(["two"]) - assert block.schema == pa.schema([("two", pa.int64())]) - assert block.to_pandas().equals(df[["two"]]) - - block = block_accessor.select(["two", "one"]) - assert block.schema == pa.schema([("two", pa.int64()), ("one", pa.int64())]) - assert block.to_pandas().equals(df[["two", "one"]]) - - with pytest.raises(ValueError): - block = block_accessor.select([lambda x: x % 3, "two"]) - - -def test_pandas_block_select(): - df = pd.DataFrame({"one": [10, 11, 12], "two": [11, 12, 13], "three": [14, 15, 16]}) - block_accessor = BlockAccessor.for_block(df) - - block = block_accessor.select(["two"]) - assert block.equals(df[["two"]]) - - block = block_accessor.select(["two", "one"]) - assert block.equals(df[["two", "one"]]) - - with pytest.raises(ValueError): - block = block_accessor.select([lambda x: x % 3, "two"]) - - -def test_simple_block_select(): - xs = list(range(100)) - block_accessor = BlockAccessor.for_block(xs) - - block = block_accessor.select([lambda x: x % 3]) - assert block == [x % 3 for x in xs] - - with pytest.raises(ValueError): - block = block_accessor.select(["foo"]) - - with pytest.raises(ValueError): - block = block_accessor.select([]) - - -def test_arrow_block_slice_copy(): - # Test that ArrowBlock slicing properly copies the underlying Arrow - # table. - def check_for_copy(table1, table2, a, b, is_copy): - expected_slice = table1.slice(a, b - a) - assert table2.equals(expected_slice) - assert table2.schema == table1.schema - assert table1.num_columns == table2.num_columns - for col1, col2 in zip(table1.columns, table2.columns): - assert col1.num_chunks == col2.num_chunks - for chunk1, chunk2 in zip(col1.chunks, col2.chunks): - bufs1 = chunk1.buffers() - bufs2 = chunk2.buffers() - expected_offset = 0 if is_copy else a - assert chunk2.offset == expected_offset - assert len(chunk2) == b - a - if is_copy: - assert bufs2[1].address != bufs1[1].address - else: - assert bufs2[1].address == bufs1[1].address - - n = 20 - df = pd.DataFrame( - {"one": list(range(n)), "two": ["a"] * n, "three": [np.nan] + [1.5] * (n - 1)} - ) - table = pa.Table.from_pandas(df) - a, b = 5, 10 - block_accessor = BlockAccessor.for_block(table) - - # Test with copy. - table2 = block_accessor.slice(a, b, True) - check_for_copy(table, table2, a, b, is_copy=True) - - # Test without copy. - table2 = block_accessor.slice(a, b, False) - check_for_copy(table, table2, a, b, is_copy=False) - - -def test_arrow_block_slice_copy_empty(): - # Test that ArrowBlock slicing properly copies the underlying Arrow - # table when the table is empty. - df = pd.DataFrame({"one": []}) - table = pa.Table.from_pandas(df) - a, b = 0, 0 - expected_slice = table.slice(a, b - a) - block_accessor = BlockAccessor.for_block(table) - - # Test with copy. - table2 = block_accessor.slice(a, b, True) - assert table2.equals(expected_slice) - assert table2.schema == table.schema - assert table2.num_rows == 0 - - # Test without copy. - table2 = block_accessor.slice(a, b, False) - assert table2.equals(expected_slice) - assert table2.schema == table.schema - assert table2.num_rows == 0 - - -def test_range_table(ray_start_regular_shared): - ds = ray.data.range_table(10, parallelism=10) - assert ds.num_blocks() == 10 - assert ds.count() == 10 - assert ds.take() == [{"value": i} for i in range(10)] - - ds = ray.data.range_table(10, parallelism=2) - assert ds.num_blocks() == 2 - assert ds.count() == 10 - assert ds.take() == [{"value": i} for i in range(10)] - - -def test_empty_shuffle(ray_start_regular_shared): - ds = ray.data.range(100, parallelism=100) - ds = ds.filter(lambda x: x) - ds = ds.map_batches(lambda x: x) - ds = ds.random_shuffle() # Would prev. crash with AssertionError: pyarrow.Table. - ds.show() - - -def test_empty_dataset(ray_start_regular_shared): - ds = ray.data.range(0) - assert ds.count() == 0 - assert ds.size_bytes() is None - assert ds.schema() is None - - ds = ray.data.range(1) - ds = ds.filter(lambda x: x > 1) - ds.fully_executed() - assert str(ds) == "Dataset(num_blocks=1, num_rows=0, schema=Unknown schema)" - - # Test map on empty dataset. - ds = ray.data.from_items([]) - ds = ds.map(lambda x: x) - ds.fully_executed() - assert ds.count() == 0 - - # Test filter on empty dataset. - ds = ray.data.from_items([]) - ds = ds.filter(lambda: True) - ds.fully_executed() - assert ds.count() == 0 - - -def test_schema(ray_start_regular_shared): - ds = ray.data.range(10, parallelism=10) - ds2 = ray.data.range_table(10, parallelism=10) - ds3 = ds2.repartition(5) - ds3.fully_executed() - ds4 = ds3.map(lambda x: {"a": "hi", "b": 1.0}).limit(5).repartition(1) - ds4.fully_executed() - assert str(ds) == "Dataset(num_blocks=10, num_rows=10, schema=)" - assert str(ds2) == "Dataset(num_blocks=10, num_rows=10, schema={value: int64})" - assert str(ds3) == "Dataset(num_blocks=5, num_rows=10, schema={value: int64})" - assert ( - str(ds4) == "Dataset(num_blocks=1, num_rows=5, schema={a: string, b: double})" - ) - - -def test_schema_lazy(ray_start_regular_shared): - ds = ray.data.range(100, parallelism=10) - # We do not kick off the read task by default. - assert ds._plan._in_blocks._num_computed() == 0 - schema = ds.schema() - assert schema == int - assert ds._plan._in_blocks._num_computed() == 1 - # Fetching the schema should not trigger execution of extra read tasks. - assert ds._plan.execute()._num_computed() == 1 - - -def test_count_lazy(ray_start_regular_shared): - ds = ray.data.range(100, parallelism=10) - # We do not kick off the read task by default. - assert ds._plan._in_blocks._num_computed() == 0 - assert ds.count() == 100 - # Getting number of rows should not trigger execution of any read tasks - # for ray.data.range(), as the number of rows is known beforehand. - assert ds._plan._in_blocks._num_computed() == 0 - - -def test_lazy_loading_exponential_rampup(ray_start_regular_shared): - ds = ray.data.range(100, parallelism=20) - - def check_num_computed(expected): - if ray.data.context.DatasetContext.get_current().use_streaming_executor: - # In streaing executor, ds.take() will not invoke partial execution - # in LazyBlocklist. - assert ds._plan.execute()._num_computed() == 0 - else: - assert ds._plan.execute()._num_computed() == expected - - check_num_computed(0) - assert ds.take(10) == list(range(10)) - check_num_computed(2) - assert ds.take(20) == list(range(20)) - check_num_computed(4) - assert ds.take(30) == list(range(30)) - check_num_computed(8) - assert ds.take(50) == list(range(50)) - check_num_computed(16) - assert ds.take(100) == list(range(100)) - check_num_computed(20) - - -def test_dataset_repr(ray_start_regular_shared): - ds = ray.data.range(10, parallelism=10) - assert repr(ds) == "Dataset(num_blocks=10, num_rows=10, schema=)" - ds = ds.map_batches(lambda x: x) - assert repr(ds) == ( - "MapBatches()\n" - "+- Dataset(num_blocks=10, num_rows=10, schema=)" - ) - ds = ds.filter(lambda x: x > 0) - assert repr(ds) == ( - "Filter\n" - "+- MapBatches()\n" - " +- Dataset(num_blocks=10, num_rows=10, schema=)" - ) - ds = ds.random_shuffle() - assert repr(ds) == ( - "RandomShuffle\n" - "+- Filter\n" - " +- MapBatches()\n" - " +- Dataset(num_blocks=10, num_rows=10, schema=)" - ) - ds.fully_executed() - assert repr(ds) == "Dataset(num_blocks=10, num_rows=9, schema=)" - ds = ds.map_batches(lambda x: x) - assert repr(ds) == ( - "MapBatches()\n" - "+- Dataset(num_blocks=10, num_rows=9, schema=)" - ) - ds1, ds2 = ds.split(2) - assert ( - repr(ds1) - == f"Dataset(num_blocks=5, num_rows={ds1.count()}, schema=)" - ) - assert ( - repr(ds2) - == f"Dataset(num_blocks=5, num_rows={ds2.count()}, schema=)" - ) - ds3 = ds1.union(ds2) - assert repr(ds3) == "Dataset(num_blocks=10, num_rows=9, schema=)" - ds = ds.zip(ds3) - assert repr(ds) == ( - "Zip\n" "+- Dataset(num_blocks=10, num_rows=9, schema=)" - ) - - def my_dummy_fn(x): - return x - - ds = ray.data.range(10, parallelism=10) - ds = ds.map_batches(my_dummy_fn) - assert repr(ds) == ( - "MapBatches(my_dummy_fn)\n" - "+- Dataset(num_blocks=10, num_rows=10, schema=)" - ) - - -@pytest.mark.parametrize("lazy", [False, True]) -def test_limit(ray_start_regular_shared, lazy): - ds = ray.data.range(100, parallelism=20) - if not lazy: - ds = ds.fully_executed() - for i in range(100): - assert ds.limit(i).take(200) == list(range(i)) - - -# NOTE: We test outside the power-of-2 range in order to ensure that we're not reading -# redundant files due to exponential ramp-up. -@pytest.mark.parametrize("limit,expected", [(10, 1), (20, 2), (30, 3), (60, 6)]) -def test_limit_no_redundant_read(ray_start_regular_shared, limit, expected): - # Test that dataset truncation eliminates redundant reads. - @ray.remote - class Counter: - def __init__(self): - self.count = 0 - - def increment(self): - self.count += 1 - - def get(self): - return self.count - - def reset(self): - self.count = 0 - - class CountingRangeDatasource(Datasource): - def __init__(self): - self.counter = Counter.remote() - - def prepare_read(self, parallelism, n): - def range_(i): - ray.get(self.counter.increment.remote()) - return [list(range(parallelism * i, parallelism * i + n))] - - return [ - ReadTask( - lambda i=i: range_(i), - BlockMetadata( - num_rows=n, - size_bytes=None, - schema=None, - input_files=None, - exec_stats=None, - ), - ) - for i in range(parallelism) - ] - - source = CountingRangeDatasource() - - ds = ray.data.read_datasource( - source, - parallelism=10, - n=10, - ) - ds2 = ds.limit(limit) - # Check content. - assert ds2.take(limit) == list(range(limit)) - # Check number of read tasks launched. - assert ray.get(source.counter.get.remote()) == expected - - -def test_limit_no_num_row_info(ray_start_regular_shared): - # Test that datasources with no number-of-rows metadata available are still able to - # be truncated, falling back to kicking off all read tasks. - class DumbOnesDatasource(Datasource): - def prepare_read(self, parallelism, n): - return parallelism * [ - ReadTask( - lambda: [[1] * n], - BlockMetadata( - num_rows=None, - size_bytes=None, - schema=None, - input_files=None, - exec_stats=None, - ), - ) - ] - - ds = ray.data.read_datasource(DumbOnesDatasource(), parallelism=10, n=10) - for i in range(1, 100): - assert ds.limit(i).take(100) == [1] * i - - -def test_convert_types(ray_start_regular_shared): - plain_ds = ray.data.range(1) - arrow_ds = plain_ds.map(lambda x: {"a": x}) - assert arrow_ds.take() == [{"a": 0}] - assert "ArrowRow" in arrow_ds.map(lambda x: str(type(x))).take()[0] - - arrow_ds = ray.data.range_table(1) - assert arrow_ds.map(lambda x: "plain_{}".format(x["value"])).take() == ["plain_0"] - # In streaming, we set batch_format to "default" (because calling - # ds.dataset_format() will still invoke bulk execution and we want - # to avoid that). As a result, it's receiving PandasRow (the defaut - # batch format), which unwraps [0] to plain 0. - if ray.data.context.DatasetContext.get_current().use_streaming_executor: - assert arrow_ds.map(lambda x: {"a": (x["value"],)}).take() == [{"a": 0}] - else: - assert arrow_ds.map(lambda x: {"a": (x["value"],)}).take() == [{"a": [0]}] - - -def test_from_items(ray_start_regular_shared): - ds = ray.data.from_items(["hello", "world"]) - assert ds.take() == ["hello", "world"] - - -@pytest.mark.parametrize("parallelism", list(range(1, 21))) -def test_from_items_parallelism(ray_start_regular_shared, parallelism): - # Test that specifying parallelism yields the expected number of blocks. - n = 20 - records = [{"a": i} for i in range(n)] - ds = ray.data.from_items(records, parallelism=parallelism) - out = ds.take_all() - assert out == records - assert ds.num_blocks() == parallelism - - -def test_from_items_parallelism_truncated(ray_start_regular_shared): - # Test that specifying parallelism greater than the number of items is truncated to - # the number of items. - n = 10 - parallelism = 20 - records = [{"a": i} for i in range(n)] - ds = ray.data.from_items(records, parallelism=parallelism) - out = ds.take_all() - assert out == records - assert ds.num_blocks() == n - - -def test_repartition_shuffle(ray_start_regular_shared): - ds = ray.data.range(20, parallelism=10) - assert ds.num_blocks() == 10 - assert ds.sum() == 190 - assert ds._block_num_rows() == [2] * 10 - - ds2 = ds.repartition(5, shuffle=True) - assert ds2.num_blocks() == 5 - assert ds2.sum() == 190 - assert ds2._block_num_rows() == [10, 10, 0, 0, 0] - - ds3 = ds2.repartition(20, shuffle=True) - assert ds3.num_blocks() == 20 - assert ds3.sum() == 190 - assert ds3._block_num_rows() == [2] * 10 + [0] * 10 - - large = ray.data.range(10000, parallelism=10) - large = large.repartition(20, shuffle=True) - assert large._block_num_rows() == [500] * 20 - - -def test_repartition_noshuffle(ray_start_regular_shared): - ds = ray.data.range(20, parallelism=10) - assert ds.num_blocks() == 10 - assert ds.sum() == 190 - assert ds._block_num_rows() == [2] * 10 - - ds2 = ds.repartition(5, shuffle=False) - assert ds2.num_blocks() == 5 - assert ds2.sum() == 190 - assert ds2._block_num_rows() == [4, 4, 4, 4, 4] - - ds3 = ds2.repartition(20, shuffle=False) - assert ds3.num_blocks() == 20 - assert ds3.sum() == 190 - assert ds3._block_num_rows() == [1] * 20 - - # Test num_partitions > num_rows - ds4 = ds.repartition(40, shuffle=False) - assert ds4.num_blocks() == 40 - blocks = ray.get(ds4.get_internal_block_refs()) - assert all(isinstance(block, list) for block in blocks), blocks - assert ds4.sum() == 190 - assert ds4._block_num_rows() == [1] * 20 + [0] * 20 - - ds5 = ray.data.range(22).repartition(4) - assert ds5.num_blocks() == 4 - assert ds5._block_num_rows() == [5, 6, 5, 6] - - large = ray.data.range(10000, parallelism=10) - large = large.repartition(20) - assert large._block_num_rows() == [500] * 20 - - -def test_repartition_shuffle_arrow(ray_start_regular_shared): - ds = ray.data.range_table(20, parallelism=10) - assert ds.num_blocks() == 10 - assert ds.count() == 20 - assert ds._block_num_rows() == [2] * 10 - - ds2 = ds.repartition(5, shuffle=True) - assert ds2.num_blocks() == 5 - assert ds2.count() == 20 - assert ds2._block_num_rows() == [10, 10, 0, 0, 0] - - ds3 = ds2.repartition(20, shuffle=True) - assert ds3.num_blocks() == 20 - assert ds3.count() == 20 - assert ds3._block_num_rows() == [2] * 10 + [0] * 10 - - large = ray.data.range_table(10000, parallelism=10) - large = large.repartition(20, shuffle=True) - assert large._block_num_rows() == [500] * 20 - - -def test_take_all(ray_start_regular_shared): - assert ray.data.range(5).take_all() == [0, 1, 2, 3, 4] - - with pytest.raises(ValueError): - assert ray.data.range(5).take_all(4) - - -def test_convert_to_pyarrow(ray_start_regular_shared, tmp_path): - ds = ray.data.range(100) - assert ds.to_dask().sum().compute()[0] == 4950 - path = os.path.join(tmp_path, "test_parquet_dir") - os.mkdir(path) - ds.write_parquet(path) - assert ray.data.read_parquet(path).count() == 100 - - -def test_pyarrow(ray_start_regular_shared): - ds = ray.data.range_table(5) - assert ds.map(lambda x: {"b": x["value"] + 2}).take() == [ - {"b": 2}, - {"b": 3}, - {"b": 4}, - {"b": 5}, - {"b": 6}, - ] - assert ds.map(lambda x: {"b": x["value"] + 2}).filter( - lambda x: x["b"] % 2 == 0 - ).take() == [{"b": 2}, {"b": 4}, {"b": 6}] - assert ds.filter(lambda x: x["value"] == 0).flat_map( - lambda x: [{"b": x["value"] + 2}, {"b": x["value"] + 20}] - ).take() == [{"b": 2}, {"b": 20}] - - -def test_sliding_window(): - arr = list(range(10)) - - # Test all windows over this iterable. - window_sizes = list(range(1, len(arr) + 1)) - for window_size in window_sizes: - windows = list(_sliding_window(arr, window_size)) - assert len(windows) == len(arr) - window_size + 1 - assert all(len(window) == window_size for window in windows) - assert all( - list(window) == arr[i : i + window_size] for i, window in enumerate(windows) - ) - - # Test window size larger than iterable length. - windows = list(_sliding_window(arr, 15)) - assert len(windows) == 1 - assert list(windows[0]) == arr - - -def test_iter_rows(ray_start_regular_shared): - # Test simple rows. - n = 10 - ds = ray.data.range(n) - for row, k in zip(ds.iter_rows(), range(n)): - assert row == k - - # Test tabular rows. - t1 = pa.Table.from_pydict({"one": [1, 2, 3], "two": [2, 3, 4]}) - t2 = pa.Table.from_pydict({"one": [4, 5, 6], "two": [5, 6, 7]}) - t3 = pa.Table.from_pydict({"one": [7, 8, 9], "two": [8, 9, 10]}) - t4 = pa.Table.from_pydict({"one": [10, 11, 12], "two": [11, 12, 13]}) - ts = [t1, t2, t3, t4] - t = pa.concat_tables(ts) - ds = ray.data.from_arrow(ts) - - def to_pylist(table): - pydict = table.to_pydict() - names = table.schema.names - pylist = [ - {column: pydict[column][row] for column in names} - for row in range(table.num_rows) - ] - return pylist - - # Default ArrowRows. - for row, t_row in zip(ds.iter_rows(), to_pylist(t)): - assert isinstance(row, TableRow) - # In streaming, we set batch_format to "default" because calling - # ds.dataset_format() will still invoke bulk execution and we want - # to avoid that. As a result, it's receiving PandasRow (the defaut - # batch format). - if ray.data.context.DatasetContext.get_current().use_streaming_executor: - assert isinstance(row, PandasRow) - else: - assert isinstance(row, ArrowRow) - assert row == t_row - - # PandasRows after conversion. - pandas_ds = ds.map_batches(lambda x: x, batch_format="pandas") - df = t.to_pandas() - for row, (index, df_row) in zip(pandas_ds.iter_rows(), df.iterrows()): - assert isinstance(row, TableRow) - assert isinstance(row, PandasRow) - assert row == df_row.to_dict() - - # Prefetch. - for row, t_row in zip(ds.iter_rows(prefetch_blocks=1), to_pylist(t)): - assert isinstance(row, TableRow) - # In streaming, we set batch_format to "default" because calling - # ds.dataset_format() will still invoke bulk execution and we want - # to avoid that. As a result, it's receiving PandasRow (the defaut - # batch format). - if ray.data.context.DatasetContext.get_current().use_streaming_executor: - assert isinstance(row, PandasRow) - else: - assert isinstance(row, ArrowRow) - assert row == t_row - - -def test_iter_batches_basic(ray_start_regular_shared): - df1 = pd.DataFrame({"one": [1, 2, 3], "two": [2, 3, 4]}) - df2 = pd.DataFrame({"one": [4, 5, 6], "two": [5, 6, 7]}) - df3 = pd.DataFrame({"one": [7, 8, 9], "two": [8, 9, 10]}) - df4 = pd.DataFrame({"one": [10, 11, 12], "two": [11, 12, 13]}) - dfs = [df1, df2, df3, df4] - ds = ray.data.from_pandas(dfs) - - # Default. - for batch, df in zip(ds.iter_batches(batch_size=None, batch_format="pandas"), dfs): - assert isinstance(batch, pd.DataFrame) - assert batch.equals(df) - - # pyarrow.Table format. - for batch, df in zip(ds.iter_batches(batch_size=None, batch_format="pyarrow"), dfs): - assert isinstance(batch, pa.Table) - assert batch.equals(pa.Table.from_pandas(df)) - - # NumPy format. - for batch, df in zip(ds.iter_batches(batch_size=None, batch_format="numpy"), dfs): - assert isinstance(batch, dict) - assert list(batch.keys()) == ["one", "two"] - assert all(isinstance(col, np.ndarray) for col in batch.values()) - pd.testing.assert_frame_equal(pd.DataFrame(batch), df) - - # Numpy format (single column). - ds2 = ds.select_columns(["one"]) - for batch, df in zip(ds2.iter_batches(batch_size=None, batch_format="numpy"), dfs): - assert isinstance(batch, dict) - assert list(batch.keys()) == ["one"] - assert all(isinstance(col, np.ndarray) for col in batch.values()) - pd.testing.assert_frame_equal(pd.DataFrame(batch), df[["one"]]) - - # Test NumPy format on Arrow blocks. - ds2 = ds.map_batches(lambda b: b, batch_size=None, batch_format="pyarrow") - for batch, df in zip(ds2.iter_batches(batch_size=None, batch_format="numpy"), dfs): - assert isinstance(batch, dict) - assert list(batch.keys()) == ["one", "two"] - assert all(isinstance(col, np.ndarray) for col in batch.values()) - pd.testing.assert_frame_equal(pd.DataFrame(batch), df) - - # Test NumPy format on Arrow blocks (single column). - ds3 = ds2.select_columns(["one"]) - for batch, df in zip(ds3.iter_batches(batch_size=None, batch_format="numpy"), dfs): - assert isinstance(batch, dict) - assert list(batch.keys()) == ["one"] - assert all(isinstance(col, np.ndarray) for col in batch.values()) - pd.testing.assert_frame_equal(pd.DataFrame(batch), df[["one"]]) - - # Native format (deprecated). - for batch, df in zip(ds.iter_batches(batch_size=None, batch_format="native"), dfs): - assert BlockAccessor.for_block(batch).to_pandas().equals(df) - - # Default format. - for batch, df in zip(ds.iter_batches(batch_size=None, batch_format="default"), dfs): - assert BlockAccessor.for_block(batch).to_pandas().equals(df) - - # Batch size. - batch_size = 2 - batches = list(ds.iter_batches(batch_size=batch_size, batch_format="pandas")) - assert all(len(batch) == batch_size for batch in batches) - assert len(batches) == math.ceil( - (len(df1) + len(df2) + len(df3) + len(df4)) / batch_size - ) - assert pd.concat(batches, ignore_index=True).equals( - pd.concat(dfs, ignore_index=True) - ) - - # Batch size larger than block. - batch_size = 4 - batches = list(ds.iter_batches(batch_size=batch_size, batch_format="pandas")) - assert all(len(batch) == batch_size for batch in batches) - assert len(batches) == math.ceil( - (len(df1) + len(df2) + len(df3) + len(df4)) / batch_size - ) - assert pd.concat(batches, ignore_index=True).equals( - pd.concat(dfs, ignore_index=True) - ) - - # Batch size larger than dataset. - batch_size = 15 - batches = list(ds.iter_batches(batch_size=batch_size, batch_format="pandas")) - assert all(len(batch) == ds.count() for batch in batches) - assert len(batches) == 1 - assert pd.concat(batches, ignore_index=True).equals( - pd.concat(dfs, ignore_index=True) - ) - - # Batch size drop partial. - batch_size = 5 - batches = list( - ds.iter_batches(batch_size=batch_size, drop_last=True, batch_format="pandas") - ) - assert all(len(batch) == batch_size for batch in batches) - assert len(batches) == (len(df1) + len(df2) + len(df3) + len(df4)) // batch_size - assert pd.concat(batches, ignore_index=True).equals( - pd.concat(dfs, ignore_index=True)[:10] - ) - - # Batch size don't drop partial. - batch_size = 5 - batches = list( - ds.iter_batches(batch_size=batch_size, drop_last=False, batch_format="pandas") - ) - assert all(len(batch) == batch_size for batch in batches[:-1]) - assert len(batches[-1]) == (len(df1) + len(df2) + len(df3) + len(df4)) % batch_size - assert len(batches) == math.ceil( - (len(df1) + len(df2) + len(df3) + len(df4)) / batch_size - ) - assert pd.concat(batches, ignore_index=True).equals( - pd.concat(dfs, ignore_index=True) - ) - - # Prefetch. - batches = list( - ds.iter_batches(prefetch_blocks=1, batch_size=None, batch_format="pandas") - ) - assert len(batches) == len(dfs) - for batch, df in zip(batches, dfs): - assert isinstance(batch, pd.DataFrame) - assert batch.equals(df) - - batch_size = 2 - batches = list( - ds.iter_batches(prefetch_blocks=2, batch_size=batch_size, batch_format="pandas") - ) - assert all(len(batch) == batch_size for batch in batches) - assert len(batches) == math.ceil( - (len(df1) + len(df2) + len(df3) + len(df4)) / batch_size - ) - assert pd.concat(batches, ignore_index=True).equals( - pd.concat(dfs, ignore_index=True) - ) - - # Prefetch more than number of blocks. - batches = list( - ds.iter_batches( - prefetch_blocks=len(dfs), batch_size=None, batch_format="pandas" - ) - ) - assert len(batches) == len(dfs) - for batch, df in zip(batches, dfs): - assert isinstance(batch, pd.DataFrame) - assert batch.equals(df) - - # Prefetch with ray.wait. - context = DatasetContext.get_current() - old_config = context.actor_prefetcher_enabled - try: - context.actor_prefetcher_enabled = False - batches = list( - ds.iter_batches(prefetch_blocks=1, batch_size=None, batch_format="pandas") - ) - assert len(batches) == len(dfs) - for batch, df in zip(batches, dfs): - assert isinstance(batch, pd.DataFrame) - assert batch.equals(df) - finally: - context.actor_prefetcher_enabled = old_config - - -def test_iter_batches_empty_block(ray_start_regular_shared): - ds = ray.data.range(1).repartition(10) - assert list(ds.iter_batches(batch_size=None)) == [[0]] - assert list(ds.iter_batches(batch_size=1, local_shuffle_buffer_size=1)) == [[0]] - - -@pytest.mark.parametrize("pipelined", [False, True]) -@pytest.mark.parametrize("ds_format", ["arrow", "pandas", "simple"]) -def test_iter_batches_local_shuffle(shutdown_only, pipelined, ds_format): - # Input validation. - # Batch size must be given for local shuffle. - with pytest.raises(ValueError): - list( - ray.data.range(100).iter_batches( - batch_size=None, local_shuffle_buffer_size=10 - ) - ) - - def range(n, parallelism=200): - if ds_format == "simple": - ds = ray.data.range(n, parallelism=parallelism) - elif ds_format == "arrow": - ds = ray.data.range_table(n, parallelism=parallelism) - elif ds_format == "pandas": - ds = ray.data.range_table(n, parallelism=parallelism).map_batches( - lambda df: df, batch_size=None, batch_format="pandas" - ) - if pipelined: - pipe = ds.repeat(2) - return pipe - else: - return ds - - def to_row_dicts(batch): - if isinstance(batch, pd.DataFrame): - batch = batch.to_dict(orient="records") - return batch - - def unbatch(batches): - return [r for batch in batches for r in to_row_dicts(batch)] - - def sort(r): - if ds_format == "simple": - return sorted(r) - return sorted(r, key=lambda v: v["value"]) - - base = range(100).take_all() - - # Local shuffle. - r1 = unbatch( - range(100, parallelism=10).iter_batches( - batch_size=3, - local_shuffle_buffer_size=25, - ) - ) - r2 = unbatch( - range(100, parallelism=10).iter_batches( - batch_size=3, - local_shuffle_buffer_size=25, - ) - ) - # Check randomness of shuffle. - assert r1 != r2, (r1, r2) - assert r1 != base - assert r2 != base - # Check content. - assert sort(r1) == sort(base) - assert sort(r2) == sort(base) - - # Set seed. - r1 = unbatch( - range(100, parallelism=10).iter_batches( - batch_size=3, - local_shuffle_buffer_size=25, - local_shuffle_seed=0, - ) - ) - r2 = unbatch( - range(100, parallelism=10).iter_batches( - batch_size=3, - local_shuffle_buffer_size=25, - local_shuffle_seed=0, - ) - ) - # Check randomness of shuffle. - assert r1 == r2, (r1, r2) - assert r1 != base - # Check content. - assert sort(r1) == sort(base) - - # Single block. - r1 = unbatch( - range(100, parallelism=1).iter_batches( - batch_size=3, - local_shuffle_buffer_size=25, - ) - ) - r2 = unbatch( - range(100, parallelism=1).iter_batches( - batch_size=3, - local_shuffle_buffer_size=25, - ) - ) - # Check randomness of shuffle. - assert r1 != r2, (r1, r2) - assert r1 != base - assert r2 != base - # Check content. - assert sort(r1) == sort(base) - assert sort(r2) == sort(base) - - # Single-row blocks. - r1 = unbatch( - range(100, parallelism=100).iter_batches( - batch_size=3, - local_shuffle_buffer_size=25, - ) - ) - r2 = unbatch( - range(100, parallelism=100).iter_batches( - batch_size=3, - local_shuffle_buffer_size=25, - ) - ) - # Check randomness of shuffle. - assert r1 != r2, (r1, r2) - assert r1 != base - assert r2 != base - # Check content. - assert sort(r1) == sort(base) - assert sort(r2) == sort(base) - - # Buffer larger than dataset. - r1 = unbatch( - range(100, parallelism=10).iter_batches( - batch_size=3, - local_shuffle_buffer_size=200, - ) - ) - r2 = unbatch( - range(100, parallelism=10).iter_batches( - batch_size=3, - local_shuffle_buffer_size=200, - ) - ) - # Check randomness of shuffle. - assert r1 != r2, (r1, r2) - assert r1 != base - assert r2 != base - # Check content. - assert sort(r1) == sort(base) - assert sort(r2) == sort(base) - - # Batch size larger than block. - r1 = unbatch( - range(100, parallelism=20).iter_batches( - batch_size=12, - local_shuffle_buffer_size=25, - ) - ) - r2 = unbatch( - range(100, parallelism=20).iter_batches( - batch_size=12, - local_shuffle_buffer_size=25, - ) - ) - # Check randomness of shuffle. - assert r1 != r2, (r1, r2) - assert r1 != base - assert r2 != base - # Check content. - assert sort(r1) == sort(base) - assert sort(r2) == sort(base) - - # Batch size larger than dataset. - r1 = unbatch( - range(100, parallelism=10).iter_batches( - batch_size=200, - local_shuffle_buffer_size=400, - ) - ) - r2 = unbatch( - range(100, parallelism=10).iter_batches( - batch_size=200, - local_shuffle_buffer_size=400, - ) - ) - # Check randomness of shuffle. - assert r1 != r2, (r1, r2) - assert r1 != base - assert r2 != base - # Check content. - assert sort(r1) == sort(base) - assert sort(r2) == sort(base) - - # Drop partial batches. - r1 = unbatch( - range(100, parallelism=10).iter_batches( - batch_size=7, - local_shuffle_buffer_size=21, - drop_last=True, - ) - ) - r2 = unbatch( - range(100, parallelism=10).iter_batches( - batch_size=7, - local_shuffle_buffer_size=21, - drop_last=True, - ) - ) - # Check randomness of shuffle. - assert r1 != r2, (r1, r2) - assert r1 != base - assert r2 != base - # Check content. - # Check that partial batches were dropped. - assert len(r1) % 7 == 0 - assert len(r2) % 7 == 0 - tmp_base = base - if ds_format in ("arrow", "pandas"): - r1 = [tuple(r.items()) for r in r1] - r2 = [tuple(r.items()) for r in r2] - tmp_base = [tuple(r.items()) for r in base] - assert set(r1) <= set(tmp_base) - assert set(r2) <= set(tmp_base) - - # Test empty dataset. - ds = ray.data.from_items([]) - r1 = unbatch(ds.iter_batches(batch_size=2, local_shuffle_buffer_size=10)) - assert len(r1) == 0 - assert r1 == ds.take() - - -def test_iter_batches_grid(ray_start_regular_shared): - # Tests slicing, batch combining, and partial batch dropping logic over - # a grid of dataset, batching, and dropping configurations. - # Grid: num_blocks x num_rows_block_1 x ... x num_rows_block_N x - # batch_size x drop_last - seed = int(time.time()) - print(f"Seeding RNG for test_iter_batches_grid with: {seed}") - random.seed(seed) - max_num_blocks = 20 - max_num_rows_per_block = 20 - num_blocks_samples = 3 - block_sizes_samples = 3 - batch_size_samples = 3 - - for num_blocks in np.random.randint(1, max_num_blocks + 1, size=num_blocks_samples): - block_sizes_list = [ - np.random.randint(1, max_num_rows_per_block + 1, size=num_blocks) - for _ in range(block_sizes_samples) - ] - for block_sizes in block_sizes_list: - # Create the dataset with the given block sizes. - dfs = [] - running_size = 0 - for block_size in block_sizes: - dfs.append( - pd.DataFrame( - {"value": list(range(running_size, running_size + block_size))} - ) - ) - running_size += block_size - num_rows = running_size - ds = ray.data.from_pandas(dfs) - for batch_size in np.random.randint( - 1, num_rows + 1, size=batch_size_samples - ): - for drop_last in (False, True): - batches = list( - ds.iter_batches( - batch_size=batch_size, - drop_last=drop_last, - batch_format="pandas", - ) - ) - if num_rows % batch_size == 0 or not drop_last: - # Number of batches should be equal to - # num_rows / batch_size, rounded up. - assert len(batches) == math.ceil(num_rows / batch_size) - # Concatenated batches should equal the DataFrame - # representation of the entire dataset. - assert pd.concat(batches, ignore_index=True).equals( - ds.to_pandas() - ) - else: - # Number of batches should be equal to - # num_rows / batch_size, rounded down. - assert len(batches) == num_rows // batch_size - # Concatenated batches should equal the DataFrame - # representation of the dataset with the partial batch - # remainder sliced off. - assert pd.concat(batches, ignore_index=True).equals( - ds.to_pandas()[: batch_size * (num_rows // batch_size)] - ) - if num_rows % batch_size == 0 or drop_last: - assert all(len(batch) == batch_size for batch in batches) - else: - assert all(len(batch) == batch_size for batch in batches[:-1]) - assert len(batches[-1]) == num_rows % batch_size - - -def test_lazy_loading_iter_batches_exponential_rampup(ray_start_regular_shared): - ds = ray.data.range(32, parallelism=8) - expected_num_blocks = [1, 2, 4, 4, 8, 8, 8, 8] - for _, expected in zip(ds.iter_batches(batch_size=None), expected_num_blocks): - if ray.data.context.DatasetContext.get_current().use_streaming_executor: - # In streaming execution of ds.iter_batches(), there is no partial - # execution so _num_computed() in LazyBlocklist is 0. - assert ds._plan.execute()._num_computed() == 0 - else: - assert ds._plan.execute()._num_computed() == expected - - -def test_add_column(ray_start_regular_shared): - ds = ray.data.range(5).add_column("foo", lambda x: 1) - assert ds.take(1) == [{"value": 0, "foo": 1}] - - ds = ray.data.range_table(5).add_column("foo", lambda x: x["value"] + 1) - assert ds.take(1) == [{"value": 0, "foo": 1}] - - ds = ray.data.range_table(5).add_column("value", lambda x: x["value"] + 1) - assert ds.take(2) == [{"value": 1}, {"value": 2}] - - with pytest.raises(ValueError): - ds = ray.data.range(5).add_column("value", 0) - - -def test_drop_columns(ray_start_regular_shared, tmp_path): - df = pd.DataFrame({"col1": [1, 2, 3], "col2": [2, 3, 4], "col3": [3, 4, 5]}) - ds1 = ray.data.from_pandas(df) - ds1.write_parquet(str(tmp_path)) - ds2 = ray.data.read_parquet(str(tmp_path)) - - for ds in [ds1, ds2]: - assert ds.drop_columns(["col2"]).take(1) == [{"col1": 1, "col3": 3}] - assert ds.drop_columns(["col1", "col3"]).take(1) == [{"col2": 2}] - assert ds.drop_columns([]).take(1) == [{"col1": 1, "col2": 2, "col3": 3}] - assert ds.drop_columns(["col1", "col2", "col3"]).take(1) == [{}] - assert ds.drop_columns(["col1", "col1", "col2", "col1"]).take(1) == [ - {"col3": 3} - ] - # Test dropping non-existent column - with pytest.raises(KeyError): - ds.drop_columns(["dummy_col", "col1", "col2"]).fully_executed() - - -def test_select_columns(ray_start_regular_shared): - # Test pandas and arrow - df = pd.DataFrame({"col1": [1, 2, 3], "col2": [2, 3, 4], "col3": [3, 4, 5]}) - ds1 = ray.data.from_pandas(df) - assert ds1.dataset_format() == "pandas" - - ds2 = ds1.map_batches(lambda pa: pa, batch_size=1, batch_format="pyarrow") - assert ds2.dataset_format() == "arrow" - - for each_ds in [ds1, ds2]: - assert each_ds.select_columns(cols=[]).take(1) == [{}] - assert each_ds.select_columns(cols=["col1", "col2", "col3"]).take(1) == [ - {"col1": 1, "col2": 2, "col3": 3} - ] - assert each_ds.select_columns(cols=["col1", "col2"]).take(1) == [ - {"col1": 1, "col2": 2} - ] - assert each_ds.select_columns(cols=["col2", "col1"]).take(1) == [ - {"col1": 1, "col2": 2} - ] - # Test selecting columns with duplicates - assert each_ds.select_columns(cols=["col1", "col2", "col2"]).schema().names == [ - "col1", - "col2", - "col2", - ] - # Test selecting a column that is not in the dataset schema - with pytest.raises(KeyError): - each_ds.select_columns(cols=["col1", "col2", "dummy_col"]).fully_executed() - - # Test simple - ds3 = ray.data.range(10) - assert ds3.dataset_format() == "simple" - with pytest.raises(ValueError): - ds3.select_columns(cols=[]).fully_executed() - - -def test_map_batches_basic(ray_start_regular_shared, tmp_path, restore_dataset_context): - ctx = DatasetContext.get_current() - ctx.execution_options.preserve_order = True - - # Test input validation - ds = ray.data.range(5) - with pytest.raises(ValueError): - ds.map_batches(lambda x: x + 1, batch_format="pyarrow", batch_size=-1).take() - - # Set up. - df = pd.DataFrame({"one": [1, 2, 3], "two": [2, 3, 4]}) - table = pa.Table.from_pandas(df) - pq.write_table(table, os.path.join(tmp_path, "test1.parquet")) - - # Test pandas - ds = ray.data.read_parquet(str(tmp_path)) - ds2 = ds.map_batches(lambda df: df + 1, batch_size=1, batch_format="pandas") - assert ds2.dataset_format() == "pandas" - ds_list = ds2.take() - values = [s["one"] for s in ds_list] - assert values == [2, 3, 4] - values = [s["two"] for s in ds_list] - assert values == [3, 4, 5] - - # Test Pyarrow - ds = ray.data.read_parquet(str(tmp_path)) - ds2 = ds.map_batches(lambda pa: pa, batch_size=1, batch_format="pyarrow") - assert ds2.dataset_format() == "arrow" - ds_list = ds2.take() - values = [s["one"] for s in ds_list] - assert values == [1, 2, 3] - values = [s["two"] for s in ds_list] - assert values == [2, 3, 4] - - # Test batch - size = 300 - ds = ray.data.range(size) - ds2 = ds.map_batches(lambda df: df + 1, batch_size=17, batch_format="pandas") - assert ds2.dataset_format() == "pandas" - ds_list = ds2.take_all() - for i in range(size): - # The pandas column is "value", and it originally has rows from 0~299. - # After the map batch, it should have 1~300. - row = ds_list[i] - assert row["value"] == i + 1 - assert ds.count() == 300 - - # Test the lambda returns different types than the batch_format - # pandas => list block - ds = ray.data.read_parquet(str(tmp_path)) - ds2 = ds.map_batches(lambda df: [1], batch_size=1) - assert ds2.dataset_format() == "simple" - ds_list = ds2.take() - assert ds_list == [1, 1, 1] - assert ds.count() == 3 - - # pyarrow => list block - ds = ray.data.read_parquet(str(tmp_path)) - ds2 = ds.map_batches(lambda df: [1], batch_size=1, batch_format="pyarrow") - assert ds2.dataset_format() == "simple" - ds_list = ds2.take() - assert ds_list == [1, 1, 1] - assert ds.count() == 3 - - # Test the wrong return value raises an exception. - ds = ray.data.read_parquet(str(tmp_path)) - with pytest.raises(ValueError): - ds_list = ds.map_batches( - lambda df: 1, batch_size=2, batch_format="pyarrow" - ).take() - - -def test_map_batches_extra_args(ray_start_regular_shared, tmp_path): - def put(x): - # We only support automatic deref in the legacy backend. - if DatasetContext.get_current().new_execution_backend: - return x - else: - return ray.put(x) - - # Test input validation - ds = ray.data.range(5) - - class Foo: - def __call__(self, df): - return df - - with pytest.raises(ValueError): - # CallableClass not supported for task compute strategy, which is the default. - ds.map_batches(Foo) - - with pytest.raises(ValueError): - # CallableClass not supported for task compute strategy. - ds.map_batches(Foo, compute="tasks") - - with pytest.raises(ValueError): - # fn_constructor_args and fn_constructor_kwargs only supported for actor - # compute strategy. - ds.map_batches( - lambda x: x, - compute="tasks", - fn_constructor_args=(1,), - fn_constructor_kwargs={"a": 1}, - ) - - with pytest.raises(ValueError): - # fn_constructor_args and fn_constructor_kwargs only supported for callable - # class UDFs. - ds.map_batches( - lambda x: x, - compute="actors", - fn_constructor_args=(1,), - fn_constructor_kwargs={"a": 1}, - ) - - # Set up. - df = pd.DataFrame({"one": [1, 2, 3], "two": [2, 3, 4]}) - table = pa.Table.from_pandas(df) - pq.write_table(table, os.path.join(tmp_path, "test1.parquet")) - - # Test extra UDF args. - # Test positional. - def udf(batch, a): - assert a == 1 - return batch + a - - ds = ray.data.read_parquet(str(tmp_path)) - ds2 = ds.map_batches( - udf, - batch_size=1, - batch_format="pandas", - fn_args=(put(1),), - ) - assert ds2.dataset_format() == "pandas" - ds_list = ds2.take() - values = [s["one"] for s in ds_list] - assert values == [2, 3, 4] - values = [s["two"] for s in ds_list] - assert values == [3, 4, 5] - - # Test kwargs. - def udf(batch, b=None): - assert b == 2 - return b * batch - - ds = ray.data.read_parquet(str(tmp_path)) - ds2 = ds.map_batches( - udf, - batch_size=1, - batch_format="pandas", - fn_kwargs={"b": put(2)}, - ) - assert ds2.dataset_format() == "pandas" - ds_list = ds2.take() - values = [s["one"] for s in ds_list] - assert values == [2, 4, 6] - values = [s["two"] for s in ds_list] - assert values == [4, 6, 8] - - # Test both. - def udf(batch, a, b=None): - assert a == 1 - assert b == 2 - return b * batch + a - - ds = ray.data.read_parquet(str(tmp_path)) - ds2 = ds.map_batches( - udf, - batch_size=1, - batch_format="pandas", - fn_args=(put(1),), - fn_kwargs={"b": put(2)}, - ) - assert ds2.dataset_format() == "pandas" - ds_list = ds2.take() - values = [s["one"] for s in ds_list] - assert values == [3, 5, 7] - values = [s["two"] for s in ds_list] - assert values == [5, 7, 9] - - # Test constructor UDF args. - # Test positional. - class CallableFn: - def __init__(self, a): - assert a == 1 - self.a = a - - def __call__(self, x): - return x + self.a - - ds = ray.data.read_parquet(str(tmp_path)) - ds2 = ds.map_batches( - CallableFn, - batch_size=1, - batch_format="pandas", - compute="actors", - fn_constructor_args=(put(1),), - ) - assert ds2.dataset_format() == "pandas" - ds_list = ds2.take() - values = [s["one"] for s in ds_list] - assert values == [2, 3, 4] - values = [s["two"] for s in ds_list] - assert values == [3, 4, 5] - - # Test kwarg. - class CallableFn: - def __init__(self, b=None): - assert b == 2 - self.b = b - - def __call__(self, x): - return self.b * x - - ds = ray.data.read_parquet(str(tmp_path)) - ds2 = ds.map_batches( - CallableFn, - batch_size=1, - batch_format="pandas", - compute="actors", - fn_constructor_kwargs={"b": put(2)}, - ) - assert ds2.dataset_format() == "pandas" - ds_list = ds2.take() - values = [s["one"] for s in ds_list] - assert values == [2, 4, 6] - values = [s["two"] for s in ds_list] - assert values == [4, 6, 8] - - # Test both. - class CallableFn: - def __init__(self, a, b=None): - assert a == 1 - assert b == 2 - self.a = a - self.b = b - - def __call__(self, x): - return self.b * x + self.a - - ds = ray.data.read_parquet(str(tmp_path)) - ds2 = ds.map_batches( - CallableFn, - batch_size=1, - batch_format="pandas", - compute="actors", - fn_constructor_args=(put(1),), - fn_constructor_kwargs={"b": put(2)}, - ) - assert ds2.dataset_format() == "pandas" - ds_list = ds2.take() - values = [s["one"] for s in ds_list] - assert values == [3, 5, 7] - values = [s["two"] for s in ds_list] - assert values == [5, 7, 9] - - # Test callable chain. - ds = ray.data.read_parquet(str(tmp_path)) - fn_constructor_args = (put(1),) - fn_constructor_kwargs = {"b": put(2)} - ds2 = ( - ds.lazy() - .map_batches( - CallableFn, - batch_size=1, - batch_format="pandas", - compute="actors", - fn_constructor_args=fn_constructor_args, - fn_constructor_kwargs=fn_constructor_kwargs, - ) - .map_batches( - CallableFn, - batch_size=1, - batch_format="pandas", - compute="actors", - fn_constructor_args=fn_constructor_args, - fn_constructor_kwargs=fn_constructor_kwargs, - ) - ) - assert ds2.dataset_format() == "pandas" - ds_list = ds2.take() - values = [s["one"] for s in ds_list] - assert values == [7, 11, 15] - values = [s["two"] for s in ds_list] - assert values == [11, 15, 19] - - # Test function + callable chain. - ds = ray.data.read_parquet(str(tmp_path)) - fn_constructor_args = (put(1),) - fn_constructor_kwargs = {"b": put(2)} - ds2 = ( - ds.lazy() - .map_batches( - lambda df, a, b=None: b * df + a, - batch_size=1, - batch_format="pandas", - compute="actors", - fn_args=(put(1),), - fn_kwargs={"b": put(2)}, - ) - .map_batches( - CallableFn, - batch_size=1, - batch_format="pandas", - compute="actors", - fn_constructor_args=fn_constructor_args, - fn_constructor_kwargs=fn_constructor_kwargs, - ) - ) - assert ds2.dataset_format() == "pandas" - ds_list = ds2.take() - values = [s["one"] for s in ds_list] - assert values == [7, 11, 15] - values = [s["two"] for s in ds_list] - assert values == [11, 15, 19] - - -def test_map_batches_generator(ray_start_regular_shared, tmp_path): - # Set up. - df = pd.DataFrame({"one": [1, 2, 3], "two": [2, 3, 4]}) - table = pa.Table.from_pandas(df) - pq.write_table(table, os.path.join(tmp_path, "test1.parquet")) - - def pandas_generator(batch: pd.DataFrame) -> Iterator[pd.DataFrame]: - for i in range(len(batch)): - yield batch.iloc[[i]] + 1 - - ds = ray.data.read_parquet(str(tmp_path)) - ds2 = ds.map_batches(pandas_generator, batch_size=1, batch_format="pandas") - assert ds2.dataset_format() == "pandas" - ds_list = ds2.take() - values = sorted([s["one"] for s in ds_list]) - assert values == [2, 3, 4] - values = sorted([s["two"] for s in ds_list]) - assert values == [3, 4, 5] - - def fail_generator(batch): - for i in range(len(batch)): - yield i - - # Test the wrong return value raises an exception. - ds = ray.data.read_parquet(str(tmp_path)) - with pytest.raises(ValueError): - ds_list = ds.map_batches( - fail_generator, batch_size=2, batch_format="pyarrow" - ).take() - - -def test_map_batches_actors_preserves_order(ray_start_regular_shared): - # Test that actor compute model preserves block order. - ds = ray.data.range(10, parallelism=5) - assert ds.map_batches(lambda x: x, compute="actors").take() == list(range(10)) - - -@pytest.mark.parametrize( - "num_rows,num_blocks,batch_size", - [ - (10, 5, 2), - (10, 1, 10), - (12, 3, 2), - ], -) -def test_map_batches_batch_mutation( - ray_start_regular_shared, num_rows, num_blocks, batch_size, restore_dataset_context -): - ctx = DatasetContext.get_current() - ctx.execution_options.preserve_order = True - - # Test that batch mutation works without encountering a read-only error (e.g. if the - # batch is a zero-copy view on data in the object store). - def mutate(df): - df["value"] += 1 - return df - - ds = ray.data.range_table(num_rows, parallelism=num_blocks).repartition(num_blocks) - # Convert to Pandas blocks. - ds = ds.map_batches(lambda df: df, batch_format="pandas", batch_size=None) - - # Apply UDF that mutates the batches. - ds = ds.map_batches(mutate, batch_size=batch_size) - assert [row["value"] for row in ds.iter_rows()] == list(range(1, num_rows + 1)) - - -@pytest.mark.parametrize( - "num_rows,num_blocks,batch_size", - [ - (10, 5, 2), - (10, 1, 10), - (12, 3, 2), - ], -) -def test_map_batches_batch_zero_copy( - ray_start_regular_shared, num_rows, num_blocks, batch_size -): - # Test that batches are zero-copy read-only views when zero_copy_batch=True. - def mutate(df): - # Check that batch is read-only. - assert not df.values.flags.writeable - df["value"] += 1 - return df - - ds = ray.data.range_table(num_rows, parallelism=num_blocks).repartition(num_blocks) - # Convert to Pandas blocks. - ds = ds.map_batches(lambda df: df, batch_format="pandas", batch_size=None) - ds.fully_executed() - - # Apply UDF that mutates the batches, which should fail since the batch is - # read-only. - with pytest.raises(ValueError, match="tried to mutate a zero-copy read-only batch"): - ds = ds.map_batches(mutate, batch_size=batch_size, zero_copy_batch=True) - ds.fully_executed() - - -BLOCK_BUNDLING_TEST_CASES = [ - (block_size, batch_size) - for batch_size in range(1, 8) - for block_size in range(1, 2 * batch_size + 1) -] - - -@pytest.mark.parametrize("block_size,batch_size", BLOCK_BUNDLING_TEST_CASES) -def test_map_batches_block_bundling_auto( - ray_start_regular_shared, block_size, batch_size -): - # Ensure that we test at least 2 batches worth of blocks. - num_blocks = max(10, 2 * batch_size // block_size) - ds = ray.data.range(num_blocks * block_size, parallelism=num_blocks) - # Confirm that we have the expected number of initial blocks. - assert ds.num_blocks() == num_blocks - - # Blocks should be bundled up to the batch size. - ds1 = ds.map_batches(lambda x: x, batch_size=batch_size).fully_executed() - assert ds1.num_blocks() == math.ceil(num_blocks / max(batch_size // block_size, 1)) - - # Blocks should not be bundled up when batch_size is not specified. - ds2 = ds.map_batches(lambda x: x).fully_executed() - assert ds2.num_blocks() == num_blocks - - -@pytest.mark.parametrize( - "block_sizes,batch_size,expected_num_blocks", - [ - ([1, 2], 3, 1), - ([2, 2, 1], 3, 2), - ([1, 2, 3, 4], 4, 3), - ([3, 1, 1, 3], 4, 2), - ([2, 4, 1, 8], 4, 4), - ([1, 1, 1, 1], 4, 1), - ([1, 0, 3, 2], 4, 2), - ([4, 4, 4, 4], 4, 4), - ], -) -def test_map_batches_block_bundling_skewed_manual( - ray_start_regular_shared, block_sizes, batch_size, expected_num_blocks -): - num_blocks = len(block_sizes) - ds = ray.data.from_pandas( - [pd.DataFrame({"a": [1] * block_size}) for block_size in block_sizes] - ) - # Confirm that we have the expected number of initial blocks. - assert ds.num_blocks() == num_blocks - ds = ds.map_batches(lambda x: x, batch_size=batch_size).fully_executed() - - # Blocks should be bundled up to the batch size. - assert ds.num_blocks() == expected_num_blocks - - -BLOCK_BUNDLING_SKEWED_TEST_CASES = [ - (block_sizes, batch_size) - for batch_size in range(1, 4) - for num_blocks in range(1, batch_size + 1) - for block_sizes in itertools.product( - range(1, 2 * batch_size + 1), repeat=num_blocks - ) -] - - -@pytest.mark.parametrize("block_sizes,batch_size", BLOCK_BUNDLING_SKEWED_TEST_CASES) -def test_map_batches_block_bundling_skewed_auto( - ray_start_regular_shared, block_sizes, batch_size -): - num_blocks = len(block_sizes) - ds = ray.data.from_pandas( - [pd.DataFrame({"a": [1] * block_size}) for block_size in block_sizes] - ) - # Confirm that we have the expected number of initial blocks. - assert ds.num_blocks() == num_blocks - ds = ds.map_batches(lambda x: x, batch_size=batch_size).fully_executed() - curr = 0 - num_out_blocks = 0 - for block_size in block_sizes: - if curr > 0 and curr + block_size > batch_size: - num_out_blocks += 1 - curr = 0 - curr += block_size - if curr > 0: - num_out_blocks += 1 - - # Blocks should be bundled up to the batch size. - assert ds.num_blocks() == num_out_blocks - - -def test_map_with_mismatched_columns(ray_start_regular_shared): - def bad_fn(row): - if row > 5: - return {"a": "hello1"} - else: - return {"b": "hello1"} - - def good_fn(row): - if row > 5: - return {"a": "hello1", "b": "hello2"} - else: - return {"b": "hello2", "a": "hello1"} - - ds = ray.data.range(10, parallelism=1) - error_message = "Current row has different columns compared to previous rows." - with pytest.raises(ValueError) as e: - ds.map(bad_fn).fully_executed() - assert error_message in str(e.value) - ds_map = ds.map(good_fn) - assert ds_map.take() == [{"a": "hello1", "b": "hello2"} for _ in range(10)] - - -def test_union(ray_start_regular_shared): - ds = ray.data.range(20, parallelism=10) - - # Test lazy union. - ds = ds.union(ds, ds, ds, ds) - assert ds.num_blocks() == 50 - assert ds.count() == 100 - assert ds.sum() == 950 - - ds = ds.union(ds) - assert ds.count() == 200 - assert ds.sum() == (950 * 2) - - # Test materialized union. - ds2 = ray.data.from_items([1, 2, 3, 4, 5]) - assert ds2.count() == 5 - ds2 = ds2.union(ds2) - assert ds2.count() == 10 - ds2 = ds2.union(ds) - assert ds2.count() == 210 - - -def test_from_dask(ray_start_regular_shared): - import dask.dataframe as dd - - df = pd.DataFrame({"one": list(range(100)), "two": list(range(100))}) - ddf = dd.from_pandas(df, npartitions=10) - ds = ray.data.from_dask(ddf) - dfds = ds.to_pandas() - assert df.equals(dfds) - - -@pytest.mark.parametrize("ds_format", ["pandas", "arrow"]) -def test_to_dask(ray_start_regular_shared, ds_format): - from ray.util.dask import ray_dask_get - - df1 = pd.DataFrame({"one": [1, 2, 3], "two": ["a", "b", "c"]}) - df2 = pd.DataFrame({"one": [4, 5, 6], "two": ["e", "f", "g"]}) - df = pd.concat([df1, df2]) - ds = ray.data.from_pandas([df1, df2]) - if ds_format == "arrow": - ds = ds.map_batches(lambda df: df, batch_format="pyarrow", batch_size=None) - ddf = ds.to_dask() - meta = ddf._meta - # Check metadata. - assert isinstance(meta, pd.DataFrame) - assert meta.empty - assert list(meta.columns) == ["one", "two"] - assert list(meta.dtypes) == [np.int64, object] - # Explicit Dask-on-Ray - assert df.equals(ddf.compute(scheduler=ray_dask_get)) - # Implicit Dask-on-Ray. - assert df.equals(ddf.compute()) - - # Explicit metadata. - df1["two"] = df1["two"].astype(pd.StringDtype()) - df2["two"] = df2["two"].astype(pd.StringDtype()) - df = pd.concat([df1, df2]) - ds = ray.data.from_pandas([df1, df2]) - if ds_format == "arrow": - ds = ds.map_batches(lambda df: df, batch_format="pyarrow", batch_size=None) - ddf = ds.to_dask( - meta=pd.DataFrame( - {"one": pd.Series(dtype=np.int16), "two": pd.Series(dtype=pd.StringDtype())} - ), - ) - meta = ddf._meta - # Check metadata. - assert isinstance(meta, pd.DataFrame) - assert meta.empty - assert list(meta.columns) == ["one", "two"] - assert list(meta.dtypes) == [np.int16, pd.StringDtype()] - # Explicit Dask-on-Ray - assert df.equals(ddf.compute(scheduler=ray_dask_get)) - # Implicit Dask-on-Ray. - assert df.equals(ddf.compute()) - - -def test_to_dask_tensor_column_cast_pandas(ray_start_regular_shared): - # Check that tensor column casting occurs when converting a Dataset to a Dask - # DataFrame. - data = np.arange(12).reshape((3, 2, 2)) - ctx = ray.data.context.DatasetContext.get_current() - original = ctx.enable_tensor_extension_casting - try: - ctx.enable_tensor_extension_casting = True - in_df = pd.DataFrame({"a": TensorArray(data)}) - ds = ray.data.from_pandas(in_df) - dtypes = ds.schema().types - assert len(dtypes) == 1 - assert isinstance(dtypes[0], TensorDtype) - out_df = ds.to_dask().compute() - assert out_df["a"].dtype.type is np.object_ - expected_df = pd.DataFrame({"a": list(data)}) - pd.testing.assert_frame_equal(out_df, expected_df) - finally: - ctx.enable_tensor_extension_casting = original - - -def test_to_dask_tensor_column_cast_arrow(ray_start_regular_shared): - # Check that tensor column casting occurs when converting a Dataset to a Dask - # DataFrame. - data = np.arange(12).reshape((3, 2, 2)) - ctx = ray.data.context.DatasetContext.get_current() - original = ctx.enable_tensor_extension_casting - try: - ctx.enable_tensor_extension_casting = True - in_table = pa.table({"a": ArrowTensorArray.from_numpy(data)}) - ds = ray.data.from_arrow(in_table) - dtype = ds.schema().field(0).type - assert isinstance(dtype, ArrowTensorType) - out_df = ds.to_dask().compute() - assert out_df["a"].dtype.type is np.object_ - expected_df = pd.DataFrame({"a": list(data)}) - pd.testing.assert_frame_equal(out_df, expected_df) - finally: - ctx.enable_tensor_extension_casting = original - - -def test_from_modin(ray_start_regular_shared): - import modin.pandas as mopd - - df = pd.DataFrame( - {"one": list(range(100)), "two": list(range(100))}, - ) - modf = mopd.DataFrame(df) - ds = ray.data.from_modin(modf) - dfds = ds.to_pandas() - assert df.equals(dfds) - - -def test_to_modin(ray_start_regular_shared): - # create two modin dataframes - # one directly from a pandas dataframe, and - # another from ray.dataset created from the original pandas dataframe - # - import modin.pandas as mopd - - df = pd.DataFrame( - {"one": list(range(100)), "two": list(range(100))}, - ) - modf1 = mopd.DataFrame(df) - ds = ray.data.from_pandas([df]) - modf2 = ds.to_modin() - assert modf1.equals(modf2) - - -@pytest.mark.parametrize("pipelined", [False, True]) -def test_iter_tf_batches(ray_start_regular_shared, pipelined): - df1 = pd.DataFrame( - {"one": [1, 2, 3], "two": [1.0, 2.0, 3.0], "label": [1.0, 2.0, 3.0]} - ) - df2 = pd.DataFrame( - {"one": [4, 5, 6], "two": [4.0, 5.0, 6.0], "label": [4.0, 5.0, 6.0]} - ) - df3 = pd.DataFrame({"one": [7, 8], "two": [7.0, 8.0], "label": [7.0, 8.0]}) - df = pd.concat([df1, df2, df3]) - ds = ray.data.from_pandas([df1, df2, df3]) - ds = maybe_pipeline(ds, pipelined) - - num_epochs = 1 if pipelined else 2 - for _ in range(num_epochs): - iterations = [] - for batch in ds.iter_tf_batches(batch_size=3): - iterations.append( - np.stack((batch["one"], batch["two"], batch["label"]), axis=1) - ) - combined_iterations = np.concatenate(iterations) - np.testing.assert_array_equal(np.sort(df.values), np.sort(combined_iterations)) - - -@pytest.mark.parametrize("pipelined", [False, True]) -def test_iter_tf_batches_tensor_ds(ray_start_regular_shared, pipelined): - arr1 = np.arange(12).reshape((3, 2, 2)) - arr2 = np.arange(12, 24).reshape((3, 2, 2)) - arr = np.concatenate((arr1, arr2)) - ds = ray.data.from_numpy([arr1, arr2]) - ds = maybe_pipeline(ds, pipelined) - - num_epochs = 1 if pipelined else 2 - for _ in range(num_epochs): - iterations = [] - for batch in ds.iter_tf_batches(batch_size=2): - iterations.append(batch) - combined_iterations = np.concatenate(iterations) - np.testing.assert_array_equal(arr, combined_iterations) - - -def test_block_builder_for_block(ray_start_regular_shared): - # list - builder = BlockBuilder.for_block(list()) - builder.add_block([1, 2]) - assert builder.build() == [1, 2] - builder.add_block([3, 4]) - assert builder.build() == [1, 2, 3, 4] - - # pandas dataframe - builder = BlockBuilder.for_block(pd.DataFrame()) - b1 = pd.DataFrame({"A": [1], "B": ["a"]}) - builder.add_block(b1) - assert builder.build().equals(b1) - b2 = pd.DataFrame({"A": [2, 3], "B": ["c", "d"]}) - builder.add_block(b2) - expected = pd.DataFrame({"A": [1, 2, 3], "B": ["a", "c", "d"]}) - assert builder.build().equals(expected) - - # pyarrow table - builder = BlockBuilder.for_block(pa.Table.from_arrays(list())) - b1 = pa.Table.from_pydict({"A": [1], "B": ["a"]}) - builder.add_block(b1) - builder.build().equals(b1) - b2 = pa.Table.from_pydict({"A": [2, 3], "B": ["c", "d"]}) - builder.add_block(b2) - expected = pa.Table.from_pydict({"A": [1, 2, 3], "B": ["a", "c", "d"]}) - builder.build().equals(expected) - - # wrong type - with pytest.raises(TypeError): - BlockBuilder.for_block(str()) - - -def test_grouped_dataset_repr(ray_start_regular_shared): - ds = ray.data.from_items([{"key": "spam"}, {"key": "ham"}, {"key": "spam"}]) - assert repr(ds.groupby("key")) == f"GroupedDataset(dataset={ds!r}, key='key')" - - -def test_groupby_arrow(ray_start_regular_shared, use_push_based_shuffle): - # Test empty dataset. - agg_ds = ( - ray.data.range_table(10) - .filter(lambda r: r["value"] > 10) - .groupby("value") - .count() - ) - assert agg_ds.count() == 0 - - -def test_groupby_errors(ray_start_regular_shared): - ds = ray.data.range(100) - - ds.groupby(None).count().show() # OK - ds.groupby(lambda x: x % 2).count().show() # OK - with pytest.raises(ValueError): - ds.groupby("foo").count().show() - - ds = ray.data.range_table(100) - ds.groupby(None).count().show() # OK - with pytest.raises(ValueError): - ds.groupby(lambda x: x % 2).count().show() - - -def test_agg_errors(ray_start_regular_shared): - ds = ray.data.range(100) - from ray.data.aggregate import Max - - ds.aggregate(Max()) # OK - ds.aggregate(Max(lambda x: x)) # OK - with pytest.raises(ValueError): - ds.aggregate(Max("foo")) - - ds = ray.data.range_table(100) - ds.aggregate(Max("value")) # OK - with pytest.raises(ValueError): - ds.aggregate(Max()) - with pytest.raises(ValueError): - ds.aggregate(Max(lambda x: x)) - with pytest.raises(ValueError): - ds.aggregate(Max("bad_field")) - - -@pytest.mark.parametrize("num_parts", [1, 30]) -def test_groupby_agg_name_conflict(ray_start_regular_shared, num_parts): - # Test aggregation name conflict. - xs = list(range(100)) - grouped_ds = ( - ray.data.from_items([{"A": (x % 3), "B": x} for x in xs]) - .repartition(num_parts) - .groupby("A") - ) - agg_ds = grouped_ds.aggregate( - AggregateFn( - init=lambda k: [0, 0], - accumulate_row=lambda a, r: [a[0] + r["B"], a[1] + 1], - merge=lambda a1, a2: [a1[0] + a2[0], a1[1] + a2[1]], - finalize=lambda a: a[0] / a[1], - name="foo", - ), - AggregateFn( - init=lambda k: [0, 0], - accumulate_row=lambda a, r: [a[0] + r["B"], a[1] + 1], - merge=lambda a1, a2: [a1[0] + a2[0], a1[1] + a2[1]], - finalize=lambda a: a[0] / a[1], - name="foo", - ), - ) - assert agg_ds.count() == 3 - assert [row.as_pydict() for row in agg_ds.sort("A").iter_rows()] == [ - {"A": 0, "foo": 49.5, "foo_2": 49.5}, - {"A": 1, "foo": 49.0, "foo_2": 49.0}, - {"A": 2, "foo": 50.0, "foo_2": 50.0}, - ] - - -@pytest.mark.parametrize("num_parts", [1, 30]) -@pytest.mark.parametrize("ds_format", ["arrow", "pandas"]) -def test_groupby_tabular_count( - ray_start_regular_shared, ds_format, num_parts, use_push_based_shuffle -): - # Test built-in count aggregation - seed = int(time.time()) - print(f"Seeding RNG for test_groupby_arrow_count with: {seed}") - random.seed(seed) - xs = list(range(100)) - random.shuffle(xs) - - def _to_pandas(ds): - return ds.map_batches(lambda x: x, batch_size=None, batch_format="pandas") - - ds = ray.data.from_items([{"A": (x % 3), "B": x} for x in xs]).repartition( - num_parts - ) - if ds_format == "pandas": - ds = _to_pandas(ds) - agg_ds = ds.groupby("A").count() - assert agg_ds.count() == 3 - assert [row.as_pydict() for row in agg_ds.sort("A").iter_rows()] == [ - {"A": 0, "count()": 34}, - {"A": 1, "count()": 33}, - {"A": 2, "count()": 33}, - ] - - -@pytest.mark.parametrize("num_parts", [1, 30]) -@pytest.mark.parametrize("ds_format", ["arrow", "pandas"]) -def test_groupby_tabular_sum( - ray_start_regular_shared, ds_format, num_parts, use_push_based_shuffle -): - # Test built-in sum aggregation - seed = int(time.time()) - print(f"Seeding RNG for test_groupby_tabular_sum with: {seed}") - random.seed(seed) - xs = list(range(100)) - random.shuffle(xs) - - def _to_pandas(ds): - return ds.map_batches(lambda x: x, batch_size=None, batch_format="pandas") - - ds = ray.data.from_items([{"A": (x % 3), "B": x} for x in xs]).repartition( - num_parts - ) - if ds_format == "pandas": - ds = _to_pandas(ds) - - agg_ds = ds.groupby("A").sum("B") - assert agg_ds.count() == 3 - assert [row.as_pydict() for row in agg_ds.sort("A").iter_rows()] == [ - {"A": 0, "sum(B)": 1683}, - {"A": 1, "sum(B)": 1617}, - {"A": 2, "sum(B)": 1650}, - ] - - # Test built-in sum aggregation with nans - ds = ray.data.from_items( - [{"A": (x % 3), "B": x} for x in xs] + [{"A": 0, "B": None}] - ).repartition(num_parts) - if ds_format == "pandas": - ds = _to_pandas(ds) - nan_grouped_ds = ds.groupby("A") - nan_agg_ds = nan_grouped_ds.sum("B") - assert nan_agg_ds.count() == 3 - assert [row.as_pydict() for row in nan_agg_ds.sort("A").iter_rows()] == [ - {"A": 0, "sum(B)": 1683}, - {"A": 1, "sum(B)": 1617}, - {"A": 2, "sum(B)": 1650}, - ] - # Test ignore_nulls=False - nan_agg_ds = nan_grouped_ds.sum("B", ignore_nulls=False) - assert nan_agg_ds.count() == 3 - pd.testing.assert_frame_equal( - nan_agg_ds.sort("A").to_pandas(), - pd.DataFrame( - { - "A": [0, 1, 2], - "sum(B)": [None, 1617, 1650], - } - ), - check_dtype=False, - ) - # Test all nans - ds = ray.data.from_items([{"A": (x % 3), "B": None} for x in xs]).repartition( - num_parts - ) - if ds_format == "pandas": - ds = _to_pandas(ds) - nan_agg_ds = ds.groupby("A").sum("B") - assert nan_agg_ds.count() == 3 - pd.testing.assert_frame_equal( - nan_agg_ds.sort("A").to_pandas(), - pd.DataFrame( - { - "A": [0, 1, 2], - "sum(B)": [None, None, None], - } - ), - ) - - -@pytest.mark.parametrize("num_parts", [1, 30]) -@pytest.mark.parametrize("ds_format", ["arrow", "pandas"]) -def test_global_tabular_sum(ray_start_regular_shared, ds_format, num_parts): - seed = int(time.time()) - print(f"Seeding RNG for test_global_arrow_sum with: {seed}") - random.seed(seed) - xs = list(range(100)) - random.shuffle(xs) - - def _to_pandas(ds): - return ds.map_batches(lambda x: x, batch_size=None, batch_format="pandas") - - # Test built-in global sum aggregation - ds = ray.data.from_items([{"A": x} for x in xs]).repartition(num_parts) - if ds_format == "pandas": - ds = _to_pandas(ds) - assert ds.sum("A") == 4950 - - # Test empty dataset - ds = ray.data.range_table(10) - if ds_format == "pandas": - ds = _to_pandas(ds) - assert ds.filter(lambda r: r["value"] > 10).sum("value") is None - - # Test built-in global sum aggregation with nans - nan_ds = ray.data.from_items([{"A": x} for x in xs] + [{"A": None}]).repartition( - num_parts - ) - if ds_format == "pandas": - nan_ds = _to_pandas(nan_ds) - assert nan_ds.sum("A") == 4950 - # Test ignore_nulls=False - assert nan_ds.sum("A", ignore_nulls=False) is None - # Test all nans - nan_ds = ray.data.from_items([{"A": None}] * len(xs)).repartition(num_parts) - if ds_format == "pandas": - nan_ds = _to_pandas(nan_ds) - assert nan_ds.sum("A") is None - assert nan_ds.sum("A", ignore_nulls=False) is None - - -@pytest.mark.parametrize("num_parts", [1, 30]) -@pytest.mark.parametrize("ds_format", ["arrow", "pandas"]) -def test_groupby_tabular_min(ray_start_regular_shared, ds_format, num_parts): - # Test built-in min aggregation - seed = int(time.time()) - print(f"Seeding RNG for test_groupby_tabular_min with: {seed}") - random.seed(seed) - xs = list(range(100)) - random.shuffle(xs) - - def _to_pandas(ds): - return ds.map_batches(lambda x: x, batch_size=None, batch_format="pandas") - - ds = ray.data.from_items([{"A": (x % 3), "B": x} for x in xs]).repartition( - num_parts - ) - if ds_format == "pandas": - ds = _to_pandas(ds) - - agg_ds = ds.groupby("A").min("B") - assert agg_ds.count() == 3 - assert [row.as_pydict() for row in agg_ds.sort("A").iter_rows()] == [ - {"A": 0, "min(B)": 0}, - {"A": 1, "min(B)": 1}, - {"A": 2, "min(B)": 2}, - ] - - # Test built-in min aggregation with nans - ds = ray.data.from_items( - [{"A": (x % 3), "B": x} for x in xs] + [{"A": 0, "B": None}] - ).repartition(num_parts) - if ds_format == "pandas": - ds = _to_pandas(ds) - nan_grouped_ds = ds.groupby("A") - nan_agg_ds = nan_grouped_ds.min("B") - assert nan_agg_ds.count() == 3 - assert [row.as_pydict() for row in nan_agg_ds.sort("A").iter_rows()] == [ - {"A": 0, "min(B)": 0}, - {"A": 1, "min(B)": 1}, - {"A": 2, "min(B)": 2}, - ] - # Test ignore_nulls=False - nan_agg_ds = nan_grouped_ds.min("B", ignore_nulls=False) - assert nan_agg_ds.count() == 3 - pd.testing.assert_frame_equal( - nan_agg_ds.sort("A").to_pandas(), - pd.DataFrame( - { - "A": [0, 1, 2], - "min(B)": [None, 1, 2], - } - ), - check_dtype=False, - ) - # Test all nans - ds = ray.data.from_items([{"A": (x % 3), "B": None} for x in xs]).repartition( - num_parts - ) - if ds_format == "pandas": - ds = _to_pandas(ds) - nan_agg_ds = ds.groupby("A").min("B") - assert nan_agg_ds.count() == 3 - pd.testing.assert_frame_equal( - nan_agg_ds.sort("A").to_pandas(), - pd.DataFrame( - { - "A": [0, 1, 2], - "min(B)": [None, None, None], - } - ), - check_dtype=False, - ) - - -@pytest.mark.parametrize("num_parts", [1, 30]) -@pytest.mark.parametrize("ds_format", ["arrow", "pandas"]) -def test_global_tabular_min(ray_start_regular_shared, ds_format, num_parts): - seed = int(time.time()) - print(f"Seeding RNG for test_global_arrow_min with: {seed}") - random.seed(seed) - xs = list(range(100)) - random.shuffle(xs) - - def _to_pandas(ds): - return ds.map_batches(lambda x: x, batch_size=None, batch_format="pandas") - - # Test built-in global min aggregation - ds = ray.data.from_items([{"A": x} for x in xs]).repartition(num_parts) - if ds_format == "pandas": - ds = _to_pandas(ds) - assert ds.min("A") == 0 - - # Test empty dataset - ds = ray.data.range_table(10) - if ds_format == "pandas": - ds = _to_pandas(ds) - assert ds.filter(lambda r: r["value"] > 10).min("value") is None - - # Test built-in global min aggregation with nans - nan_ds = ray.data.from_items([{"A": x} for x in xs] + [{"A": None}]).repartition( - num_parts - ) - if ds_format == "pandas": - nan_ds = _to_pandas(nan_ds) - assert nan_ds.min("A") == 0 - # Test ignore_nulls=False - assert nan_ds.min("A", ignore_nulls=False) is None - # Test all nans - nan_ds = ray.data.from_items([{"A": None}] * len(xs)).repartition(num_parts) - if ds_format == "pandas": - nan_ds = _to_pandas(nan_ds) - assert nan_ds.min("A") is None - assert nan_ds.min("A", ignore_nulls=False) is None - - -@pytest.mark.parametrize("num_parts", [1, 30]) -@pytest.mark.parametrize("ds_format", ["arrow", "pandas"]) -def test_groupby_tabular_max(ray_start_regular_shared, ds_format, num_parts): - # Test built-in max aggregation - seed = int(time.time()) - print(f"Seeding RNG for test_groupby_tabular_max with: {seed}") - random.seed(seed) - xs = list(range(100)) - random.shuffle(xs) - - def _to_pandas(ds): - return ds.map_batches(lambda x: x, batch_size=None, batch_format="pandas") - - ds = ray.data.from_items([{"A": (x % 3), "B": x} for x in xs]).repartition( - num_parts - ) - if ds_format == "pandas": - ds = _to_pandas(ds) - - agg_ds = ds.groupby("A").max("B") - assert agg_ds.count() == 3 - assert [row.as_pydict() for row in agg_ds.sort("A").iter_rows()] == [ - {"A": 0, "max(B)": 99}, - {"A": 1, "max(B)": 97}, - {"A": 2, "max(B)": 98}, - ] - - # Test built-in min aggregation with nans - ds = ray.data.from_items( - [{"A": (x % 3), "B": x} for x in xs] + [{"A": 0, "B": None}] - ).repartition(num_parts) - if ds_format == "pandas": - ds = _to_pandas(ds) - nan_grouped_ds = ds.groupby("A") - nan_agg_ds = nan_grouped_ds.max("B") - assert nan_agg_ds.count() == 3 - assert [row.as_pydict() for row in nan_agg_ds.sort("A").iter_rows()] == [ - {"A": 0, "max(B)": 99}, - {"A": 1, "max(B)": 97}, - {"A": 2, "max(B)": 98}, - ] - # Test ignore_nulls=False - nan_agg_ds = nan_grouped_ds.max("B", ignore_nulls=False) - assert nan_agg_ds.count() == 3 - pd.testing.assert_frame_equal( - nan_agg_ds.sort("A").to_pandas(), - pd.DataFrame( - { - "A": [0, 1, 2], - "max(B)": [None, 97, 98], - } - ), - check_dtype=False, - ) - # Test all nans - ds = ray.data.from_items([{"A": (x % 3), "B": None} for x in xs]).repartition( - num_parts - ) - if ds_format == "pandas": - ds = _to_pandas(ds) - nan_agg_ds = ds.groupby("A").max("B") - assert nan_agg_ds.count() == 3 - pd.testing.assert_frame_equal( - nan_agg_ds.sort("A").to_pandas(), - pd.DataFrame( - { - "A": [0, 1, 2], - "max(B)": [None, None, None], - } - ), - check_dtype=False, - ) - - -@pytest.mark.parametrize("num_parts", [1, 30]) -@pytest.mark.parametrize("ds_format", ["arrow", "pandas"]) -def test_global_tabular_max(ray_start_regular_shared, ds_format, num_parts): - seed = int(time.time()) - print(f"Seeding RNG for test_global_arrow_max with: {seed}") - random.seed(seed) - xs = list(range(100)) - random.shuffle(xs) - - def _to_pandas(ds): - return ds.map_batches(lambda x: x, batch_size=None, batch_format="pandas") - - # Test built-in global max aggregation - ds = ray.data.from_items([{"A": x} for x in xs]).repartition(num_parts) - if ds_format == "pandas": - ds = _to_pandas(ds) - assert ds.max("A") == 99 - - # Test empty dataset - ds = ray.data.range_table(10) - if ds_format == "pandas": - ds = _to_pandas(ds) - assert ds.filter(lambda r: r["value"] > 10).max("value") is None - - # Test built-in global max aggregation with nans - nan_ds = ray.data.from_items([{"A": x} for x in xs] + [{"A": None}]).repartition( - num_parts - ) - if ds_format == "pandas": - nan_ds = _to_pandas(nan_ds) - assert nan_ds.max("A") == 99 - # Test ignore_nulls=False - assert nan_ds.max("A", ignore_nulls=False) is None - # Test all nans - nan_ds = ray.data.from_items([{"A": None}] * len(xs)).repartition(num_parts) - if ds_format == "pandas": - nan_ds = _to_pandas(nan_ds) - assert nan_ds.max("A") is None - assert nan_ds.max("A", ignore_nulls=False) is None - - -@pytest.mark.parametrize("num_parts", [1, 30]) -@pytest.mark.parametrize("ds_format", ["arrow", "pandas"]) -def test_groupby_tabular_mean(ray_start_regular_shared, ds_format, num_parts): - # Test built-in mean aggregation - seed = int(time.time()) - print(f"Seeding RNG for test_groupby_tabular_mean with: {seed}") - random.seed(seed) - xs = list(range(100)) - random.shuffle(xs) - - def _to_pandas(ds): - return ds.map_batches(lambda x: x, batch_size=None, batch_format="pandas") - - ds = ray.data.from_items([{"A": (x % 3), "B": x} for x in xs]).repartition( - num_parts - ) - if ds_format == "pandas": - ds = _to_pandas(ds) - - agg_ds = ds.groupby("A").mean("B") - assert agg_ds.count() == 3 - assert [row.as_pydict() for row in agg_ds.sort("A").iter_rows()] == [ - {"A": 0, "mean(B)": 49.5}, - {"A": 1, "mean(B)": 49.0}, - {"A": 2, "mean(B)": 50.0}, - ] - - # Test built-in mean aggregation with nans - ds = ray.data.from_items( - [{"A": (x % 3), "B": x} for x in xs] + [{"A": 0, "B": None}] - ).repartition(num_parts) - if ds_format == "pandas": - ds = _to_pandas(ds) - nan_grouped_ds = ds.groupby("A") - nan_agg_ds = nan_grouped_ds.mean("B") - assert nan_agg_ds.count() == 3 - assert [row.as_pydict() for row in nan_agg_ds.sort("A").iter_rows()] == [ - {"A": 0, "mean(B)": 49.5}, - {"A": 1, "mean(B)": 49.0}, - {"A": 2, "mean(B)": 50.0}, - ] - # Test ignore_nulls=False - nan_agg_ds = nan_grouped_ds.mean("B", ignore_nulls=False) - assert nan_agg_ds.count() == 3 - pd.testing.assert_frame_equal( - nan_agg_ds.sort("A").to_pandas(), - pd.DataFrame( - { - "A": [0, 1, 2], - "mean(B)": [None, 49.0, 50.0], - } - ), - check_dtype=False, - ) - # Test all nans - ds = ray.data.from_items([{"A": (x % 3), "B": None} for x in xs]).repartition( - num_parts - ) - if ds_format == "pandas": - ds = _to_pandas(ds) - nan_agg_ds = ds.groupby("A").mean("B") - assert nan_agg_ds.count() == 3 - pd.testing.assert_frame_equal( - nan_agg_ds.sort("A").to_pandas(), - pd.DataFrame( - { - "A": [0, 1, 2], - "mean(B)": [None, None, None], - } - ), - check_dtype=False, - ) - - -@pytest.mark.parametrize("num_parts", [1, 30]) -@pytest.mark.parametrize("ds_format", ["arrow", "pandas"]) -def test_global_tabular_mean(ray_start_regular_shared, ds_format, num_parts): - seed = int(time.time()) - print(f"Seeding RNG for test_global_arrow_mean with: {seed}") - random.seed(seed) - xs = list(range(100)) - random.shuffle(xs) - - def _to_pandas(ds): - return ds.map_batches(lambda x: x, batch_size=None, batch_format="pandas") - - # Test built-in global mean aggregation - ds = ray.data.from_items([{"A": x} for x in xs]).repartition(num_parts) - if ds_format == "pandas": - ds = _to_pandas(ds) - assert ds.mean("A") == 49.5 - - # Test empty dataset - ds = ray.data.range_table(10) - if ds_format == "pandas": - ds = _to_pandas(ds) - assert ds.filter(lambda r: r["value"] > 10).mean("value") is None - - # Test built-in global mean aggregation with nans - nan_ds = ray.data.from_items([{"A": x} for x in xs] + [{"A": None}]).repartition( - num_parts - ) - if ds_format == "pandas": - nan_ds = _to_pandas(nan_ds) - assert nan_ds.mean("A") == 49.5 - # Test ignore_nulls=False - assert nan_ds.mean("A", ignore_nulls=False) is None - # Test all nans - nan_ds = ray.data.from_items([{"A": None}] * len(xs)).repartition(num_parts) - if ds_format == "pandas": - nan_ds = _to_pandas(nan_ds) - assert nan_ds.mean("A") is None - assert nan_ds.mean("A", ignore_nulls=False) is None - - -@pytest.mark.parametrize("num_parts", [1, 30]) -@pytest.mark.parametrize("ds_format", ["arrow", "pandas"]) -def test_groupby_tabular_std(ray_start_regular_shared, ds_format, num_parts): - # Test built-in std aggregation - seed = int(time.time()) - print(f"Seeding RNG for test_groupby_tabular_std with: {seed}") - random.seed(seed) - xs = list(range(100)) - random.shuffle(xs) - - def _to_arrow(ds): - return ds.map_batches(lambda x: x, batch_size=None, batch_format="pyarrow") - - df = pd.DataFrame({"A": [x % 3 for x in xs], "B": xs}) - ds = ray.data.from_pandas(df).repartition(num_parts) - if ds_format == "arrow": - ds = _to_arrow(ds) - agg_ds = ds.groupby("A").std("B") - assert agg_ds.count() == 3 - result = agg_ds.to_pandas()["std(B)"].to_numpy() - expected = df.groupby("A")["B"].std().to_numpy() - np.testing.assert_array_almost_equal(result, expected) - # ddof of 0 - ds = ray.data.from_pandas(df).repartition(num_parts) - if ds_format == "arrow": - ds = _to_arrow(ds) - agg_ds = ds.groupby("A").std("B", ddof=0) - assert agg_ds.count() == 3 - result = agg_ds.to_pandas()["std(B)"].to_numpy() - expected = df.groupby("A")["B"].std(ddof=0).to_numpy() - np.testing.assert_array_almost_equal(result, expected) - - # Test built-in std aggregation with nans - nan_df = pd.DataFrame({"A": [x % 3 for x in xs] + [0], "B": xs + [None]}) - ds = ray.data.from_pandas(nan_df).repartition(num_parts) - if ds_format == "arrow": - ds = _to_arrow(ds) - nan_grouped_ds = ds.groupby("A") - nan_agg_ds = nan_grouped_ds.std("B") - assert nan_agg_ds.count() == 3 - result = nan_agg_ds.to_pandas()["std(B)"].to_numpy() - expected = nan_df.groupby("A")["B"].std().to_numpy() - np.testing.assert_array_almost_equal(result, expected) - # Test ignore_nulls=False - nan_agg_ds = nan_grouped_ds.std("B", ignore_nulls=False) - assert nan_agg_ds.count() == 3 - result = nan_agg_ds.to_pandas()["std(B)"].to_numpy() - expected = nan_df.groupby("A")["B"].std() - expected[0] = None - np.testing.assert_array_almost_equal(result, expected) - # Test all nans - nan_df = pd.DataFrame({"A": [x % 3 for x in xs], "B": [None] * len(xs)}) - ds = ray.data.from_pandas(nan_df).repartition(num_parts) - if ds_format == "arrow": - ds = _to_arrow(ds) - nan_agg_ds = ds.groupby("A").std("B", ignore_nulls=False) - assert nan_agg_ds.count() == 3 - result = nan_agg_ds.to_pandas()["std(B)"].to_numpy() - expected = pd.Series([None] * 3) - np.testing.assert_array_equal(result, expected) - - -@pytest.mark.parametrize("num_parts", [1, 30]) -@pytest.mark.parametrize("ds_format", ["arrow", "pandas"]) -def test_global_tabular_std(ray_start_regular_shared, ds_format, num_parts): - seed = int(time.time()) - print(f"Seeding RNG for test_global_arrow_std with: {seed}") - random.seed(seed) - xs = list(range(100)) - random.shuffle(xs) - - def _to_arrow(ds): - return ds.map_batches(lambda x: x, batch_size=None, batch_format="pyarrow") - - def _to_pandas(ds): - return ds.map_batches(lambda x: x, batch_size=None, batch_format="pandas") - - # Test built-in global max aggregation - df = pd.DataFrame({"A": xs}) - ds = ray.data.from_pandas(df).repartition(num_parts) - if ds_format == "arrow": - ds = _to_arrow(ds) - assert math.isclose(ds.std("A"), df["A"].std()) - assert math.isclose(ds.std("A", ddof=0), df["A"].std(ddof=0)) - - # Test empty dataset - ds = ray.data.from_pandas(pd.DataFrame({"A": []})) - if ds_format == "arrow": - ds = _to_arrow(ds) - assert ds.std("A") is None - # Test edge cases - ds = ray.data.from_pandas(pd.DataFrame({"A": [3]})) - if ds_format == "arrow": - ds = _to_arrow(ds) - assert ds.std("A") == 0 - - # Test built-in global std aggregation with nans - nan_df = pd.DataFrame({"A": xs + [None]}) - nan_ds = ray.data.from_pandas(nan_df).repartition(num_parts) - if ds_format == "arrow": - nan_ds = _to_arrow(nan_ds) - assert math.isclose(nan_ds.std("A"), nan_df["A"].std()) - # Test ignore_nulls=False - assert nan_ds.std("A", ignore_nulls=False) is None - # Test all nans - nan_ds = ray.data.from_items([{"A": None}] * len(xs)).repartition(num_parts) - if ds_format == "pandas": - nan_ds = _to_pandas(nan_ds) - assert nan_ds.std("A") is None - assert nan_ds.std("A", ignore_nulls=False) is None - - -@pytest.mark.parametrize("num_parts", [1, 30]) -def test_groupby_arrow_multicolumn(ray_start_regular_shared, num_parts): - # Test built-in mean aggregation on multiple columns - seed = int(time.time()) - print(f"Seeding RNG for test_groupby_arrow_multicolumn with: {seed}") - random.seed(seed) - xs = list(range(100)) - random.shuffle(xs) - df = pd.DataFrame({"A": [x % 3 for x in xs], "B": xs, "C": [2 * x for x in xs]}) - agg_ds = ( - ray.data.from_pandas(df).repartition(num_parts).groupby("A").mean(["B", "C"]) - ) - assert agg_ds.count() == 3 - assert [row.as_pydict() for row in agg_ds.sort("A").iter_rows()] == [ - {"A": 0, "mean(B)": 49.5, "mean(C)": 99.0}, - {"A": 1, "mean(B)": 49.0, "mean(C)": 98.0}, - {"A": 2, "mean(B)": 50.0, "mean(C)": 100.0}, - ] - - # Test that unspecified agg column ==> agg on all columns except for - # groupby keys. - agg_ds = ray.data.from_pandas(df).repartition(num_parts).groupby("A").mean() - assert agg_ds.count() == 3 - assert [row.as_pydict() for row in agg_ds.sort("A").iter_rows()] == [ - {"A": 0, "mean(B)": 49.5, "mean(C)": 99.0}, - {"A": 1, "mean(B)": 49.0, "mean(C)": 98.0}, - {"A": 2, "mean(B)": 50.0, "mean(C)": 100.0}, - ] - - # Test built-in global mean aggregation - df = pd.DataFrame({"A": xs, "B": [2 * x for x in xs]}) - result_row = ray.data.from_pandas(df).repartition(num_parts).mean(["A", "B"]) - assert result_row["mean(A)"] == df["A"].mean() - assert result_row["mean(B)"] == df["B"].mean() - - -def test_groupby_agg_bad_on(ray_start_regular_shared): - # Test bad on for groupby aggregation - xs = list(range(100)) - df = pd.DataFrame({"A": [x % 3 for x in xs], "B": xs, "C": [2 * x for x in xs]}) - # Wrong type. - with pytest.raises(TypeError): - ray.data.from_pandas(df).groupby("A").mean(5).fully_executed() - with pytest.raises(TypeError): - ray.data.from_pandas(df).groupby("A").mean([5]).fully_executed() - # Empty list. - with pytest.raises(ValueError): - ray.data.from_pandas(df).groupby("A").mean([]).fully_executed() - # Nonexistent column. - with pytest.raises(ValueError): - ray.data.from_pandas(df).groupby("A").mean("D").fully_executed() - with pytest.raises(ValueError): - ray.data.from_pandas(df).groupby("A").mean(["B", "D"]).fully_executed() - # Columns for simple Dataset. - with pytest.raises(ValueError): - ray.data.from_items(xs).groupby(lambda x: x % 3 == 0).mean("A").fully_executed() - - # Test bad on for global aggregation - # Wrong type. - with pytest.raises(TypeError): - ray.data.from_pandas(df).mean(5).fully_executed() - with pytest.raises(TypeError): - ray.data.from_pandas(df).mean([5]).fully_executed() - # Empty list. - with pytest.raises(ValueError): - ray.data.from_pandas(df).mean([]).fully_executed() - # Nonexistent column. - with pytest.raises(ValueError): - ray.data.from_pandas(df).mean("D").fully_executed() - with pytest.raises(ValueError): - ray.data.from_pandas(df).mean(["B", "D"]).fully_executed() - # Columns for simple Dataset. - with pytest.raises(ValueError): - ray.data.from_items(xs).mean("A").fully_executed() - - -@pytest.mark.parametrize("num_parts", [1, 30]) -def test_groupby_arrow_multi_agg(ray_start_regular_shared, num_parts): - seed = int(time.time()) - print(f"Seeding RNG for test_groupby_arrow_multi_agg with: {seed}") - random.seed(seed) - xs = list(range(100)) - random.shuffle(xs) - df = pd.DataFrame({"A": [x % 3 for x in xs], "B": xs}) - agg_ds = ( - ray.data.from_pandas(df) - .repartition(num_parts) - .groupby("A") - .aggregate( - Count(), - Sum("B"), - Min("B"), - Max("B"), - Mean("B"), - Std("B"), - ) - ) - assert agg_ds.count() == 3 - agg_df = agg_ds.to_pandas() - expected_grouped = df.groupby("A")["B"] - np.testing.assert_array_equal(agg_df["count()"].to_numpy(), [34, 33, 33]) - for agg in ["sum", "min", "max", "mean", "std"]: - result = agg_df[f"{agg}(B)"].to_numpy() - expected = getattr(expected_grouped, agg)().to_numpy() - if agg == "std": - np.testing.assert_array_almost_equal(result, expected) - else: - np.testing.assert_array_equal(result, expected) - # Test built-in global std aggregation - df = pd.DataFrame({"A": xs}) - - result_row = ( - ray.data.from_pandas(df) - .repartition(num_parts) - .aggregate( - Sum("A"), - Min("A"), - Max("A"), - Mean("A"), - Std("A"), - ) - ) - for agg in ["sum", "min", "max", "mean", "std"]: - result = result_row[f"{agg}(A)"] - expected = getattr(df["A"], agg)() - if agg == "std": - assert math.isclose(result, expected) - else: - assert result == expected - - -def test_groupby_simple(ray_start_regular_shared): - seed = int(time.time()) - print(f"Seeding RNG for test_groupby_simple with: {seed}") - random.seed(seed) - parallelism = 3 - xs = [ - ("A", 2), - ("A", 4), - ("A", 9), - ("B", 10), - ("B", 20), - ("C", 3), - ("C", 5), - ("C", 8), - ("C", 12), - ] - random.shuffle(xs) - ds = ray.data.from_items(xs, parallelism=parallelism) - - # Mean aggregation - agg_ds = ds.groupby(lambda r: r[0]).aggregate( - AggregateFn( - init=lambda k: (0, 0), - accumulate_row=lambda a, r: (a[0] + r[1], a[1] + 1), - merge=lambda a1, a2: (a1[0] + a2[0], a1[1] + a2[1]), - finalize=lambda a: a[0] / a[1], - ) - ) - assert agg_ds.count() == 3 - assert agg_ds.sort(key=lambda r: r[0]).take(3) == [("A", 5), ("B", 15), ("C", 7)] - - # Test None row - parallelism = 2 - xs = ["A", "A", "A", None, None, None, "B"] - random.shuffle(xs) - ds = ray.data.from_items(xs, parallelism=parallelism) - # Count aggregation - agg_ds = ds.groupby(lambda r: str(r)).aggregate( - AggregateFn( - init=lambda k: 0, - accumulate_row=lambda a, r: a + 1, - merge=lambda a1, a2: a1 + a2, - ) - ) - assert agg_ds.count() == 3 - assert agg_ds.sort(key=lambda r: str(r[0])).take(3) == [ - ("A", 3), - ("B", 1), - ("None", 3), - ] - - # Test empty dataset. - ds = ray.data.from_items([]) - agg_ds = ds.groupby(lambda r: r[0]).aggregate( - AggregateFn( - init=lambda k: 1 / 0, # should never reach here - accumulate_row=lambda a, r: 1 / 0, - merge=lambda a1, a2: 1 / 0, - finalize=lambda a: 1 / 0, - ) - ) - assert agg_ds.count() == 0 - assert agg_ds.take() == ds.take() - agg_ds = ray.data.range(10).filter(lambda r: r > 10).groupby(lambda r: r).count() - assert agg_ds.count() == 0 - - -@pytest.mark.parametrize("num_parts", [1, 30]) -def test_groupby_simple_count(ray_start_regular_shared, num_parts): - # Test built-in count aggregation - seed = int(time.time()) - print(f"Seeding RNG for test_groupby_simple_count with: {seed}") - random.seed(seed) - xs = list(range(100)) - random.shuffle(xs) - agg_ds = ( - ray.data.from_items(xs).repartition(num_parts).groupby(lambda x: x % 3).count() - ) - assert agg_ds.count() == 3 - assert agg_ds.sort(key=lambda r: r[0]).take(3) == [(0, 34), (1, 33), (2, 33)] - - -@pytest.mark.parametrize("num_parts", [1, 30]) -def test_groupby_simple_sum(ray_start_regular_shared, num_parts): - # Test built-in sum aggregation - seed = int(time.time()) - print(f"Seeding RNG for test_groupby_simple_sum with: {seed}") - random.seed(seed) - xs = list(range(100)) - random.shuffle(xs) - agg_ds = ( - ray.data.from_items(xs).repartition(num_parts).groupby(lambda x: x % 3).sum() - ) - assert agg_ds.count() == 3 - assert agg_ds.sort(key=lambda r: r[0]).take(3) == [(0, 1683), (1, 1617), (2, 1650)] - - # Test built-in sum aggregation with nans - nan_grouped_ds = ( - ray.data.from_items(xs + [None]) - .repartition(num_parts) - .groupby(lambda x: int(x or 0) % 3) - ) - nan_agg_ds = nan_grouped_ds.sum() - assert nan_agg_ds.count() == 3 - assert nan_agg_ds.sort(key=lambda r: r[0]).take(3) == [ - (0, 1683), - (1, 1617), - (2, 1650), - ] - # Test ignore_nulls=False - nan_agg_ds = nan_grouped_ds.sum(ignore_nulls=False) - assert nan_agg_ds.count() == 3 - assert nan_agg_ds.sort(key=lambda r: r[0]).take(3) == [ - (0, None), - (1, 1617), - (2, 1650), - ] - # Test all nans - nan_agg_ds = ( - ray.data.from_items([None] * len(xs)) - .repartition(num_parts) - .groupby(lambda x: 0) - .sum() - ) - assert nan_agg_ds.count() == 1 - assert nan_agg_ds.sort(key=lambda r: r[0]).take(1) == [(0, None)] - - # Test built-in global sum aggregation - assert ray.data.from_items(xs).repartition(num_parts).sum() == 4950 - assert ray.data.range(10).filter(lambda r: r > 10).sum() is None - - # Test built-in global sum aggregation with nans - nan_ds = ray.data.from_items(xs + [None]).repartition(num_parts) - assert nan_ds.sum() == 4950 - # Test ignore_nulls=False - assert nan_ds.sum(ignore_nulls=False) is None - # Test all nans - nan_ds = ray.data.from_items([None] * len(xs)).repartition(num_parts) - assert nan_ds.sum() is None - - -def test_map_batches_preserve_empty_blocks(ray_start_regular_shared): - ds = ray.data.range(10, parallelism=10) - ds = ds.map_batches(lambda x: []) - ds = ds.map_batches(lambda x: x) - assert ds.num_blocks() == 10, ds - - -def test_map_batches_combine_empty_blocks(ray_start_regular_shared): - xs = [x % 3 for x in list(range(100))] - - # ds1 has 1 block which contains 100 rows. - ds1 = ray.data.from_items(xs).repartition(1).sort().map_batches(lambda x: x) - assert ds1._block_num_rows() == [100] - - # ds2 has 30 blocks, but only 3 of them are non-empty - ds2 = ( - ray.data.from_items(xs) - .repartition(30) - .sort() - .map_batches(lambda x: x, batch_size=1) - ) - assert len(ds2._block_num_rows()) == 3 - count = sum(1 for x in ds2._block_num_rows() if x > 0) - assert count == 3 - - # The number of partitions should not affect the map_batches() result. - assert ds1.take_all() == ds2.take_all() - - -def test_groupby_map_groups_for_empty_dataset(ray_start_regular_shared): - ds = ray.data.from_items([]) - mapped = ds.groupby(lambda x: x % 3).map_groups(lambda x: [min(x) * min(x)]) - assert mapped.count() == 0 - assert mapped.take_all() == [] - - -def test_groupby_map_groups_merging_empty_result(ray_start_regular_shared): - ds = ray.data.from_items([1, 2, 3]) - # This needs to merge empty and non-empty results from different groups. - mapped = ds.groupby(lambda x: x).map_groups(lambda x: [] if x == [1] else x) - assert mapped.count() == 2 - assert mapped.take_all() == [2, 3] - - -def test_groupby_map_groups_merging_invalid_result(ray_start_regular_shared): - ds = ray.data.from_items([1, 2, 3]) - grouped = ds.groupby(lambda x: x) - - # The UDF returns None, which is invalid. - with pytest.raises(TypeError): - grouped.map_groups(lambda x: None if x == [1] else x).fully_executed() - - -@pytest.mark.parametrize("num_parts", [1, 2, 30]) -def test_groupby_map_groups_for_none_groupkey(ray_start_regular_shared, num_parts): - ds = ray.data.from_items(list(range(100))) - mapped = ( - ds.repartition(num_parts).groupby(None).map_groups(lambda x: [min(x) + max(x)]) - ) - assert mapped.count() == 1 - assert mapped.take_all() == [99] - - -@pytest.mark.parametrize("num_parts", [1, 2, 30]) -def test_groupby_map_groups_returning_empty_result(ray_start_regular_shared, num_parts): - xs = list(range(100)) - mapped = ( - ray.data.from_items(xs) - .repartition(num_parts) - .groupby(lambda x: x % 3) - .map_groups(lambda x: []) - ) - assert mapped.count() == 0 - assert mapped.take_all() == [] - - -def test_groupby_map_groups_perf(ray_start_regular_shared): - data_list = [x % 100 for x in range(5000000)] - ds = ray.data.from_pandas(pd.DataFrame({"A": data_list})) - start = time.perf_counter() - ds.groupby("A").map_groups(lambda df: df) - end = time.perf_counter() - # On a t3.2xlarge instance, it ran in about 5 seconds, so expecting it has to - # finish within about 10x of that time, unless something went wrong. - assert end - start < 60 - - -@pytest.mark.parametrize("num_parts", [1, 2, 3, 30]) -def test_groupby_map_groups_for_list(ray_start_regular_shared, num_parts): - seed = int(time.time()) - print(f"Seeding RNG for test_groupby_simple_count with: {seed}") - random.seed(seed) - xs = list(range(100)) - random.shuffle(xs) - mapped = ( - ray.data.from_items(xs) - .repartition(num_parts) - .groupby(lambda x: x % 3) - .map_groups(lambda x: [min(x) * min(x)]) - ) - assert mapped.count() == 3 - assert mapped.take_all() == [0, 1, 4] - - -@pytest.mark.parametrize("num_parts", [1, 2, 3, 30]) -def test_groupby_map_groups_for_pandas(ray_start_regular_shared, num_parts): - df = pd.DataFrame({"A": "a a b".split(), "B": [1, 1, 3], "C": [4, 6, 5]}) - grouped = ray.data.from_pandas(df).repartition(num_parts).groupby("A") - - # Normalize the numeric columns (i.e. B and C) for each group. - mapped = grouped.map_groups( - lambda g: g.apply( - lambda col: col / g[col.name].sum() if col.name in ["B", "C"] else col - ) - ) - - # The function (i.e. the normalization) performed on each group doesn't - # aggregate rows, so we still have 3 rows. - assert mapped.count() == 3 - expected = pd.DataFrame( - {"A": ["a", "a", "b"], "B": [0.5, 0.5, 1.000000], "C": [0.4, 0.6, 1.0]} - ) - assert mapped.to_pandas().equals(expected) - - -@pytest.mark.parametrize("num_parts", [1, 2, 3, 30]) -def test_groupby_map_groups_for_arrow(ray_start_regular_shared, num_parts): - at = pa.Table.from_pydict({"A": "a a b".split(), "B": [1, 1, 3], "C": [4, 6, 5]}) - grouped = ray.data.from_arrow(at).repartition(num_parts).groupby("A") - - # Normalize the numeric columns (i.e. B and C) for each group. - def normalize(at: pa.Table): - r = at.select("A") - sb = pa.compute.sum(at.column("B")).cast(pa.float64()) - r = r.append_column("B", pa.compute.divide(at.column("B"), sb)) - sc = pa.compute.sum(at.column("C")).cast(pa.float64()) - r = r.append_column("C", pa.compute.divide(at.column("C"), sc)) - return r - - mapped = grouped.map_groups(normalize, batch_format="pyarrow") - - # The function (i.e. the normalization) performed on each group doesn't - # aggregate rows, so we still have 3 rows. - assert mapped.count() == 3 - expected = pa.Table.from_pydict( - {"A": ["a", "a", "b"], "B": [0.5, 0.5, 1], "C": [0.4, 0.6, 1]} - ) - result = pa.Table.from_pandas(mapped.to_pandas()) - assert result.equals(expected) - - -def test_groupby_map_groups_for_numpy(ray_start_regular_shared): - ds = ray.data.from_items( - [ - {"group": 1, "value": 1}, - {"group": 1, "value": 2}, - {"group": 2, "value": 3}, - {"group": 2, "value": 4}, - ] - ) - - def func(group): - # Test output type is NumPy format. - return {"group": group["group"] + 1, "value": group["value"] + 1} - - ds = ds.groupby("group").map_groups(func, batch_format="numpy") - expected = pa.Table.from_pydict({"group": [2, 2, 3, 3], "value": [2, 3, 4, 5]}) - result = pa.Table.from_pandas(ds.to_pandas()) - assert result.equals(expected) - - -def test_groupby_map_groups_with_different_types(ray_start_regular_shared): - ds = ray.data.from_items( - [ - {"group": 1, "value": 1}, - {"group": 1, "value": 2}, - {"group": 2, "value": 3}, - {"group": 2, "value": 4}, - ] - ) - - def func(group): - # Test output type is Python list, different from input type. - return [group["value"][0]] - - ds = ds.groupby("group").map_groups(func) - assert sorted(ds.take()) == [1, 3] - - -@pytest.mark.parametrize("num_parts", [1, 30]) -def test_groupby_simple_min(ray_start_regular_shared, num_parts): - # Test built-in min aggregation - seed = int(time.time()) - print(f"Seeding RNG for test_groupby_simple_min with: {seed}") - random.seed(seed) - xs = list(range(100)) - random.shuffle(xs) - agg_ds = ( - ray.data.from_items(xs).repartition(num_parts).groupby(lambda x: x % 3).min() - ) - assert agg_ds.count() == 3 - assert agg_ds.sort(key=lambda r: r[0]).take(3) == [(0, 0), (1, 1), (2, 2)] - - # Test built-in min aggregation with nans - nan_grouped_ds = ( - ray.data.from_items(xs + [None]) - .repartition(num_parts) - .groupby(lambda x: int(x or 0) % 3) - ) - nan_agg_ds = nan_grouped_ds.min() - assert nan_agg_ds.count() == 3 - assert nan_agg_ds.sort(key=lambda r: r[0]).take(3) == [(0, 0), (1, 1), (2, 2)] - # Test ignore_nulls=False - nan_agg_ds = nan_grouped_ds.min(ignore_nulls=False) - assert nan_agg_ds.count() == 3 - assert nan_agg_ds.sort(key=lambda r: r[0]).take(3) == [(0, None), (1, 1), (2, 2)] - # Test all nans - nan_agg_ds = ( - ray.data.from_items([None] * len(xs)) - .repartition(num_parts) - .groupby(lambda x: 0) - .min() - ) - assert nan_agg_ds.count() == 1 - assert nan_agg_ds.sort(key=lambda r: r[0]).take(1) == [(0, None)] - - # Test built-in global min aggregation - assert ray.data.from_items(xs).repartition(num_parts).min() == 0 - assert ray.data.range(10).filter(lambda r: r > 10).min() is None - - # Test built-in global min aggregation with nans - nan_ds = ray.data.from_items(xs + [None]).repartition(num_parts) - assert nan_ds.min() == 0 - # Test ignore_nulls=False - assert nan_ds.min(ignore_nulls=False) is None - # Test all nans - nan_ds = ray.data.from_items([None] * len(xs)).repartition(num_parts) - assert nan_ds.min() is None - - -@pytest.mark.parametrize("num_parts", [1, 30]) -def test_groupby_simple_max(ray_start_regular_shared, num_parts): - # Test built-in max aggregation - seed = int(time.time()) - print(f"Seeding RNG for test_groupby_simple_max with: {seed}") - random.seed(seed) - xs = list(range(100)) - random.shuffle(xs) - agg_ds = ( - ray.data.from_items(xs).repartition(num_parts).groupby(lambda x: x % 3).max() - ) - assert agg_ds.count() == 3 - assert agg_ds.sort(key=lambda r: r[0]).take(3) == [(0, 99), (1, 97), (2, 98)] - - # Test built-in max aggregation with nans - nan_grouped_ds = ( - ray.data.from_items(xs + [None]) - .repartition(num_parts) - .groupby(lambda x: int(x or 0) % 3) - ) - nan_agg_ds = nan_grouped_ds.max() - assert nan_agg_ds.count() == 3 - assert nan_agg_ds.sort(key=lambda r: r[0]).take(3) == [(0, 99), (1, 97), (2, 98)] - # Test ignore_nulls=False - nan_agg_ds = nan_grouped_ds.max(ignore_nulls=False) - assert nan_agg_ds.count() == 3 - assert nan_agg_ds.sort(key=lambda r: r[0]).take(3) == [(0, None), (1, 97), (2, 98)] - # Test all nans - nan_agg_ds = ( - ray.data.from_items([None] * len(xs)) - .repartition(num_parts) - .groupby(lambda x: 0) - .max() - ) - assert nan_agg_ds.count() == 1 - assert nan_agg_ds.sort(key=lambda r: r[0]).take(1) == [(0, None)] - - # Test built-in global max aggregation - assert ray.data.from_items(xs).repartition(num_parts).max() == 99 - assert ray.data.range(10).filter(lambda r: r > 10).max() is None - - # Test built-in global max aggregation with nans - nan_ds = ray.data.from_items(xs + [None]).repartition(num_parts) - assert nan_ds.max() == 99 - # Test ignore_nulls=False - assert nan_ds.max(ignore_nulls=False) is None - # Test all nans - nan_ds = ray.data.from_items([None] * len(xs)).repartition(num_parts) - assert nan_ds.max() is None - - -@pytest.mark.parametrize("num_parts", [1, 30]) -def test_groupby_simple_mean(ray_start_regular_shared, num_parts): - # Test built-in mean aggregation - seed = int(time.time()) - print(f"Seeding RNG for test_groupby_simple_mean with: {seed}") - random.seed(seed) - xs = list(range(100)) - random.shuffle(xs) - agg_ds = ( - ray.data.from_items(xs).repartition(num_parts).groupby(lambda x: x % 3).mean() - ) - assert agg_ds.count() == 3 - assert agg_ds.sort(key=lambda r: r[0]).take(3) == [(0, 49.5), (1, 49.0), (2, 50.0)] - - # Test built-in mean aggregation with nans - nan_grouped_ds = ( - ray.data.from_items(xs + [None]) - .repartition(num_parts) - .groupby(lambda x: int(x or 0) % 3) - ) - nan_agg_ds = nan_grouped_ds.mean() - assert nan_agg_ds.count() == 3 - assert nan_agg_ds.sort(key=lambda r: r[0]).take(3) == [ - (0, 49.5), - (1, 49.0), - (2, 50.0), - ] - # Test ignore_nulls=False - nan_agg_ds = nan_grouped_ds.mean(ignore_nulls=False) - assert nan_agg_ds.count() == 3 - assert nan_agg_ds.sort(key=lambda r: r[0]).take(3) == [ - (0, None), - (1, 49.0), - (2, 50.0), - ] - # Test all nans - nan_agg_ds = ( - ray.data.from_items([None] * len(xs)) - .repartition(num_parts) - .groupby(lambda x: 0) - .mean() - ) - assert nan_agg_ds.count() == 1 - assert nan_agg_ds.sort(key=lambda r: r[0]).take(1) == [(0, None)] - - # Test built-in global mean aggregation - assert ray.data.from_items(xs).repartition(num_parts).mean() == 49.5 - # Test empty dataset - assert ray.data.range(10).filter(lambda r: r > 10).mean() is None - - # Test built-in global mean aggregation with nans - nan_ds = ray.data.from_items(xs + [None]).repartition(num_parts) - assert nan_ds.mean() == 49.5 - # Test ignore_nulls=False - assert nan_ds.mean(ignore_nulls=False) is None - # Test all nans - nan_ds = ray.data.from_items([None] * len(xs)).repartition(num_parts) - assert nan_ds.mean() is None - - -@pytest.mark.parametrize("num_parts", [1, 30]) -def test_groupby_simple_std(ray_start_regular_shared, num_parts): - # Test built-in std aggregation - seed = int(time.time()) - print(f"Seeding RNG for test_groupby_simple_std with: {seed}") - random.seed(seed) - xs = list(range(100)) - random.shuffle(xs) - agg_ds = ( - ray.data.from_items(xs).repartition(num_parts).groupby(lambda x: x % 3).std() - ) - assert agg_ds.count() == 3 - df = pd.DataFrame({"A": [x % 3 for x in xs], "B": xs}) - expected = df.groupby("A")["B"].std() - result = agg_ds.sort(key=lambda r: r[0]).take(3) - groups, stds = zip(*result) - result_df = pd.DataFrame({"A": list(groups), "B": list(stds)}) - result_df = result_df.set_index("A") - pd.testing.assert_series_equal(result_df["B"], expected) - # ddof of 0 - agg_ds = ( - ray.data.from_items(xs) - .repartition(num_parts) - .groupby(lambda x: x % 3) - .std(ddof=0) - ) - assert agg_ds.count() == 3 - df = pd.DataFrame({"A": [x % 3 for x in xs], "B": xs}) - expected = df.groupby("A")["B"].std(ddof=0) - result = agg_ds.sort(key=lambda r: r[0]).take(3) - groups, stds = zip(*result) - result_df = pd.DataFrame({"A": list(groups), "B": list(stds)}) - result_df = result_df.set_index("A") - pd.testing.assert_series_equal(result_df["B"], expected) - - # Test built-in std aggregation with nans - nan_grouped_ds = ( - ray.data.from_items(xs + [None]) - .repartition(num_parts) - .groupby(lambda x: int(x or 0) % 3) - ) - nan_agg_ds = nan_grouped_ds.std() - assert nan_agg_ds.count() == 3 - nan_df = pd.DataFrame({"A": [x % 3 for x in xs] + [0], "B": xs + [None]}) - expected = nan_df.groupby("A")["B"].std() - result = nan_agg_ds.sort(key=lambda r: r[0]).take(3) - groups, stds = zip(*result) - result_df = pd.DataFrame({"A": list(groups), "B": list(stds)}) - result_df = result_df.set_index("A") - pd.testing.assert_series_equal(result_df["B"], expected) - # Test ignore_nulls=False - nan_agg_ds = nan_grouped_ds.std(ignore_nulls=False) - assert nan_agg_ds.count() == 3 - expected = nan_df.groupby("A")["B"].std() - expected[0] = None - result = nan_agg_ds.sort(key=lambda r: r[0]).take(3) - groups, stds = zip(*result) - result_df = pd.DataFrame({"A": list(groups), "B": list(stds)}) - result_df = result_df.set_index("A") - pd.testing.assert_series_equal(result_df["B"], expected) - # Test all nans - nan_agg_ds = ( - ray.data.from_items([None] * len(xs)) - .repartition(num_parts) - .groupby(lambda x: 0) - .std(ignore_nulls=False) - ) - assert nan_agg_ds.count() == 1 - expected = pd.Series([None], name="B") - expected.index.rename("A", inplace=True) - result = nan_agg_ds.sort(key=lambda r: r[0]).take(1) - groups, stds = zip(*result) - result_df = pd.DataFrame({"A": list(groups), "B": list(stds)}) - result_df = result_df.set_index("A") - pd.testing.assert_series_equal(result_df["B"], expected) - - # Test built-in global std aggregation - assert math.isclose( - ray.data.from_items(xs).repartition(num_parts).std(), pd.Series(xs).std() - ) - # ddof of 0 - assert math.isclose( - ray.data.from_items(xs).repartition(num_parts).std(ddof=0), - pd.Series(xs).std(ddof=0), - ) - - # Test empty dataset - assert ray.data.from_items([]).std() is None - # Test edge cases - assert ray.data.from_items([3]).std() == 0 - - # Test built-in global std aggregation with nans - nan_ds = ray.data.from_items(xs + [None]).repartition(num_parts) - assert math.isclose(nan_ds.std(), pd.Series(xs).std()) - # Test ignore_nulls=False - assert nan_ds.std(ignore_nulls=False) is None - # Test all nans - nan_ds = ray.data.from_items([None] * len(xs)).repartition(num_parts) - assert nan_ds.std() is None - - -@pytest.mark.parametrize("num_parts", [1, 30]) -def test_groupby_simple_multilambda(ray_start_regular_shared, num_parts): - # Test built-in mean aggregation - seed = int(time.time()) - print(f"Seeding RNG for test_groupby_simple_multilambda with: {seed}") - random.seed(seed) - xs = list(range(100)) - random.shuffle(xs) - agg_ds = ( - ray.data.from_items([[x, 2 * x] for x in xs]) - .repartition(num_parts) - .groupby(lambda x: x[0] % 3) - .mean([lambda x: x[0], lambda x: x[1]]) - ) - assert agg_ds.count() == 3 - assert agg_ds.sort(key=lambda r: r[0]).take(3) == [ - (0, 49.5, 99.0), - (1, 49.0, 98.0), - (2, 50.0, 100.0), - ] - # Test built-in global mean aggregation - assert ray.data.from_items([[x, 2 * x] for x in xs]).repartition(num_parts).mean( - [lambda x: x[0], lambda x: x[1]] - ) == (49.5, 99.0) - assert ray.data.from_items([[x, 2 * x] for x in range(10)]).filter( - lambda r: r[0] > 10 - ).mean([lambda x: x[0], lambda x: x[1]]) == (None, None) - - -@pytest.mark.parametrize("num_parts", [1, 30]) -def test_groupby_simple_multi_agg(ray_start_regular_shared, num_parts): - seed = int(time.time()) - print(f"Seeding RNG for test_groupby_simple_multi_agg with: {seed}") - random.seed(seed) - xs = list(range(100)) - random.shuffle(xs) - df = pd.DataFrame({"A": [x % 3 for x in xs], "B": xs}) - agg_ds = ( - ray.data.from_items(xs) - .repartition(num_parts) - .groupby(lambda x: x % 3) - .aggregate( - Count(), - Sum(), - Min(), - Max(), - Mean(), - Std(), - ) - ) - assert agg_ds.count() == 3 - result = agg_ds.sort(key=lambda r: r[0]).take(3) - groups, counts, sums, mins, maxs, means, stds = zip(*result) - agg_df = pd.DataFrame( - { - "groups": list(groups), - "count": list(counts), - "sum": list(sums), - "min": list(mins), - "max": list(maxs), - "mean": list(means), - "std": list(stds), - } - ) - agg_df = agg_df.set_index("groups") - df = pd.DataFrame({"groups": [x % 3 for x in xs], "B": xs}) - expected_grouped = df.groupby("groups")["B"] - np.testing.assert_array_equal(agg_df["count"].to_numpy(), [34, 33, 33]) - for agg in ["sum", "min", "max", "mean", "std"]: - result = agg_df[agg].to_numpy() - expected = getattr(expected_grouped, agg)().to_numpy() - if agg == "std": - np.testing.assert_array_almost_equal(result, expected) - else: - np.testing.assert_array_equal(result, expected) - # Test built-in global multi-aggregation - result_row = ( - ray.data.from_items(xs) - .repartition(num_parts) - .aggregate( - Sum(), - Min(), - Max(), - Mean(), - Std(), - ) - ) - series = pd.Series(xs) - for idx, agg in enumerate(["sum", "min", "max", "mean", "std"]): - result = result_row[idx] - expected = getattr(series, agg)() - if agg == "std": - assert math.isclose(result, expected) - else: - assert result == expected - - -def test_column_name_type_check(ray_start_regular_shared): - df = pd.DataFrame({"1": np.random.rand(10), "a": np.random.rand(10)}) - ds = ray.data.from_pandas(df) - expected_str = "Dataset(num_blocks=1, num_rows=10, schema={1: float64, a: float64})" - assert str(ds) == expected_str, str(ds) - df = pd.DataFrame({1: np.random.rand(10), "a": np.random.rand(10)}) - with pytest.raises(ValueError): - ray.data.from_pandas(df) - - -def test_len(ray_start_regular_shared): - ds = ray.data.range(1) - with pytest.raises(AttributeError): - len(ds) - - -def test_random_sample(ray_start_regular_shared): - import math - - def ensure_sample_size_close(dataset, sample_percent=0.5): - r1 = ds.random_sample(sample_percent) - assert math.isclose( - r1.count(), int(ds.count() * sample_percent), rel_tol=2, abs_tol=2 - ) - - ds = ray.data.range(10, parallelism=2) - ensure_sample_size_close(ds) - - ds = ray.data.range_table(10, parallelism=2) - ensure_sample_size_close(ds) - - ds = ray.data.range_tensor(5, parallelism=2, shape=(2, 2)) - ensure_sample_size_close(ds) - - # imbalanced datasets - ds1 = ray.data.range(1, parallelism=1) - ds2 = ray.data.range(2, parallelism=1) - ds3 = ray.data.range(3, parallelism=1) - # noinspection PyTypeChecker - ds = ds1.union(ds2).union(ds3) - ensure_sample_size_close(ds) - # Small datasets - ds1 = ray.data.range(5, parallelism=5) - ensure_sample_size_close(ds1) - - -def test_random_sample_checks(ray_start_regular_shared): - with pytest.raises(ValueError): - # Cannot sample -1 - ray.data.range(1).random_sample(-1) - with pytest.raises(ValueError): - # Cannot sample from empty dataset - ray.data.range(0).random_sample(0.2) - with pytest.raises(ValueError): - # Cannot sample fraction > 1 - ray.data.range(1).random_sample(10) - - -def test_random_block_order_schema(ray_start_regular_shared): - df = pd.DataFrame({"a": np.random.rand(10), "b": np.random.rand(10)}) - ds = ray.data.from_pandas(df).randomize_block_order() - ds.schema().names == ["a", "b"] - - -def test_random_block_order(ray_start_regular_shared, restore_dataset_context): - ctx = DatasetContext.get_current() - ctx.execution_options.preserve_order = True - - # Test BlockList.randomize_block_order. - ds = ray.data.range(12).repartition(4) - ds = ds.randomize_block_order(seed=0) - - results = ds.take() - expected = [6, 7, 8, 0, 1, 2, 3, 4, 5, 9, 10, 11] - assert results == expected - - # Test LazyBlockList.randomize_block_order. - context = DatasetContext.get_current() - try: - original_optimize_fuse_read_stages = context.optimize_fuse_read_stages - context.optimize_fuse_read_stages = False - - lazy_blocklist_ds = ray.data.range(12, parallelism=4) - lazy_blocklist_ds = lazy_blocklist_ds.randomize_block_order(seed=0) - lazy_blocklist_results = lazy_blocklist_ds.take() - lazy_blocklist_expected = [6, 7, 8, 0, 1, 2, 3, 4, 5, 9, 10, 11] - assert lazy_blocklist_results == lazy_blocklist_expected - finally: - context.optimize_fuse_read_stages = original_optimize_fuse_read_stages - - -# NOTE: All tests above share a Ray cluster, while the tests below do not. These -# tests should only be carefully reordered to retain this invariant! - - -@pytest.mark.parametrize("pipelined", [False, True]) -def test_random_shuffle(shutdown_only, pipelined, use_push_based_shuffle): - def range(n, parallelism=200): - ds = ray.data.range(n, parallelism=parallelism) - if pipelined: - pipe = ds.repeat(2) - pipe.random_shuffle = pipe.random_shuffle_each_window - return pipe - else: - return ds - - r1 = range(100).random_shuffle().take(999) - r2 = range(100).random_shuffle().take(999) - assert r1 != r2, (r1, r2) - - r1 = range(100, parallelism=1).random_shuffle().take(999) - r2 = range(100, parallelism=1).random_shuffle().take(999) - assert r1 != r2, (r1, r2) - - # TODO(swang): fix this - if not use_push_based_shuffle: - if not pipelined: - assert range(100).random_shuffle(num_blocks=1).num_blocks() == 1 - r1 = range(100).random_shuffle(num_blocks=1).take(999) - r2 = range(100).random_shuffle(num_blocks=1).take(999) - assert r1 != r2, (r1, r2) - - r0 = range(100, parallelism=5).take(999) - r1 = range(100, parallelism=5).random_shuffle(seed=0).take(999) - r2 = range(100, parallelism=5).random_shuffle(seed=0).take(999) - r3 = range(100, parallelism=5).random_shuffle(seed=12345).take(999) - assert r1 == r2, (r1, r2) - assert r1 != r0, (r1, r0) - assert r1 != r3, (r1, r3) - - r0 = ray.data.range_table(100, parallelism=5).take(999) - r1 = ray.data.range_table(100, parallelism=5).random_shuffle(seed=0).take(999) - r2 = ray.data.range_table(100, parallelism=5).random_shuffle(seed=0).take(999) - assert r1 == r2, (r1, r2) - assert r1 != r0, (r1, r0) - - # Test move. - ds = range(100, parallelism=2) - r1 = ds.random_shuffle().take(999) - if pipelined: - with pytest.raises(RuntimeError): - ds = ds.map(lambda x: x).take(999) - else: - ds = ds.map(lambda x: x).take(999) - r2 = range(100).random_shuffle().take(999) - assert r1 != r2, (r1, r2) - - # Test empty dataset. - ds = ray.data.from_items([]) - r1 = ds.random_shuffle() - assert r1.count() == 0 - assert r1.take() == ds.take() - - -def test_random_shuffle_check_random(shutdown_only): - # Rows from the same input should not be contiguous in the final output. - num_files = 10 - num_rows = 100 - items = [i for i in range(num_files) for _ in range(num_rows)] - ds = ray.data.from_items(items, parallelism=num_files) - out = ds.random_shuffle().take(num_files * num_rows) - for i in range(num_files): - part = out[i * num_rows : (i + 1) * num_rows] - seen = set() - num_contiguous = 1 - prev = -1 - for x in part: - if prev != x: - prev = x - num_contiguous = 1 - else: - num_contiguous += 1 - assert num_contiguous < ( - num_rows / num_files - ), f"{part} contains too many contiguous rows from same input block" - seen.add(x) - assert ( - set(range(num_files)) == seen - ), f"{part} does not contain elements from all input blocks" - - # Rows from the same input should appear in a different order in the - # output. - num_files = 10 - num_rows = 100 - items = [j for i in range(num_files) for j in range(num_rows)] - ds = ray.data.from_items(items, parallelism=num_files) - out = ds.random_shuffle().take(num_files * num_rows) - for i in range(num_files): - part = out[i * num_rows : (i + 1) * num_rows] - num_increasing = 0 - prev = -1 - for x in part: - if x >= prev: - num_increasing += 1 - else: - assert num_increasing < ( - num_rows / num_files - ), f"{part} contains non-shuffled rows from input blocks" - num_increasing = 0 - prev = x - - -def test_unsupported_pyarrow_versions_check(shutdown_only, unsupported_pyarrow_version): - # Test that unsupported pyarrow versions cause an error to be raised upon the - # initial pyarrow use. - ray.init(runtime_env={"pip": [f"pyarrow=={unsupported_pyarrow_version}"]}) - - # Test Arrow-native creation APIs. - # Test range_table. - with pytest.raises(ImportError): - ray.data.range_table(10).take_all() - - # Test from_arrow. - with pytest.raises(ImportError): - ray.data.from_arrow(pa.table({"a": [1, 2]})) - - # Test read_parquet. - with pytest.raises(ImportError): - ray.data.read_parquet("example://iris.parquet").take_all() - - # Test from_numpy (we use Arrow for representing the tensors). - with pytest.raises(ImportError): - ray.data.from_numpy(np.arange(12).reshape((3, 2, 2))) - - -def test_unsupported_pyarrow_versions_check_disabled( - shutdown_only, - unsupported_pyarrow_version, - disable_pyarrow_version_check, -): - # Test that unsupported pyarrow versions DO NOT cause an error to be raised upon the - # initial pyarrow use when the version check is disabled. - ray.init( - runtime_env={ - "pip": [f"pyarrow=={unsupported_pyarrow_version}"], - "env_vars": {"RAY_DISABLE_PYARROW_VERSION_CHECK": "1"}, - }, - ) - - # Test Arrow-native creation APIs. - # Test range_table. - try: - ray.data.range_table(10).take_all() - except ImportError as e: - pytest.fail(f"_check_pyarrow_version failed unexpectedly: {e}") - - # Test from_arrow. - try: - ray.data.from_arrow(pa.table({"a": [1, 2]})) - except ImportError as e: - pytest.fail(f"_check_pyarrow_version failed unexpectedly: {e}") - - # Test read_parquet. - try: - ray.data.read_parquet("example://iris.parquet").take_all() - except ImportError as e: - pytest.fail(f"_check_pyarrow_version failed unexpectedly: {e}") - - # Test from_numpy (we use Arrow for representing the tensors). - try: - ray.data.from_numpy(np.arange(12).reshape((3, 2, 2))) - except ImportError as e: - pytest.fail(f"_check_pyarrow_version failed unexpectedly: {e}") - - -def test_random_shuffle_with_custom_resource(ray_start_cluster): - cluster = ray_start_cluster - # Create two nodes which have different custom resources. - cluster.add_node( - resources={"foo": 100}, - num_cpus=1, - ) - cluster.add_node(resources={"bar": 100}, num_cpus=1) - - ray.init(cluster.address) - - # Run dataset in "bar" nodes. - ds = ray.data.read_parquet( - "example://parquet_images_mini", - parallelism=2, - ray_remote_args={"resources": {"bar": 1}}, - ) - ds = ds.random_shuffle(resources={"bar": 1}).fully_executed() - assert "1 nodes used" in ds.stats() - assert "2 nodes used" not in ds.stats() - - -def test_read_write_local_node_ray_client(ray_start_cluster_enabled): - cluster = ray_start_cluster_enabled - cluster.add_node(num_cpus=4) - cluster.head_node._ray_params.ray_client_server_port = "10004" - cluster.head_node.start_ray_client_server() - address = "ray://localhost:10004" - - import tempfile - - data_path = tempfile.mkdtemp() - df = pd.DataFrame({"one": list(range(0, 10)), "two": list(range(10, 20))}) - path = os.path.join(data_path, "test.parquet") - df.to_parquet(path) - - # Read/write from Ray Client will result in error. - ray.init(address) - with pytest.raises(ValueError): - ds = ray.data.read_parquet("local://" + path).fully_executed() - ds = ray.data.from_pandas(df) - with pytest.raises(ValueError): - ds.write_parquet("local://" + data_path).fully_executed() - - -def test_read_warning_large_parallelism(ray_start_regular, propagate_logs, caplog): - with caplog.at_level(logging.WARNING, logger="ray.data.read_api"): - ray.data.range(5000, parallelism=5000).fully_executed() - assert ( - "The requested parallelism of 5000 is " - "more than 4x the number of available CPU slots in the cluster" in caplog.text - ), caplog.text - - -def test_read_write_local_node(ray_start_cluster): - cluster = ray_start_cluster - cluster.add_node( - resources={"bar:1": 100}, - num_cpus=10, - _system_config={"max_direct_call_object_size": 0}, - ) - cluster.add_node(resources={"bar:2": 100}, num_cpus=10) - cluster.add_node(resources={"bar:3": 100}, num_cpus=10) - - ray.init(cluster.address) - - import os - import tempfile - - data_path = tempfile.mkdtemp() - num_files = 5 - for idx in range(num_files): - df = pd.DataFrame( - {"one": list(range(idx, idx + 10)), "two": list(range(idx + 10, idx + 20))} - ) - path = os.path.join(data_path, f"test{idx}.parquet") - df.to_parquet(path) - - ctx = ray.data.context.DatasetContext.get_current() - ctx.read_write_local_node = True - - def check_dataset_is_local(ds): - blocks = ds.get_internal_block_refs() - assert len(blocks) == num_files - ray.wait(blocks, num_returns=len(blocks), fetch_local=False) - location_data = ray.experimental.get_object_locations(blocks) - locations = [] - for block in blocks: - locations.extend(location_data[block]["node_ids"]) - assert set(locations) == {ray.get_runtime_context().get_node_id()} - - local_path = "local://" + data_path - # Plain read. - ds = ray.data.read_parquet(local_path).fully_executed() - check_dataset_is_local(ds) - - # SPREAD scheduling got overridden when read local scheme. - ds = ray.data.read_parquet( - local_path, ray_remote_args={"scheduling_strategy": "SPREAD"} - ).fully_executed() - check_dataset_is_local(ds) - - # With fusion. - ds = ray.data.read_parquet(local_path).map(lambda x: x).fully_executed() - check_dataset_is_local(ds) - - # Write back to local scheme. - output = os.path.join(local_path, "test_read_write_local_node") - ds.write_parquet(output) - assert "1 nodes used" in ds.stats(), ds.stats() - ray.data.read_parquet(output).take_all() == ds.take_all() - - # Mixing paths of local and non-local scheme is invalid. - with pytest.raises(ValueError): - ds = ray.data.read_parquet( - [local_path + "/test1.parquet", data_path + "/test2.parquet"] - ).fully_executed() - with pytest.raises(ValueError): - ds = ray.data.read_parquet( - [local_path + "/test1.parquet", "example://iris.parquet"] - ).fully_executed() - with pytest.raises(ValueError): - ds = ray.data.read_parquet( - ["example://iris.parquet", local_path + "/test1.parquet"] - ).fully_executed() - - -def test_random_shuffle_spread(ray_start_cluster, use_push_based_shuffle): - cluster = ray_start_cluster - cluster.add_node( - resources={"bar:1": 100}, - num_cpus=10, - _system_config={"max_direct_call_object_size": 0}, - ) - cluster.add_node(resources={"bar:2": 100}, num_cpus=10) - cluster.add_node(resources={"bar:3": 100}, num_cpus=0) - - ray.init(cluster.address) - - @ray.remote - def get_node_id(): - return ray.get_runtime_context().get_node_id() - - node1_id = ray.get(get_node_id.options(resources={"bar:1": 1}).remote()) - node2_id = ray.get(get_node_id.options(resources={"bar:2": 1}).remote()) - - ds = ray.data.range(100, parallelism=2).random_shuffle() - blocks = ds.get_internal_block_refs() - ray.wait(blocks, num_returns=len(blocks), fetch_local=False) - location_data = ray.experimental.get_object_locations(blocks) - locations = [] - for block in blocks: - locations.extend(location_data[block]["node_ids"]) - assert "2 nodes used" in ds.stats() - - if not use_push_based_shuffle: - # We don't check this for push-based shuffle since it will try to - # colocate reduce tasks to improve locality. - assert set(locations) == {node1_id, node2_id} - - -def test_parquet_read_spread(ray_start_cluster, tmp_path): - cluster = ray_start_cluster - cluster.add_node( - resources={"bar:1": 100}, - num_cpus=10, - _system_config={"max_direct_call_object_size": 0}, - ) - cluster.add_node(resources={"bar:2": 100}, num_cpus=10) - cluster.add_node(resources={"bar:3": 100}, num_cpus=0) - - ray.init(cluster.address) - - @ray.remote - def get_node_id(): - return ray.get_runtime_context().get_node_id() - - node1_id = ray.get(get_node_id.options(resources={"bar:1": 1}).remote()) - node2_id = ray.get(get_node_id.options(resources={"bar:2": 1}).remote()) - - data_path = str(tmp_path) - df1 = pd.DataFrame({"one": list(range(100)), "two": list(range(100, 200))}) - path1 = os.path.join(data_path, "test1.parquet") - df1.to_parquet(path1) - df2 = pd.DataFrame({"one": list(range(300, 400)), "two": list(range(400, 500))}) - path2 = os.path.join(data_path, "test2.parquet") - df2.to_parquet(path2) - - ds = ray.data.read_parquet(data_path) - - # Force reads. - blocks = ds.get_internal_block_refs() - assert len(blocks) == 2 - - ray.wait(blocks, num_returns=len(blocks), fetch_local=False) - location_data = ray.experimental.get_object_locations(blocks) - locations = [] - for block in blocks: - locations.extend(location_data[block]["node_ids"]) - assert set(locations) == {node1_id, node2_id} - - -def test_stats_actor_cap_num_stats(ray_start_cluster): - actor = _StatsActor.remote(3) - metadatas = [] - task_idx = 0 - for uuid in range(3): - metadatas.append( - BlockMetadata( - num_rows=uuid, - size_bytes=None, - schema=None, - input_files=None, - exec_stats=None, - ) - ) - num_stats = uuid + 1 - actor.record_start.remote(uuid) - assert ray.get(actor._get_stats_dict_size.remote()) == ( - num_stats, - num_stats - 1, - num_stats - 1, - ) - actor.record_task.remote(uuid, task_idx, [metadatas[-1]]) - assert ray.get(actor._get_stats_dict_size.remote()) == ( - num_stats, - num_stats, - num_stats, - ) - for uuid in range(3): - assert ray.get(actor.get.remote(uuid))[0][task_idx] == [metadatas[uuid]] - # Add the fourth stats to exceed the limit. - actor.record_start.remote(3) - # The first stats (with uuid=0) should have been purged. - assert ray.get(actor.get.remote(0))[0] == {} - # The start_time has 3 entries because we just added it above with record_start(). - assert ray.get(actor._get_stats_dict_size.remote()) == (3, 2, 2) - - -@ray.remote -class Counter: - def __init__(self): - self.value = 0 - - def increment(self): - self.value += 1 - return self.value - - -class FlakyCSVDatasource(CSVDatasource): - def __init__(self): - self.counter = Counter.remote() - - def _read_stream(self, f: "pa.NativeFile", path: str, **reader_args): - count = self.counter.increment.remote() - if ray.get(count) == 1: - raise ValueError("oops") - else: - for block in CSVDatasource._read_stream(self, f, path, **reader_args): - yield block - - def _write_block(self, f: "pa.NativeFile", block: BlockAccessor, **writer_args): - count = self.counter.increment.remote() - if ray.get(count) == 1: - raise ValueError("oops") - else: - CSVDatasource._write_block(self, f, block, **writer_args) - - -def test_dataset_retry_exceptions(ray_start_regular, local_path): - df1 = pd.DataFrame({"one": [1, 2, 3], "two": ["a", "b", "c"]}) - path1 = os.path.join(local_path, "test1.csv") - df1.to_csv(path1, index=False, storage_options={}) - ds1 = ray.data.read_datasource(FlakyCSVDatasource(), parallelism=1, paths=path1) - ds1.write_datasource(FlakyCSVDatasource(), path=local_path, dataset_uuid="data") - assert df1.equals( - pd.read_csv(os.path.join(local_path, "data_000000.csv"), storage_options={}) - ) - - counter = Counter.remote() - - def flaky_mapper(x): - count = counter.increment.remote() - if ray.get(count) == 1: - raise ValueError("oops") - else: - return ray.get(count) - - assert sorted(ds1.map(flaky_mapper).take()) == [2, 3, 4] - - with pytest.raises(ValueError): - ray.data.read_datasource( - FlakyCSVDatasource(), - parallelism=1, - paths=path1, - ray_remote_args={"retry_exceptions": False}, - ).take() - - -def test_split_is_not_disruptive(ray_start_regular): - ds = ray.data.range(100, parallelism=10).map_batches(lambda x: x).lazy() - - def verify_integrity(splits): - for dss in splits: - for batch in dss.iter_batches(): - pass - for batch in ds.iter_batches(): - pass - - # No block splitting invovled: split 10 even blocks into 2 groups. - verify_integrity(ds.split(2, equal=True)) - # Block splitting invovled: split 10 even blocks into 3 groups. - verify_integrity(ds.split(3, equal=True)) - - # Same as above but having tranforms post converting to lazy. - verify_integrity(ds.map_batches(lambda x: x).split(2, equal=True)) - verify_integrity(ds.map_batches(lambda x: x).split(3, equal=True)) - - # Same as above but having in-place tranforms post converting to lazy. - verify_integrity(ds.randomize_block_order().split(2, equal=True)) - verify_integrity(ds.randomize_block_order().split(3, equal=True)) - - -def test_datasource(ray_start_regular): - source = ray.data.datasource.RandomIntRowDatasource() - assert len(ray.data.read_datasource(source, n=10, num_columns=2).take()) == 10 - source = ray.data.datasource.RangeDatasource() - assert ray.data.read_datasource(source, n=10).take() == list(range(10)) - - -def test_polars_lazy_import(shutdown_only): - import sys - - ctx = ray.data.context.DatasetContext.get_current() - - try: - original_use_polars = ctx.use_polars - ctx.use_polars = True - - num_items = 100 - parallelism = 4 - ray.init(num_cpus=4) - - @ray.remote - def f(should_import_polars): - # Sleep to spread the tasks. - time.sleep(1) - polars_imported = "polars" in sys.modules.keys() - return polars_imported == should_import_polars - - # We should not use polars for non-Arrow sort. - _ = ray.data.range(num_items, parallelism=parallelism).sort() - assert all(ray.get([f.remote(False) for _ in range(parallelism)])) - - a = range(100) - dfs = [] - partition_size = num_items // parallelism - for i in range(parallelism): - dfs.append( - pd.DataFrame({"a": a[i * partition_size : (i + 1) * partition_size]}) - ) - # At least one worker should have imported polars. - _ = ( - ray.data.from_pandas(dfs) - .map_batches(lambda t: t, batch_format="pyarrow", batch_size=None) - .sort(key="a") - .fully_executed() - ) - assert any(ray.get([f.remote(True) for _ in range(parallelism)])) - - finally: - ctx.use_polars = original_use_polars - - -def test_actor_pool_strategy_apply_interrupt(shutdown_only): - """Test that _apply kills the actor pool if an interrupt is raised.""" - ray.init(include_dashboard=False, num_cpus=1) - - cpus = ray.available_resources()["CPU"] - ds = ray.data.range(5, parallelism=5) - aps = ray.data.ActorPoolStrategy(max_size=5) - blocks = ds._plan.execute() - - # Start some actors, the first one sends a SIGINT, emulating a KeyboardInterrupt - def test_func(block): - for i, _ in enumerate(BlockAccessor.for_block(block).iter_rows()): - if i == 0: - os.kill(os.getpid(), signal.SIGINT) - else: - time.sleep(1000) - return block - - # No need to test ActorPoolStrategy in new execution backend. - if not DatasetContext.get_current().new_execution_backend: - with pytest.raises(ray.exceptions.RayTaskError): - aps._apply(test_func, {}, blocks, False) - - # Check that all actors have been killed by counting the available CPUs - wait_for_condition(lambda: (ray.available_resources().get("CPU", 0) == cpus)) - - -def test_actor_pool_strategy_default_num_actors(shutdown_only): - def f(x): - import time - - time.sleep(1) - return x - - num_cpus = 5 - ray.init(num_cpus=num_cpus) - compute_strategy = ray.data.ActorPoolStrategy() - ray.data.range(10, parallelism=10).map_batches( - f, batch_size=1, compute=compute_strategy - ).fully_executed() - - # The new execution backend is not using the ActorPoolStrategy under - # the hood, so the expectation here applies only to the old backend. - # TODO(https://github.com/ray-project/ray/issues/31723): we should check - # the num of workers once we have autoscaling in new execution backend. - if not DatasetContext.get_current().new_execution_backend: - expected_max_num_workers = math.ceil( - num_cpus * (1 / compute_strategy.ready_to_total_workers_ratio) - ) - assert ( - compute_strategy.num_workers >= num_cpus - and compute_strategy.num_workers <= expected_max_num_workers - ), "Number of actors is out of the expected bound" - - -def test_actor_pool_strategy_bundles_to_max_actors(shutdown_only): - """Tests that blocks are bundled up to the specified max number of actors.""" - - def f(x): - return x - - max_size = 2 - compute_strategy = ray.data.ActorPoolStrategy(max_size=max_size) - ds = ( - ray.data.range(10, parallelism=10) - .map_batches(f, batch_size=None, compute=compute_strategy) - .fully_executed() - ) - - # TODO(https://github.com/ray-project/ray/issues/31723): implement the feature - # of capping bundle size by actor pool size, and then re-enable this test. - if not DatasetContext.get_current().new_execution_backend: - assert f"{max_size}/{max_size} blocks" in ds.stats() - - # Check batch size is still respected. - ds = ( - ray.data.range(10, parallelism=10) - .map_batches(f, batch_size=10, compute=compute_strategy) - .fully_executed() - ) - - assert "1/1 blocks" in ds.stats() - - -def test_default_batch_format(shutdown_only): - ds = ray.data.range(100) - assert ds.default_batch_format() == list - - ds = ray.data.range_tensor(100) - assert ds.default_batch_format() == np.ndarray - - df = pd.DataFrame({"foo": ["a", "b"], "bar": [0, 1]}) - ds = ray.data.from_pandas(df) - assert ds.default_batch_format() == pd.DataFrame - - -def test_dataset_schema_after_read_stats(ray_start_cluster): - cluster = ray_start_cluster - cluster.add_node(num_cpus=1) - ray.init(cluster.address) - cluster.add_node(num_cpus=1, resources={"foo": 1}) - ds = ray.data.read_csv( - "example://iris.csv", ray_remote_args={"resources": {"foo": 1}} - ) - schema = ds.schema() - ds.stats() - assert schema == ds.schema() - - -class LoggerWarningCalled(Exception): - """Custom exception used in test_warning_execute_with_no_cpu() and - test_nowarning_execute_with_cpu(). Raised when the `logger.warning` method - is called, so that we can kick out of `plan.execute()` by catching this Exception - and check logging was done properly.""" - - pass - - -def test_warning_execute_with_no_cpu(ray_start_cluster): - """Tests ExecutionPlan.execute() to ensure a warning is logged - when no CPU resources are available.""" - # Create one node with no CPUs to trigger the Dataset warning - cluster = ray_start_cluster - cluster.add_node(num_cpus=0) - - logger = DatasetLogger("ray.data._internal.plan").get_logger() - with patch.object( - logger, - "warning", - side_effect=LoggerWarningCalled, - ) as mock_logger: - try: - ds = ray.data.range(10) - ds = ds.map_batches(lambda x: x) - ds.take() - except Exception as e: - if ray.data.context.DatasetContext.get_current().use_streaming_executor: - assert isinstance(e, ValueError) - assert "exceeds the execution limits ExecutionResources(cpu=0.0" in str( - e - ) - else: - assert isinstance(e, LoggerWarningCalled) - logger_args, logger_kwargs = mock_logger.call_args - assert ( - "Warning: The Ray cluster currently does not have " - in logger_args[0] - ) - - -def test_nowarning_execute_with_cpu(ray_start_cluster_init): - """Tests ExecutionPlan.execute() to ensure no warning is logged - when there are available CPU resources.""" - # Create one node with CPUs to avoid triggering the Dataset warning - ray.init(ray_start_cluster_init.address) - - logger = DatasetLogger("ray.data._internal.plan").get_logger() - with patch.object( - logger, - "warning", - side_effect=LoggerWarningCalled, - ) as mock_logger: - ds = ray.data.range(10) - ds = ds.map_batches(lambda x: x) - ds.take() - mock_logger.assert_not_called() - - -if __name__ == "__main__": - import sys - - sys.exit(pytest.main(["-v", __file__])) diff --git a/python/ray/data/tests/test_dataset_all_to_all.py b/python/ray/data/tests/test_dataset_all_to_all.py new file mode 100644 index 0000000000000..e9150b52f2f24 --- /dev/null +++ b/python/ray/data/tests/test_dataset_all_to_all.py @@ -0,0 +1,1703 @@ +import itertools +import math +import random +import time + +import numpy as np +import pandas as pd +import pyarrow as pa +import pytest + +import ray +from ray.data.aggregate import AggregateFn, Count, Max, Mean, Min, Std, Sum +from ray.data.context import DatasetContext +from ray.data.tests.conftest import * # noqa +from ray.tests.conftest import * # noqa + + +def test_zip(ray_start_regular_shared): + ds1 = ray.data.range(5, parallelism=5) + ds2 = ray.data.range(5, parallelism=5).map(lambda x: x + 1) + ds = ds1.zip(ds2) + assert ds.schema() == tuple + assert ds.take() == [(0, 1), (1, 2), (2, 3), (3, 4), (4, 5)] + with pytest.raises(ValueError): + ds.zip(ray.data.range(3)).fully_executed() + + +@pytest.mark.parametrize( + "num_blocks1,num_blocks2", + list(itertools.combinations_with_replacement(range(1, 12), 2)), +) +def test_zip_different_num_blocks_combinations( + ray_start_regular_shared, num_blocks1, num_blocks2 +): + n = 12 + ds1 = ray.data.range(n, parallelism=num_blocks1) + ds2 = ray.data.range(n, parallelism=num_blocks2).map(lambda x: x + 1) + ds = ds1.zip(ds2) + assert ds.schema() == tuple + assert ds.take() == list(zip(range(n), range(1, n + 1))) + + +@pytest.mark.parametrize( + "num_cols1,num_cols2,should_invert", + [ + (1, 1, False), + (4, 1, False), + (1, 4, True), + (1, 10, True), + (10, 10, False), + ], +) +def test_zip_different_num_blocks_split_smallest( + ray_start_regular_shared, + num_cols1, + num_cols2, + should_invert, +): + n = 12 + num_blocks1 = 4 + num_blocks2 = 2 + ds1 = ray.data.from_items( + [{str(i): i for i in range(num_cols1)}] * n, parallelism=num_blocks1 + ) + ds2 = ray.data.from_items( + [{str(i): i for i in range(num_cols1, num_cols1 + num_cols2)}] * n, + parallelism=num_blocks2, + ) + ds = ds1.zip(ds2).fully_executed() + num_blocks = ds._plan._snapshot_blocks.executed_num_blocks() + assert ds.take() == [{str(i): i for i in range(num_cols1 + num_cols2)}] * n + if should_invert: + assert num_blocks == num_blocks2 + else: + assert num_blocks == num_blocks1 + + +def test_zip_pandas(ray_start_regular_shared): + ds1 = ray.data.from_pandas(pd.DataFrame({"col1": [1, 2], "col2": [4, 5]})) + ds2 = ray.data.from_pandas(pd.DataFrame({"col3": ["a", "b"], "col4": ["d", "e"]})) + ds = ds1.zip(ds2) + assert ds.count() == 2 + assert "{col1: int64, col2: int64, col3: object, col4: object}" in str(ds) + result = [r.as_pydict() for r in ds.take()] + assert result[0] == {"col1": 1, "col2": 4, "col3": "a", "col4": "d"} + + ds3 = ray.data.from_pandas(pd.DataFrame({"col2": ["a", "b"], "col4": ["d", "e"]})) + ds = ds1.zip(ds3) + assert ds.count() == 2 + assert "{col1: int64, col2: int64, col2_1: object, col4: object}" in str(ds) + result = [r.as_pydict() for r in ds.take()] + assert result[0] == {"col1": 1, "col2": 4, "col2_1": "a", "col4": "d"} + + +def test_zip_arrow(ray_start_regular_shared): + ds1 = ray.data.range_table(5).map(lambda r: {"id": r["value"]}) + ds2 = ray.data.range_table(5).map( + lambda r: {"a": r["value"] + 1, "b": r["value"] + 2} + ) + ds = ds1.zip(ds2) + assert ds.count() == 5 + assert "{id: int64, a: int64, b: int64}" in str(ds) + result = [r.as_pydict() for r in ds.take()] + assert result[0] == {"id": 0, "a": 1, "b": 2} + + # Test duplicate column names. + ds = ds1.zip(ds1).zip(ds1) + assert ds.count() == 5 + assert "{id: int64, id_1: int64, id_2: int64}" in str(ds) + result = [r.as_pydict() for r in ds.take()] + assert result[0] == {"id": 0, "id_1": 0, "id_2": 0} + + +def test_empty_shuffle(ray_start_regular_shared): + ds = ray.data.range(100, parallelism=100) + ds = ds.filter(lambda x: x) + ds = ds.map_batches(lambda x: x) + ds = ds.random_shuffle() # Would prev. crash with AssertionError: pyarrow.Table. + ds.show() + + +def test_repartition_shuffle(ray_start_regular_shared): + ds = ray.data.range(20, parallelism=10) + assert ds.num_blocks() == 10 + assert ds.sum() == 190 + assert ds._block_num_rows() == [2] * 10 + + ds2 = ds.repartition(5, shuffle=True) + assert ds2.num_blocks() == 5 + assert ds2.sum() == 190 + assert ds2._block_num_rows() == [10, 10, 0, 0, 0] + + ds3 = ds2.repartition(20, shuffle=True) + assert ds3.num_blocks() == 20 + assert ds3.sum() == 190 + assert ds3._block_num_rows() == [2] * 10 + [0] * 10 + + large = ray.data.range(10000, parallelism=10) + large = large.repartition(20, shuffle=True) + assert large._block_num_rows() == [500] * 20 + + +def test_repartition_noshuffle(ray_start_regular_shared): + ds = ray.data.range(20, parallelism=10) + assert ds.num_blocks() == 10 + assert ds.sum() == 190 + assert ds._block_num_rows() == [2] * 10 + + ds2 = ds.repartition(5, shuffle=False) + assert ds2.num_blocks() == 5 + assert ds2.sum() == 190 + assert ds2._block_num_rows() == [4, 4, 4, 4, 4] + + ds3 = ds2.repartition(20, shuffle=False) + assert ds3.num_blocks() == 20 + assert ds3.sum() == 190 + assert ds3._block_num_rows() == [1] * 20 + + # Test num_partitions > num_rows + ds4 = ds.repartition(40, shuffle=False) + assert ds4.num_blocks() == 40 + blocks = ray.get(ds4.get_internal_block_refs()) + assert all(isinstance(block, list) for block in blocks), blocks + assert ds4.sum() == 190 + assert ds4._block_num_rows() == [1] * 20 + [0] * 20 + + ds5 = ray.data.range(22).repartition(4) + assert ds5.num_blocks() == 4 + assert ds5._block_num_rows() == [5, 6, 5, 6] + + large = ray.data.range(10000, parallelism=10) + large = large.repartition(20) + assert large._block_num_rows() == [500] * 20 + + +def test_repartition_shuffle_arrow(ray_start_regular_shared): + ds = ray.data.range_table(20, parallelism=10) + assert ds.num_blocks() == 10 + assert ds.count() == 20 + assert ds._block_num_rows() == [2] * 10 + + ds2 = ds.repartition(5, shuffle=True) + assert ds2.num_blocks() == 5 + assert ds2.count() == 20 + assert ds2._block_num_rows() == [10, 10, 0, 0, 0] + + ds3 = ds2.repartition(20, shuffle=True) + assert ds3.num_blocks() == 20 + assert ds3.count() == 20 + assert ds3._block_num_rows() == [2] * 10 + [0] * 10 + + large = ray.data.range_table(10000, parallelism=10) + large = large.repartition(20, shuffle=True) + assert large._block_num_rows() == [500] * 20 + + +def test_grouped_dataset_repr(ray_start_regular_shared): + ds = ray.data.from_items([{"key": "spam"}, {"key": "ham"}, {"key": "spam"}]) + assert repr(ds.groupby("key")) == f"GroupedDataset(dataset={ds!r}, key='key')" + + +def test_groupby_arrow(ray_start_regular_shared, use_push_based_shuffle): + # Test empty dataset. + agg_ds = ( + ray.data.range_table(10) + .filter(lambda r: r["value"] > 10) + .groupby("value") + .count() + ) + assert agg_ds.count() == 0 + + +def test_groupby_errors(ray_start_regular_shared): + ds = ray.data.range(100) + + ds.groupby(None).count().show() # OK + ds.groupby(lambda x: x % 2).count().show() # OK + with pytest.raises(ValueError): + ds.groupby("foo").count().show() + + ds = ray.data.range_table(100) + ds.groupby(None).count().show() # OK + with pytest.raises(ValueError): + ds.groupby(lambda x: x % 2).count().show() + + +def test_agg_errors(ray_start_regular_shared): + ds = ray.data.range(100) + from ray.data.aggregate import Max + + ds.aggregate(Max()) # OK + ds.aggregate(Max(lambda x: x)) # OK + with pytest.raises(ValueError): + ds.aggregate(Max("foo")) + + ds = ray.data.range_table(100) + ds.aggregate(Max("value")) # OK + with pytest.raises(ValueError): + ds.aggregate(Max()) + with pytest.raises(ValueError): + ds.aggregate(Max(lambda x: x)) + with pytest.raises(ValueError): + ds.aggregate(Max("bad_field")) + + +@pytest.mark.parametrize("num_parts", [1, 30]) +def test_groupby_agg_name_conflict(ray_start_regular_shared, num_parts): + # Test aggregation name conflict. + xs = list(range(100)) + grouped_ds = ( + ray.data.from_items([{"A": (x % 3), "B": x} for x in xs]) + .repartition(num_parts) + .groupby("A") + ) + agg_ds = grouped_ds.aggregate( + AggregateFn( + init=lambda k: [0, 0], + accumulate_row=lambda a, r: [a[0] + r["B"], a[1] + 1], + merge=lambda a1, a2: [a1[0] + a2[0], a1[1] + a2[1]], + finalize=lambda a: a[0] / a[1], + name="foo", + ), + AggregateFn( + init=lambda k: [0, 0], + accumulate_row=lambda a, r: [a[0] + r["B"], a[1] + 1], + merge=lambda a1, a2: [a1[0] + a2[0], a1[1] + a2[1]], + finalize=lambda a: a[0] / a[1], + name="foo", + ), + ) + assert agg_ds.count() == 3 + assert [row.as_pydict() for row in agg_ds.sort("A").iter_rows()] == [ + {"A": 0, "foo": 49.5, "foo_2": 49.5}, + {"A": 1, "foo": 49.0, "foo_2": 49.0}, + {"A": 2, "foo": 50.0, "foo_2": 50.0}, + ] + + +@pytest.mark.parametrize("num_parts", [1, 30]) +@pytest.mark.parametrize("ds_format", ["arrow", "pandas"]) +def test_groupby_tabular_count( + ray_start_regular_shared, ds_format, num_parts, use_push_based_shuffle +): + # Test built-in count aggregation + seed = int(time.time()) + print(f"Seeding RNG for test_groupby_arrow_count with: {seed}") + random.seed(seed) + xs = list(range(100)) + random.shuffle(xs) + + def _to_pandas(ds): + return ds.map_batches(lambda x: x, batch_size=None, batch_format="pandas") + + ds = ray.data.from_items([{"A": (x % 3), "B": x} for x in xs]).repartition( + num_parts + ) + if ds_format == "pandas": + ds = _to_pandas(ds) + agg_ds = ds.groupby("A").count() + assert agg_ds.count() == 3 + assert [row.as_pydict() for row in agg_ds.sort("A").iter_rows()] == [ + {"A": 0, "count()": 34}, + {"A": 1, "count()": 33}, + {"A": 2, "count()": 33}, + ] + + +@pytest.mark.parametrize("num_parts", [1, 30]) +@pytest.mark.parametrize("ds_format", ["arrow", "pandas"]) +def test_groupby_tabular_sum( + ray_start_regular_shared, ds_format, num_parts, use_push_based_shuffle +): + # Test built-in sum aggregation + seed = int(time.time()) + print(f"Seeding RNG for test_groupby_tabular_sum with: {seed}") + random.seed(seed) + xs = list(range(100)) + random.shuffle(xs) + + def _to_pandas(ds): + return ds.map_batches(lambda x: x, batch_size=None, batch_format="pandas") + + ds = ray.data.from_items([{"A": (x % 3), "B": x} for x in xs]).repartition( + num_parts + ) + if ds_format == "pandas": + ds = _to_pandas(ds) + + agg_ds = ds.groupby("A").sum("B") + assert agg_ds.count() == 3 + assert [row.as_pydict() for row in agg_ds.sort("A").iter_rows()] == [ + {"A": 0, "sum(B)": 1683}, + {"A": 1, "sum(B)": 1617}, + {"A": 2, "sum(B)": 1650}, + ] + + # Test built-in sum aggregation with nans + ds = ray.data.from_items( + [{"A": (x % 3), "B": x} for x in xs] + [{"A": 0, "B": None}] + ).repartition(num_parts) + if ds_format == "pandas": + ds = _to_pandas(ds) + nan_grouped_ds = ds.groupby("A") + nan_agg_ds = nan_grouped_ds.sum("B") + assert nan_agg_ds.count() == 3 + assert [row.as_pydict() for row in nan_agg_ds.sort("A").iter_rows()] == [ + {"A": 0, "sum(B)": 1683}, + {"A": 1, "sum(B)": 1617}, + {"A": 2, "sum(B)": 1650}, + ] + # Test ignore_nulls=False + nan_agg_ds = nan_grouped_ds.sum("B", ignore_nulls=False) + assert nan_agg_ds.count() == 3 + pd.testing.assert_frame_equal( + nan_agg_ds.sort("A").to_pandas(), + pd.DataFrame( + { + "A": [0, 1, 2], + "sum(B)": [None, 1617, 1650], + } + ), + check_dtype=False, + ) + # Test all nans + ds = ray.data.from_items([{"A": (x % 3), "B": None} for x in xs]).repartition( + num_parts + ) + if ds_format == "pandas": + ds = _to_pandas(ds) + nan_agg_ds = ds.groupby("A").sum("B") + assert nan_agg_ds.count() == 3 + pd.testing.assert_frame_equal( + nan_agg_ds.sort("A").to_pandas(), + pd.DataFrame( + { + "A": [0, 1, 2], + "sum(B)": [None, None, None], + } + ), + ) + + +@pytest.mark.parametrize("num_parts", [1, 30]) +@pytest.mark.parametrize("ds_format", ["arrow", "pandas"]) +def test_global_tabular_sum(ray_start_regular_shared, ds_format, num_parts): + seed = int(time.time()) + print(f"Seeding RNG for test_global_arrow_sum with: {seed}") + random.seed(seed) + xs = list(range(100)) + random.shuffle(xs) + + def _to_pandas(ds): + return ds.map_batches(lambda x: x, batch_size=None, batch_format="pandas") + + # Test built-in global sum aggregation + ds = ray.data.from_items([{"A": x} for x in xs]).repartition(num_parts) + if ds_format == "pandas": + ds = _to_pandas(ds) + assert ds.sum("A") == 4950 + + # Test empty dataset + ds = ray.data.range_table(10) + if ds_format == "pandas": + ds = _to_pandas(ds) + assert ds.filter(lambda r: r["value"] > 10).sum("value") is None + + # Test built-in global sum aggregation with nans + nan_ds = ray.data.from_items([{"A": x} for x in xs] + [{"A": None}]).repartition( + num_parts + ) + if ds_format == "pandas": + nan_ds = _to_pandas(nan_ds) + assert nan_ds.sum("A") == 4950 + # Test ignore_nulls=False + assert nan_ds.sum("A", ignore_nulls=False) is None + # Test all nans + nan_ds = ray.data.from_items([{"A": None}] * len(xs)).repartition(num_parts) + if ds_format == "pandas": + nan_ds = _to_pandas(nan_ds) + assert nan_ds.sum("A") is None + assert nan_ds.sum("A", ignore_nulls=False) is None + + +@pytest.mark.parametrize("num_parts", [1, 30]) +@pytest.mark.parametrize("ds_format", ["arrow", "pandas"]) +def test_groupby_tabular_min(ray_start_regular_shared, ds_format, num_parts): + # Test built-in min aggregation + seed = int(time.time()) + print(f"Seeding RNG for test_groupby_tabular_min with: {seed}") + random.seed(seed) + xs = list(range(100)) + random.shuffle(xs) + + def _to_pandas(ds): + return ds.map_batches(lambda x: x, batch_size=None, batch_format="pandas") + + ds = ray.data.from_items([{"A": (x % 3), "B": x} for x in xs]).repartition( + num_parts + ) + if ds_format == "pandas": + ds = _to_pandas(ds) + + agg_ds = ds.groupby("A").min("B") + assert agg_ds.count() == 3 + assert [row.as_pydict() for row in agg_ds.sort("A").iter_rows()] == [ + {"A": 0, "min(B)": 0}, + {"A": 1, "min(B)": 1}, + {"A": 2, "min(B)": 2}, + ] + + # Test built-in min aggregation with nans + ds = ray.data.from_items( + [{"A": (x % 3), "B": x} for x in xs] + [{"A": 0, "B": None}] + ).repartition(num_parts) + if ds_format == "pandas": + ds = _to_pandas(ds) + nan_grouped_ds = ds.groupby("A") + nan_agg_ds = nan_grouped_ds.min("B") + assert nan_agg_ds.count() == 3 + assert [row.as_pydict() for row in nan_agg_ds.sort("A").iter_rows()] == [ + {"A": 0, "min(B)": 0}, + {"A": 1, "min(B)": 1}, + {"A": 2, "min(B)": 2}, + ] + # Test ignore_nulls=False + nan_agg_ds = nan_grouped_ds.min("B", ignore_nulls=False) + assert nan_agg_ds.count() == 3 + pd.testing.assert_frame_equal( + nan_agg_ds.sort("A").to_pandas(), + pd.DataFrame( + { + "A": [0, 1, 2], + "min(B)": [None, 1, 2], + } + ), + check_dtype=False, + ) + # Test all nans + ds = ray.data.from_items([{"A": (x % 3), "B": None} for x in xs]).repartition( + num_parts + ) + if ds_format == "pandas": + ds = _to_pandas(ds) + nan_agg_ds = ds.groupby("A").min("B") + assert nan_agg_ds.count() == 3 + pd.testing.assert_frame_equal( + nan_agg_ds.sort("A").to_pandas(), + pd.DataFrame( + { + "A": [0, 1, 2], + "min(B)": [None, None, None], + } + ), + check_dtype=False, + ) + + +@pytest.mark.parametrize("num_parts", [1, 30]) +@pytest.mark.parametrize("ds_format", ["arrow", "pandas"]) +def test_groupby_tabular_max(ray_start_regular_shared, ds_format, num_parts): + # Test built-in max aggregation + seed = int(time.time()) + print(f"Seeding RNG for test_groupby_tabular_max with: {seed}") + random.seed(seed) + xs = list(range(100)) + random.shuffle(xs) + + def _to_pandas(ds): + return ds.map_batches(lambda x: x, batch_size=None, batch_format="pandas") + + ds = ray.data.from_items([{"A": (x % 3), "B": x} for x in xs]).repartition( + num_parts + ) + if ds_format == "pandas": + ds = _to_pandas(ds) + + agg_ds = ds.groupby("A").max("B") + assert agg_ds.count() == 3 + assert [row.as_pydict() for row in agg_ds.sort("A").iter_rows()] == [ + {"A": 0, "max(B)": 99}, + {"A": 1, "max(B)": 97}, + {"A": 2, "max(B)": 98}, + ] + + # Test built-in min aggregation with nans + ds = ray.data.from_items( + [{"A": (x % 3), "B": x} for x in xs] + [{"A": 0, "B": None}] + ).repartition(num_parts) + if ds_format == "pandas": + ds = _to_pandas(ds) + nan_grouped_ds = ds.groupby("A") + nan_agg_ds = nan_grouped_ds.max("B") + assert nan_agg_ds.count() == 3 + assert [row.as_pydict() for row in nan_agg_ds.sort("A").iter_rows()] == [ + {"A": 0, "max(B)": 99}, + {"A": 1, "max(B)": 97}, + {"A": 2, "max(B)": 98}, + ] + # Test ignore_nulls=False + nan_agg_ds = nan_grouped_ds.max("B", ignore_nulls=False) + assert nan_agg_ds.count() == 3 + pd.testing.assert_frame_equal( + nan_agg_ds.sort("A").to_pandas(), + pd.DataFrame( + { + "A": [0, 1, 2], + "max(B)": [None, 97, 98], + } + ), + check_dtype=False, + ) + # Test all nans + ds = ray.data.from_items([{"A": (x % 3), "B": None} for x in xs]).repartition( + num_parts + ) + if ds_format == "pandas": + ds = _to_pandas(ds) + nan_agg_ds = ds.groupby("A").max("B") + assert nan_agg_ds.count() == 3 + pd.testing.assert_frame_equal( + nan_agg_ds.sort("A").to_pandas(), + pd.DataFrame( + { + "A": [0, 1, 2], + "max(B)": [None, None, None], + } + ), + check_dtype=False, + ) + + +@pytest.mark.parametrize("num_parts", [1, 30]) +@pytest.mark.parametrize("ds_format", ["arrow", "pandas"]) +def test_groupby_tabular_mean(ray_start_regular_shared, ds_format, num_parts): + # Test built-in mean aggregation + seed = int(time.time()) + print(f"Seeding RNG for test_groupby_tabular_mean with: {seed}") + random.seed(seed) + xs = list(range(100)) + random.shuffle(xs) + + def _to_pandas(ds): + return ds.map_batches(lambda x: x, batch_size=None, batch_format="pandas") + + ds = ray.data.from_items([{"A": (x % 3), "B": x} for x in xs]).repartition( + num_parts + ) + if ds_format == "pandas": + ds = _to_pandas(ds) + + agg_ds = ds.groupby("A").mean("B") + assert agg_ds.count() == 3 + assert [row.as_pydict() for row in agg_ds.sort("A").iter_rows()] == [ + {"A": 0, "mean(B)": 49.5}, + {"A": 1, "mean(B)": 49.0}, + {"A": 2, "mean(B)": 50.0}, + ] + + # Test built-in mean aggregation with nans + ds = ray.data.from_items( + [{"A": (x % 3), "B": x} for x in xs] + [{"A": 0, "B": None}] + ).repartition(num_parts) + if ds_format == "pandas": + ds = _to_pandas(ds) + nan_grouped_ds = ds.groupby("A") + nan_agg_ds = nan_grouped_ds.mean("B") + assert nan_agg_ds.count() == 3 + assert [row.as_pydict() for row in nan_agg_ds.sort("A").iter_rows()] == [ + {"A": 0, "mean(B)": 49.5}, + {"A": 1, "mean(B)": 49.0}, + {"A": 2, "mean(B)": 50.0}, + ] + # Test ignore_nulls=False + nan_agg_ds = nan_grouped_ds.mean("B", ignore_nulls=False) + assert nan_agg_ds.count() == 3 + pd.testing.assert_frame_equal( + nan_agg_ds.sort("A").to_pandas(), + pd.DataFrame( + { + "A": [0, 1, 2], + "mean(B)": [None, 49.0, 50.0], + } + ), + check_dtype=False, + ) + # Test all nans + ds = ray.data.from_items([{"A": (x % 3), "B": None} for x in xs]).repartition( + num_parts + ) + if ds_format == "pandas": + ds = _to_pandas(ds) + nan_agg_ds = ds.groupby("A").mean("B") + assert nan_agg_ds.count() == 3 + pd.testing.assert_frame_equal( + nan_agg_ds.sort("A").to_pandas(), + pd.DataFrame( + { + "A": [0, 1, 2], + "mean(B)": [None, None, None], + } + ), + check_dtype=False, + ) + + +@pytest.mark.parametrize("num_parts", [1, 30]) +@pytest.mark.parametrize("ds_format", ["arrow", "pandas"]) +def test_groupby_tabular_std(ray_start_regular_shared, ds_format, num_parts): + # Test built-in std aggregation + seed = int(time.time()) + print(f"Seeding RNG for test_groupby_tabular_std with: {seed}") + random.seed(seed) + xs = list(range(100)) + random.shuffle(xs) + + def _to_arrow(ds): + return ds.map_batches(lambda x: x, batch_size=None, batch_format="pyarrow") + + df = pd.DataFrame({"A": [x % 3 for x in xs], "B": xs}) + ds = ray.data.from_pandas(df).repartition(num_parts) + if ds_format == "arrow": + ds = _to_arrow(ds) + agg_ds = ds.groupby("A").std("B") + assert agg_ds.count() == 3 + result = agg_ds.to_pandas()["std(B)"].to_numpy() + expected = df.groupby("A")["B"].std().to_numpy() + np.testing.assert_array_almost_equal(result, expected) + # ddof of 0 + ds = ray.data.from_pandas(df).repartition(num_parts) + if ds_format == "arrow": + ds = _to_arrow(ds) + agg_ds = ds.groupby("A").std("B", ddof=0) + assert agg_ds.count() == 3 + result = agg_ds.to_pandas()["std(B)"].to_numpy() + expected = df.groupby("A")["B"].std(ddof=0).to_numpy() + np.testing.assert_array_almost_equal(result, expected) + + # Test built-in std aggregation with nans + nan_df = pd.DataFrame({"A": [x % 3 for x in xs] + [0], "B": xs + [None]}) + ds = ray.data.from_pandas(nan_df).repartition(num_parts) + if ds_format == "arrow": + ds = _to_arrow(ds) + nan_grouped_ds = ds.groupby("A") + nan_agg_ds = nan_grouped_ds.std("B") + assert nan_agg_ds.count() == 3 + result = nan_agg_ds.to_pandas()["std(B)"].to_numpy() + expected = nan_df.groupby("A")["B"].std().to_numpy() + np.testing.assert_array_almost_equal(result, expected) + # Test ignore_nulls=False + nan_agg_ds = nan_grouped_ds.std("B", ignore_nulls=False) + assert nan_agg_ds.count() == 3 + result = nan_agg_ds.to_pandas()["std(B)"].to_numpy() + expected = nan_df.groupby("A")["B"].std() + expected[0] = None + np.testing.assert_array_almost_equal(result, expected) + # Test all nans + nan_df = pd.DataFrame({"A": [x % 3 for x in xs], "B": [None] * len(xs)}) + ds = ray.data.from_pandas(nan_df).repartition(num_parts) + if ds_format == "arrow": + ds = _to_arrow(ds) + nan_agg_ds = ds.groupby("A").std("B", ignore_nulls=False) + assert nan_agg_ds.count() == 3 + result = nan_agg_ds.to_pandas()["std(B)"].to_numpy() + expected = pd.Series([None] * 3) + np.testing.assert_array_equal(result, expected) + + +@pytest.mark.parametrize("num_parts", [1, 30]) +def test_groupby_arrow_multicolumn(ray_start_regular_shared, num_parts): + # Test built-in mean aggregation on multiple columns + seed = int(time.time()) + print(f"Seeding RNG for test_groupby_arrow_multicolumn with: {seed}") + random.seed(seed) + xs = list(range(100)) + random.shuffle(xs) + df = pd.DataFrame({"A": [x % 3 for x in xs], "B": xs, "C": [2 * x for x in xs]}) + agg_ds = ( + ray.data.from_pandas(df).repartition(num_parts).groupby("A").mean(["B", "C"]) + ) + assert agg_ds.count() == 3 + assert [row.as_pydict() for row in agg_ds.sort("A").iter_rows()] == [ + {"A": 0, "mean(B)": 49.5, "mean(C)": 99.0}, + {"A": 1, "mean(B)": 49.0, "mean(C)": 98.0}, + {"A": 2, "mean(B)": 50.0, "mean(C)": 100.0}, + ] + + # Test that unspecified agg column ==> agg on all columns except for + # groupby keys. + agg_ds = ray.data.from_pandas(df).repartition(num_parts).groupby("A").mean() + assert agg_ds.count() == 3 + assert [row.as_pydict() for row in agg_ds.sort("A").iter_rows()] == [ + {"A": 0, "mean(B)": 49.5, "mean(C)": 99.0}, + {"A": 1, "mean(B)": 49.0, "mean(C)": 98.0}, + {"A": 2, "mean(B)": 50.0, "mean(C)": 100.0}, + ] + + # Test built-in global mean aggregation + df = pd.DataFrame({"A": xs, "B": [2 * x for x in xs]}) + result_row = ray.data.from_pandas(df).repartition(num_parts).mean(["A", "B"]) + assert result_row["mean(A)"] == df["A"].mean() + assert result_row["mean(B)"] == df["B"].mean() + + +def test_groupby_agg_bad_on(ray_start_regular_shared): + # Test bad on for groupby aggregation + xs = list(range(100)) + df = pd.DataFrame({"A": [x % 3 for x in xs], "B": xs, "C": [2 * x for x in xs]}) + # Wrong type. + with pytest.raises(TypeError): + ray.data.from_pandas(df).groupby("A").mean(5).fully_executed() + with pytest.raises(TypeError): + ray.data.from_pandas(df).groupby("A").mean([5]).fully_executed() + # Empty list. + with pytest.raises(ValueError): + ray.data.from_pandas(df).groupby("A").mean([]).fully_executed() + # Nonexistent column. + with pytest.raises(ValueError): + ray.data.from_pandas(df).groupby("A").mean("D").fully_executed() + with pytest.raises(ValueError): + ray.data.from_pandas(df).groupby("A").mean(["B", "D"]).fully_executed() + # Columns for simple Dataset. + with pytest.raises(ValueError): + ray.data.from_items(xs).groupby(lambda x: x % 3 == 0).mean("A").fully_executed() + + # Test bad on for global aggregation + # Wrong type. + with pytest.raises(TypeError): + ray.data.from_pandas(df).mean(5).fully_executed() + with pytest.raises(TypeError): + ray.data.from_pandas(df).mean([5]).fully_executed() + # Empty list. + with pytest.raises(ValueError): + ray.data.from_pandas(df).mean([]).fully_executed() + # Nonexistent column. + with pytest.raises(ValueError): + ray.data.from_pandas(df).mean("D").fully_executed() + with pytest.raises(ValueError): + ray.data.from_pandas(df).mean(["B", "D"]).fully_executed() + # Columns for simple Dataset. + with pytest.raises(ValueError): + ray.data.from_items(xs).mean("A").fully_executed() + + +@pytest.mark.parametrize("num_parts", [1, 30]) +def test_groupby_arrow_multi_agg(ray_start_regular_shared, num_parts): + seed = int(time.time()) + print(f"Seeding RNG for test_groupby_arrow_multi_agg with: {seed}") + random.seed(seed) + xs = list(range(100)) + random.shuffle(xs) + df = pd.DataFrame({"A": [x % 3 for x in xs], "B": xs}) + agg_ds = ( + ray.data.from_pandas(df) + .repartition(num_parts) + .groupby("A") + .aggregate( + Count(), + Sum("B"), + Min("B"), + Max("B"), + Mean("B"), + Std("B"), + ) + ) + assert agg_ds.count() == 3 + agg_df = agg_ds.to_pandas() + expected_grouped = df.groupby("A")["B"] + np.testing.assert_array_equal(agg_df["count()"].to_numpy(), [34, 33, 33]) + for agg in ["sum", "min", "max", "mean", "std"]: + result = agg_df[f"{agg}(B)"].to_numpy() + expected = getattr(expected_grouped, agg)().to_numpy() + if agg == "std": + np.testing.assert_array_almost_equal(result, expected) + else: + np.testing.assert_array_equal(result, expected) + # Test built-in global std aggregation + df = pd.DataFrame({"A": xs}) + + result_row = ( + ray.data.from_pandas(df) + .repartition(num_parts) + .aggregate( + Sum("A"), + Min("A"), + Max("A"), + Mean("A"), + Std("A"), + ) + ) + for agg in ["sum", "min", "max", "mean", "std"]: + result = result_row[f"{agg}(A)"] + expected = getattr(df["A"], agg)() + if agg == "std": + assert math.isclose(result, expected) + else: + assert result == expected + + +def test_groupby_simple(ray_start_regular_shared): + seed = int(time.time()) + print(f"Seeding RNG for test_groupby_simple with: {seed}") + random.seed(seed) + parallelism = 3 + xs = [ + ("A", 2), + ("A", 4), + ("A", 9), + ("B", 10), + ("B", 20), + ("C", 3), + ("C", 5), + ("C", 8), + ("C", 12), + ] + random.shuffle(xs) + ds = ray.data.from_items(xs, parallelism=parallelism) + + # Mean aggregation + agg_ds = ds.groupby(lambda r: r[0]).aggregate( + AggregateFn( + init=lambda k: (0, 0), + accumulate_row=lambda a, r: (a[0] + r[1], a[1] + 1), + merge=lambda a1, a2: (a1[0] + a2[0], a1[1] + a2[1]), + finalize=lambda a: a[0] / a[1], + ) + ) + assert agg_ds.count() == 3 + assert agg_ds.sort(key=lambda r: r[0]).take(3) == [("A", 5), ("B", 15), ("C", 7)] + + # Test None row + parallelism = 2 + xs = ["A", "A", "A", None, None, None, "B"] + random.shuffle(xs) + ds = ray.data.from_items(xs, parallelism=parallelism) + # Count aggregation + agg_ds = ds.groupby(lambda r: str(r)).aggregate( + AggregateFn( + init=lambda k: 0, + accumulate_row=lambda a, r: a + 1, + merge=lambda a1, a2: a1 + a2, + ) + ) + assert agg_ds.count() == 3 + assert agg_ds.sort(key=lambda r: str(r[0])).take(3) == [ + ("A", 3), + ("B", 1), + ("None", 3), + ] + + # Test empty dataset. + ds = ray.data.from_items([]) + agg_ds = ds.groupby(lambda r: r[0]).aggregate( + AggregateFn( + init=lambda k: 1 / 0, # should never reach here + accumulate_row=lambda a, r: 1 / 0, + merge=lambda a1, a2: 1 / 0, + finalize=lambda a: 1 / 0, + ) + ) + assert agg_ds.count() == 0 + assert agg_ds.take() == ds.take() + agg_ds = ray.data.range(10).filter(lambda r: r > 10).groupby(lambda r: r).count() + assert agg_ds.count() == 0 + + +@pytest.mark.parametrize("num_parts", [1, 30]) +def test_groupby_simple_count(ray_start_regular_shared, num_parts): + # Test built-in count aggregation + seed = int(time.time()) + print(f"Seeding RNG for test_groupby_simple_count with: {seed}") + random.seed(seed) + xs = list(range(100)) + random.shuffle(xs) + agg_ds = ( + ray.data.from_items(xs).repartition(num_parts).groupby(lambda x: x % 3).count() + ) + assert agg_ds.count() == 3 + assert agg_ds.sort(key=lambda r: r[0]).take(3) == [(0, 34), (1, 33), (2, 33)] + + +@pytest.mark.parametrize("num_parts", [1, 30]) +def test_groupby_simple_sum(ray_start_regular_shared, num_parts): + # Test built-in sum aggregation + seed = int(time.time()) + print(f"Seeding RNG for test_groupby_simple_sum with: {seed}") + random.seed(seed) + xs = list(range(100)) + random.shuffle(xs) + agg_ds = ( + ray.data.from_items(xs).repartition(num_parts).groupby(lambda x: x % 3).sum() + ) + assert agg_ds.count() == 3 + assert agg_ds.sort(key=lambda r: r[0]).take(3) == [(0, 1683), (1, 1617), (2, 1650)] + + # Test built-in sum aggregation with nans + nan_grouped_ds = ( + ray.data.from_items(xs + [None]) + .repartition(num_parts) + .groupby(lambda x: int(x or 0) % 3) + ) + nan_agg_ds = nan_grouped_ds.sum() + assert nan_agg_ds.count() == 3 + assert nan_agg_ds.sort(key=lambda r: r[0]).take(3) == [ + (0, 1683), + (1, 1617), + (2, 1650), + ] + # Test ignore_nulls=False + nan_agg_ds = nan_grouped_ds.sum(ignore_nulls=False) + assert nan_agg_ds.count() == 3 + assert nan_agg_ds.sort(key=lambda r: r[0]).take(3) == [ + (0, None), + (1, 1617), + (2, 1650), + ] + # Test all nans + nan_agg_ds = ( + ray.data.from_items([None] * len(xs)) + .repartition(num_parts) + .groupby(lambda x: 0) + .sum() + ) + assert nan_agg_ds.count() == 1 + assert nan_agg_ds.sort(key=lambda r: r[0]).take(1) == [(0, None)] + + # Test built-in global sum aggregation + assert ray.data.from_items(xs).repartition(num_parts).sum() == 4950 + assert ray.data.range(10).filter(lambda r: r > 10).sum() is None + + # Test built-in global sum aggregation with nans + nan_ds = ray.data.from_items(xs + [None]).repartition(num_parts) + assert nan_ds.sum() == 4950 + # Test ignore_nulls=False + assert nan_ds.sum(ignore_nulls=False) is None + # Test all nans + nan_ds = ray.data.from_items([None] * len(xs)).repartition(num_parts) + assert nan_ds.sum() is None + + +def test_groupby_map_groups_for_empty_dataset(ray_start_regular_shared): + ds = ray.data.from_items([]) + mapped = ds.groupby(lambda x: x % 3).map_groups(lambda x: [min(x) * min(x)]) + assert mapped.count() == 0 + assert mapped.take_all() == [] + + +def test_groupby_map_groups_merging_empty_result(ray_start_regular_shared): + ds = ray.data.from_items([1, 2, 3]) + # This needs to merge empty and non-empty results from different groups. + mapped = ds.groupby(lambda x: x).map_groups(lambda x: [] if x == [1] else x) + assert mapped.count() == 2 + assert mapped.take_all() == [2, 3] + + +def test_groupby_map_groups_merging_invalid_result(ray_start_regular_shared): + ds = ray.data.from_items([1, 2, 3]) + grouped = ds.groupby(lambda x: x) + + # The UDF returns None, which is invalid. + with pytest.raises(TypeError): + grouped.map_groups(lambda x: None if x == [1] else x).fully_executed() + + +@pytest.mark.parametrize("num_parts", [1, 2, 30]) +def test_groupby_map_groups_for_none_groupkey(ray_start_regular_shared, num_parts): + ds = ray.data.from_items(list(range(100))) + mapped = ( + ds.repartition(num_parts).groupby(None).map_groups(lambda x: [min(x) + max(x)]) + ) + assert mapped.count() == 1 + assert mapped.take_all() == [99] + + +@pytest.mark.parametrize("num_parts", [1, 2, 30]) +def test_groupby_map_groups_returning_empty_result(ray_start_regular_shared, num_parts): + xs = list(range(100)) + mapped = ( + ray.data.from_items(xs) + .repartition(num_parts) + .groupby(lambda x: x % 3) + .map_groups(lambda x: []) + ) + assert mapped.count() == 0 + assert mapped.take_all() == [] + + +def test_groupby_map_groups_perf(ray_start_regular_shared): + data_list = [x % 100 for x in range(5000000)] + ds = ray.data.from_pandas(pd.DataFrame({"A": data_list})) + start = time.perf_counter() + ds.groupby("A").map_groups(lambda df: df) + end = time.perf_counter() + # On a t3.2xlarge instance, it ran in about 5 seconds, so expecting it has to + # finish within about 10x of that time, unless something went wrong. + assert end - start < 60 + + +@pytest.mark.parametrize("num_parts", [1, 2, 3, 30]) +def test_groupby_map_groups_for_list(ray_start_regular_shared, num_parts): + seed = int(time.time()) + print(f"Seeding RNG for test_groupby_simple_count with: {seed}") + random.seed(seed) + xs = list(range(100)) + random.shuffle(xs) + mapped = ( + ray.data.from_items(xs) + .repartition(num_parts) + .groupby(lambda x: x % 3) + .map_groups(lambda x: [min(x) * min(x)]) + ) + assert mapped.count() == 3 + assert mapped.take_all() == [0, 1, 4] + + +@pytest.mark.parametrize("num_parts", [1, 2, 3, 30]) +def test_groupby_map_groups_for_pandas(ray_start_regular_shared, num_parts): + df = pd.DataFrame({"A": "a a b".split(), "B": [1, 1, 3], "C": [4, 6, 5]}) + grouped = ray.data.from_pandas(df).repartition(num_parts).groupby("A") + + # Normalize the numeric columns (i.e. B and C) for each group. + mapped = grouped.map_groups( + lambda g: g.apply( + lambda col: col / g[col.name].sum() if col.name in ["B", "C"] else col + ) + ) + + # The function (i.e. the normalization) performed on each group doesn't + # aggregate rows, so we still have 3 rows. + assert mapped.count() == 3 + expected = pd.DataFrame( + {"A": ["a", "a", "b"], "B": [0.5, 0.5, 1.000000], "C": [0.4, 0.6, 1.0]} + ) + assert mapped.to_pandas().equals(expected) + + +@pytest.mark.parametrize("num_parts", [1, 2, 3, 30]) +def test_groupby_map_groups_for_arrow(ray_start_regular_shared, num_parts): + at = pa.Table.from_pydict({"A": "a a b".split(), "B": [1, 1, 3], "C": [4, 6, 5]}) + grouped = ray.data.from_arrow(at).repartition(num_parts).groupby("A") + + # Normalize the numeric columns (i.e. B and C) for each group. + def normalize(at: pa.Table): + r = at.select("A") + sb = pa.compute.sum(at.column("B")).cast(pa.float64()) + r = r.append_column("B", pa.compute.divide(at.column("B"), sb)) + sc = pa.compute.sum(at.column("C")).cast(pa.float64()) + r = r.append_column("C", pa.compute.divide(at.column("C"), sc)) + return r + + mapped = grouped.map_groups(normalize, batch_format="pyarrow") + + # The function (i.e. the normalization) performed on each group doesn't + # aggregate rows, so we still have 3 rows. + assert mapped.count() == 3 + expected = pa.Table.from_pydict( + {"A": ["a", "a", "b"], "B": [0.5, 0.5, 1], "C": [0.4, 0.6, 1]} + ) + result = pa.Table.from_pandas(mapped.to_pandas()) + assert result.equals(expected) + + +def test_groupby_map_groups_for_numpy(ray_start_regular_shared): + ds = ray.data.from_items( + [ + {"group": 1, "value": 1}, + {"group": 1, "value": 2}, + {"group": 2, "value": 3}, + {"group": 2, "value": 4}, + ] + ) + + def func(group): + # Test output type is NumPy format. + return {"group": group["group"] + 1, "value": group["value"] + 1} + + ds = ds.groupby("group").map_groups(func, batch_format="numpy") + expected = pa.Table.from_pydict({"group": [2, 2, 3, 3], "value": [2, 3, 4, 5]}) + result = pa.Table.from_pandas(ds.to_pandas()) + assert result.equals(expected) + + +def test_groupby_map_groups_with_different_types(ray_start_regular_shared): + ds = ray.data.from_items( + [ + {"group": 1, "value": 1}, + {"group": 1, "value": 2}, + {"group": 2, "value": 3}, + {"group": 2, "value": 4}, + ] + ) + + def func(group): + # Test output type is Python list, different from input type. + return [group["value"][0]] + + ds = ds.groupby("group").map_groups(func) + assert sorted(ds.take()) == [1, 3] + + +@pytest.mark.parametrize("num_parts", [1, 30]) +def test_groupby_simple_min(ray_start_regular_shared, num_parts): + # Test built-in min aggregation + seed = int(time.time()) + print(f"Seeding RNG for test_groupby_simple_min with: {seed}") + random.seed(seed) + xs = list(range(100)) + random.shuffle(xs) + agg_ds = ( + ray.data.from_items(xs).repartition(num_parts).groupby(lambda x: x % 3).min() + ) + assert agg_ds.count() == 3 + assert agg_ds.sort(key=lambda r: r[0]).take(3) == [(0, 0), (1, 1), (2, 2)] + + # Test built-in min aggregation with nans + nan_grouped_ds = ( + ray.data.from_items(xs + [None]) + .repartition(num_parts) + .groupby(lambda x: int(x or 0) % 3) + ) + nan_agg_ds = nan_grouped_ds.min() + assert nan_agg_ds.count() == 3 + assert nan_agg_ds.sort(key=lambda r: r[0]).take(3) == [(0, 0), (1, 1), (2, 2)] + # Test ignore_nulls=False + nan_agg_ds = nan_grouped_ds.min(ignore_nulls=False) + assert nan_agg_ds.count() == 3 + assert nan_agg_ds.sort(key=lambda r: r[0]).take(3) == [(0, None), (1, 1), (2, 2)] + # Test all nans + nan_agg_ds = ( + ray.data.from_items([None] * len(xs)) + .repartition(num_parts) + .groupby(lambda x: 0) + .min() + ) + assert nan_agg_ds.count() == 1 + assert nan_agg_ds.sort(key=lambda r: r[0]).take(1) == [(0, None)] + + # Test built-in global min aggregation + assert ray.data.from_items(xs).repartition(num_parts).min() == 0 + assert ray.data.range(10).filter(lambda r: r > 10).min() is None + + # Test built-in global min aggregation with nans + nan_ds = ray.data.from_items(xs + [None]).repartition(num_parts) + assert nan_ds.min() == 0 + # Test ignore_nulls=False + assert nan_ds.min(ignore_nulls=False) is None + # Test all nans + nan_ds = ray.data.from_items([None] * len(xs)).repartition(num_parts) + assert nan_ds.min() is None + + +@pytest.mark.parametrize("num_parts", [1, 30]) +def test_groupby_simple_max(ray_start_regular_shared, num_parts): + # Test built-in max aggregation + seed = int(time.time()) + print(f"Seeding RNG for test_groupby_simple_max with: {seed}") + random.seed(seed) + xs = list(range(100)) + random.shuffle(xs) + agg_ds = ( + ray.data.from_items(xs).repartition(num_parts).groupby(lambda x: x % 3).max() + ) + assert agg_ds.count() == 3 + assert agg_ds.sort(key=lambda r: r[0]).take(3) == [(0, 99), (1, 97), (2, 98)] + + # Test built-in max aggregation with nans + nan_grouped_ds = ( + ray.data.from_items(xs + [None]) + .repartition(num_parts) + .groupby(lambda x: int(x or 0) % 3) + ) + nan_agg_ds = nan_grouped_ds.max() + assert nan_agg_ds.count() == 3 + assert nan_agg_ds.sort(key=lambda r: r[0]).take(3) == [(0, 99), (1, 97), (2, 98)] + # Test ignore_nulls=False + nan_agg_ds = nan_grouped_ds.max(ignore_nulls=False) + assert nan_agg_ds.count() == 3 + assert nan_agg_ds.sort(key=lambda r: r[0]).take(3) == [(0, None), (1, 97), (2, 98)] + # Test all nans + nan_agg_ds = ( + ray.data.from_items([None] * len(xs)) + .repartition(num_parts) + .groupby(lambda x: 0) + .max() + ) + assert nan_agg_ds.count() == 1 + assert nan_agg_ds.sort(key=lambda r: r[0]).take(1) == [(0, None)] + + # Test built-in global max aggregation + assert ray.data.from_items(xs).repartition(num_parts).max() == 99 + assert ray.data.range(10).filter(lambda r: r > 10).max() is None + + # Test built-in global max aggregation with nans + nan_ds = ray.data.from_items(xs + [None]).repartition(num_parts) + assert nan_ds.max() == 99 + # Test ignore_nulls=False + assert nan_ds.max(ignore_nulls=False) is None + # Test all nans + nan_ds = ray.data.from_items([None] * len(xs)).repartition(num_parts) + assert nan_ds.max() is None + + +@pytest.mark.parametrize("num_parts", [1, 30]) +def test_groupby_simple_mean(ray_start_regular_shared, num_parts): + # Test built-in mean aggregation + seed = int(time.time()) + print(f"Seeding RNG for test_groupby_simple_mean with: {seed}") + random.seed(seed) + xs = list(range(100)) + random.shuffle(xs) + agg_ds = ( + ray.data.from_items(xs).repartition(num_parts).groupby(lambda x: x % 3).mean() + ) + assert agg_ds.count() == 3 + assert agg_ds.sort(key=lambda r: r[0]).take(3) == [(0, 49.5), (1, 49.0), (2, 50.0)] + + # Test built-in mean aggregation with nans + nan_grouped_ds = ( + ray.data.from_items(xs + [None]) + .repartition(num_parts) + .groupby(lambda x: int(x or 0) % 3) + ) + nan_agg_ds = nan_grouped_ds.mean() + assert nan_agg_ds.count() == 3 + assert nan_agg_ds.sort(key=lambda r: r[0]).take(3) == [ + (0, 49.5), + (1, 49.0), + (2, 50.0), + ] + # Test ignore_nulls=False + nan_agg_ds = nan_grouped_ds.mean(ignore_nulls=False) + assert nan_agg_ds.count() == 3 + assert nan_agg_ds.sort(key=lambda r: r[0]).take(3) == [ + (0, None), + (1, 49.0), + (2, 50.0), + ] + # Test all nans + nan_agg_ds = ( + ray.data.from_items([None] * len(xs)) + .repartition(num_parts) + .groupby(lambda x: 0) + .mean() + ) + assert nan_agg_ds.count() == 1 + assert nan_agg_ds.sort(key=lambda r: r[0]).take(1) == [(0, None)] + + # Test built-in global mean aggregation + assert ray.data.from_items(xs).repartition(num_parts).mean() == 49.5 + # Test empty dataset + assert ray.data.range(10).filter(lambda r: r > 10).mean() is None + + # Test built-in global mean aggregation with nans + nan_ds = ray.data.from_items(xs + [None]).repartition(num_parts) + assert nan_ds.mean() == 49.5 + # Test ignore_nulls=False + assert nan_ds.mean(ignore_nulls=False) is None + # Test all nans + nan_ds = ray.data.from_items([None] * len(xs)).repartition(num_parts) + assert nan_ds.mean() is None + + +@pytest.mark.parametrize("num_parts", [1, 30]) +def test_groupby_simple_std(ray_start_regular_shared, num_parts): + # Test built-in std aggregation + seed = int(time.time()) + print(f"Seeding RNG for test_groupby_simple_std with: {seed}") + random.seed(seed) + xs = list(range(100)) + random.shuffle(xs) + agg_ds = ( + ray.data.from_items(xs).repartition(num_parts).groupby(lambda x: x % 3).std() + ) + assert agg_ds.count() == 3 + df = pd.DataFrame({"A": [x % 3 for x in xs], "B": xs}) + expected = df.groupby("A")["B"].std() + result = agg_ds.sort(key=lambda r: r[0]).take(3) + groups, stds = zip(*result) + result_df = pd.DataFrame({"A": list(groups), "B": list(stds)}) + result_df = result_df.set_index("A") + pd.testing.assert_series_equal(result_df["B"], expected) + # ddof of 0 + agg_ds = ( + ray.data.from_items(xs) + .repartition(num_parts) + .groupby(lambda x: x % 3) + .std(ddof=0) + ) + assert agg_ds.count() == 3 + df = pd.DataFrame({"A": [x % 3 for x in xs], "B": xs}) + expected = df.groupby("A")["B"].std(ddof=0) + result = agg_ds.sort(key=lambda r: r[0]).take(3) + groups, stds = zip(*result) + result_df = pd.DataFrame({"A": list(groups), "B": list(stds)}) + result_df = result_df.set_index("A") + pd.testing.assert_series_equal(result_df["B"], expected) + + # Test built-in std aggregation with nans + nan_grouped_ds = ( + ray.data.from_items(xs + [None]) + .repartition(num_parts) + .groupby(lambda x: int(x or 0) % 3) + ) + nan_agg_ds = nan_grouped_ds.std() + assert nan_agg_ds.count() == 3 + nan_df = pd.DataFrame({"A": [x % 3 for x in xs] + [0], "B": xs + [None]}) + expected = nan_df.groupby("A")["B"].std() + result = nan_agg_ds.sort(key=lambda r: r[0]).take(3) + groups, stds = zip(*result) + result_df = pd.DataFrame({"A": list(groups), "B": list(stds)}) + result_df = result_df.set_index("A") + pd.testing.assert_series_equal(result_df["B"], expected) + # Test ignore_nulls=False + nan_agg_ds = nan_grouped_ds.std(ignore_nulls=False) + assert nan_agg_ds.count() == 3 + expected = nan_df.groupby("A")["B"].std() + expected[0] = None + result = nan_agg_ds.sort(key=lambda r: r[0]).take(3) + groups, stds = zip(*result) + result_df = pd.DataFrame({"A": list(groups), "B": list(stds)}) + result_df = result_df.set_index("A") + pd.testing.assert_series_equal(result_df["B"], expected) + # Test all nans + nan_agg_ds = ( + ray.data.from_items([None] * len(xs)) + .repartition(num_parts) + .groupby(lambda x: 0) + .std(ignore_nulls=False) + ) + assert nan_agg_ds.count() == 1 + expected = pd.Series([None], name="B") + expected.index.rename("A", inplace=True) + result = nan_agg_ds.sort(key=lambda r: r[0]).take(1) + groups, stds = zip(*result) + result_df = pd.DataFrame({"A": list(groups), "B": list(stds)}) + result_df = result_df.set_index("A") + pd.testing.assert_series_equal(result_df["B"], expected) + + # Test built-in global std aggregation + assert math.isclose( + ray.data.from_items(xs).repartition(num_parts).std(), pd.Series(xs).std() + ) + # ddof of 0 + assert math.isclose( + ray.data.from_items(xs).repartition(num_parts).std(ddof=0), + pd.Series(xs).std(ddof=0), + ) + + # Test empty dataset + assert ray.data.from_items([]).std() is None + # Test edge cases + assert ray.data.from_items([3]).std() == 0 + + # Test built-in global std aggregation with nans + nan_ds = ray.data.from_items(xs + [None]).repartition(num_parts) + assert math.isclose(nan_ds.std(), pd.Series(xs).std()) + # Test ignore_nulls=False + assert nan_ds.std(ignore_nulls=False) is None + # Test all nans + nan_ds = ray.data.from_items([None] * len(xs)).repartition(num_parts) + assert nan_ds.std() is None + + +@pytest.mark.parametrize("num_parts", [1, 30]) +def test_groupby_simple_multilambda(ray_start_regular_shared, num_parts): + # Test built-in mean aggregation + seed = int(time.time()) + print(f"Seeding RNG for test_groupby_simple_multilambda with: {seed}") + random.seed(seed) + xs = list(range(100)) + random.shuffle(xs) + agg_ds = ( + ray.data.from_items([[x, 2 * x] for x in xs]) + .repartition(num_parts) + .groupby(lambda x: x[0] % 3) + .mean([lambda x: x[0], lambda x: x[1]]) + ) + assert agg_ds.count() == 3 + assert agg_ds.sort(key=lambda r: r[0]).take(3) == [ + (0, 49.5, 99.0), + (1, 49.0, 98.0), + (2, 50.0, 100.0), + ] + # Test built-in global mean aggregation + assert ray.data.from_items([[x, 2 * x] for x in xs]).repartition(num_parts).mean( + [lambda x: x[0], lambda x: x[1]] + ) == (49.5, 99.0) + assert ray.data.from_items([[x, 2 * x] for x in range(10)]).filter( + lambda r: r[0] > 10 + ).mean([lambda x: x[0], lambda x: x[1]]) == (None, None) + + +@pytest.mark.parametrize("num_parts", [1, 30]) +def test_groupby_simple_multi_agg(ray_start_regular_shared, num_parts): + seed = int(time.time()) + print(f"Seeding RNG for test_groupby_simple_multi_agg with: {seed}") + random.seed(seed) + xs = list(range(100)) + random.shuffle(xs) + df = pd.DataFrame({"A": [x % 3 for x in xs], "B": xs}) + agg_ds = ( + ray.data.from_items(xs) + .repartition(num_parts) + .groupby(lambda x: x % 3) + .aggregate( + Count(), + Sum(), + Min(), + Max(), + Mean(), + Std(), + ) + ) + assert agg_ds.count() == 3 + result = agg_ds.sort(key=lambda r: r[0]).take(3) + groups, counts, sums, mins, maxs, means, stds = zip(*result) + agg_df = pd.DataFrame( + { + "groups": list(groups), + "count": list(counts), + "sum": list(sums), + "min": list(mins), + "max": list(maxs), + "mean": list(means), + "std": list(stds), + } + ) + agg_df = agg_df.set_index("groups") + df = pd.DataFrame({"groups": [x % 3 for x in xs], "B": xs}) + expected_grouped = df.groupby("groups")["B"] + np.testing.assert_array_equal(agg_df["count"].to_numpy(), [34, 33, 33]) + for agg in ["sum", "min", "max", "mean", "std"]: + result = agg_df[agg].to_numpy() + expected = getattr(expected_grouped, agg)().to_numpy() + if agg == "std": + np.testing.assert_array_almost_equal(result, expected) + else: + np.testing.assert_array_equal(result, expected) + # Test built-in global multi-aggregation + result_row = ( + ray.data.from_items(xs) + .repartition(num_parts) + .aggregate( + Sum(), + Min(), + Max(), + Mean(), + Std(), + ) + ) + series = pd.Series(xs) + for idx, agg in enumerate(["sum", "min", "max", "mean", "std"]): + result = result_row[idx] + expected = getattr(series, agg)() + if agg == "std": + assert math.isclose(result, expected) + else: + assert result == expected + + +def test_random_block_order_schema(ray_start_regular_shared): + df = pd.DataFrame({"a": np.random.rand(10), "b": np.random.rand(10)}) + ds = ray.data.from_pandas(df).randomize_block_order() + ds.schema().names == ["a", "b"] + + +def test_random_block_order(ray_start_regular_shared, restore_dataset_context): + ctx = DatasetContext.get_current() + ctx.execution_options.preserve_order = True + + # Test BlockList.randomize_block_order. + ds = ray.data.range(12).repartition(4) + ds = ds.randomize_block_order(seed=0) + + results = ds.take() + expected = [6, 7, 8, 0, 1, 2, 3, 4, 5, 9, 10, 11] + assert results == expected + + # Test LazyBlockList.randomize_block_order. + context = DatasetContext.get_current() + try: + original_optimize_fuse_read_stages = context.optimize_fuse_read_stages + context.optimize_fuse_read_stages = False + + lazy_blocklist_ds = ray.data.range(12, parallelism=4) + lazy_blocklist_ds = lazy_blocklist_ds.randomize_block_order(seed=0) + lazy_blocklist_results = lazy_blocklist_ds.take() + lazy_blocklist_expected = [6, 7, 8, 0, 1, 2, 3, 4, 5, 9, 10, 11] + assert lazy_blocklist_results == lazy_blocklist_expected + finally: + context.optimize_fuse_read_stages = original_optimize_fuse_read_stages + + +# NOTE: All tests above share a Ray cluster, while the tests below do not. These +# tests should only be carefully reordered to retain this invariant! + + +@pytest.mark.parametrize("pipelined", [False, True]) +def test_random_shuffle(shutdown_only, pipelined, use_push_based_shuffle): + def range(n, parallelism=200): + ds = ray.data.range(n, parallelism=parallelism) + if pipelined: + pipe = ds.repeat(2) + pipe.random_shuffle = pipe.random_shuffle_each_window + return pipe + else: + return ds + + r1 = range(100).random_shuffle().take(999) + r2 = range(100).random_shuffle().take(999) + assert r1 != r2, (r1, r2) + + r1 = range(100, parallelism=1).random_shuffle().take(999) + r2 = range(100, parallelism=1).random_shuffle().take(999) + assert r1 != r2, (r1, r2) + + # TODO(swang): fix this + if not use_push_based_shuffle: + if not pipelined: + assert range(100).random_shuffle(num_blocks=1).num_blocks() == 1 + r1 = range(100).random_shuffle(num_blocks=1).take(999) + r2 = range(100).random_shuffle(num_blocks=1).take(999) + assert r1 != r2, (r1, r2) + + r0 = range(100, parallelism=5).take(999) + r1 = range(100, parallelism=5).random_shuffle(seed=0).take(999) + r2 = range(100, parallelism=5).random_shuffle(seed=0).take(999) + r3 = range(100, parallelism=5).random_shuffle(seed=12345).take(999) + assert r1 == r2, (r1, r2) + assert r1 != r0, (r1, r0) + assert r1 != r3, (r1, r3) + + r0 = ray.data.range_table(100, parallelism=5).take(999) + r1 = ray.data.range_table(100, parallelism=5).random_shuffle(seed=0).take(999) + r2 = ray.data.range_table(100, parallelism=5).random_shuffle(seed=0).take(999) + assert r1 == r2, (r1, r2) + assert r1 != r0, (r1, r0) + + # Test move. + ds = range(100, parallelism=2) + r1 = ds.random_shuffle().take(999) + if pipelined: + with pytest.raises(RuntimeError): + ds = ds.map(lambda x: x).take(999) + else: + ds = ds.map(lambda x: x).take(999) + r2 = range(100).random_shuffle().take(999) + assert r1 != r2, (r1, r2) + + # Test empty dataset. + ds = ray.data.from_items([]) + r1 = ds.random_shuffle() + assert r1.count() == 0 + assert r1.take() == ds.take() + + +def test_random_shuffle_check_random(shutdown_only): + # Rows from the same input should not be contiguous in the final output. + num_files = 10 + num_rows = 100 + items = [i for i in range(num_files) for _ in range(num_rows)] + ds = ray.data.from_items(items, parallelism=num_files) + out = ds.random_shuffle().take(num_files * num_rows) + for i in range(num_files): + part = out[i * num_rows : (i + 1) * num_rows] + seen = set() + num_contiguous = 1 + prev = -1 + for x in part: + if prev != x: + prev = x + num_contiguous = 1 + else: + num_contiguous += 1 + assert num_contiguous < ( + num_rows / num_files + ), f"{part} contains too many contiguous rows from same input block" + seen.add(x) + assert ( + set(range(num_files)) == seen + ), f"{part} does not contain elements from all input blocks" + + # Rows from the same input should appear in a different order in the + # output. + num_files = 10 + num_rows = 100 + items = [j for i in range(num_files) for j in range(num_rows)] + ds = ray.data.from_items(items, parallelism=num_files) + out = ds.random_shuffle().take(num_files * num_rows) + for i in range(num_files): + part = out[i * num_rows : (i + 1) * num_rows] + num_increasing = 0 + prev = -1 + for x in part: + if x >= prev: + num_increasing += 1 + else: + assert num_increasing < ( + num_rows / num_files + ), f"{part} contains non-shuffled rows from input blocks" + num_increasing = 0 + prev = x + + +def test_random_shuffle_with_custom_resource(ray_start_cluster): + cluster = ray_start_cluster + # Create two nodes which have different custom resources. + cluster.add_node( + resources={"foo": 100}, + num_cpus=1, + ) + cluster.add_node(resources={"bar": 100}, num_cpus=1) + + ray.init(cluster.address) + + # Run dataset in "bar" nodes. + ds = ray.data.read_parquet( + "example://parquet_images_mini", + parallelism=2, + ray_remote_args={"resources": {"bar": 1}}, + ) + ds = ds.random_shuffle(resources={"bar": 1}).fully_executed() + assert "1 nodes used" in ds.stats() + assert "2 nodes used" not in ds.stats() + + +def test_random_shuffle_spread(ray_start_cluster, use_push_based_shuffle): + cluster = ray_start_cluster + cluster.add_node( + resources={"bar:1": 100}, + num_cpus=10, + _system_config={"max_direct_call_object_size": 0}, + ) + cluster.add_node(resources={"bar:2": 100}, num_cpus=10) + cluster.add_node(resources={"bar:3": 100}, num_cpus=0) + + ray.init(cluster.address) + + @ray.remote + def get_node_id(): + return ray.get_runtime_context().get_node_id() + + node1_id = ray.get(get_node_id.options(resources={"bar:1": 1}).remote()) + node2_id = ray.get(get_node_id.options(resources={"bar:2": 1}).remote()) + + ds = ray.data.range(100, parallelism=2).random_shuffle() + blocks = ds.get_internal_block_refs() + ray.wait(blocks, num_returns=len(blocks), fetch_local=False) + location_data = ray.experimental.get_object_locations(blocks) + locations = [] + for block in blocks: + locations.extend(location_data[block]["node_ids"]) + assert "2 nodes used" in ds.stats() + + if not use_push_based_shuffle: + # We don't check this for push-based shuffle since it will try to + # colocate reduce tasks to improve locality. + assert set(locations) == {node1_id, node2_id} + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main(["-v", __file__])) diff --git a/python/ray/data/tests/test_dataset_consumption.py b/python/ray/data/tests/test_dataset_consumption.py new file mode 100644 index 0000000000000..b3a5bbfd9e058 --- /dev/null +++ b/python/ray/data/tests/test_dataset_consumption.py @@ -0,0 +1,1716 @@ +import logging +import math +import os +import random +import time + +import numpy as np +import pandas as pd +import pyarrow as pa +import pytest +from unittest.mock import patch + +import ray +from ray.data._internal.arrow_block import ArrowRow +from ray.data._internal.block_builder import BlockBuilder +from ray.data._internal.dataset_logger import DatasetLogger +from ray.data._internal.lazy_block_list import LazyBlockList +from ray.data._internal.pandas_block import PandasRow +from ray.data.block import BlockAccessor, BlockMetadata +from ray.data.context import DatasetContext +from ray.data.dataset import Dataset, _sliding_window +from ray.data.datasource.datasource import Datasource, ReadTask +from ray.data.datasource.csv_datasource import CSVDatasource +from ray.data.row import TableRow +from ray.data.tests.conftest import * # noqa +from ray.tests.conftest import * # noqa +from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy + + +def maybe_pipeline(ds, enabled): + if enabled: + return ds.window(blocks_per_window=1) + else: + return ds + + +@pytest.mark.parametrize("pipelined", [False, True]) +def test_avoid_placement_group_capture(shutdown_only, pipelined): + ray.init(num_cpus=2) + + @ray.remote + def run(): + ds0 = ray.data.range(5) + ds = maybe_pipeline(ds0, pipelined) + assert sorted(ds.map(lambda x: x + 1).take()) == [1, 2, 3, 4, 5] + ds = maybe_pipeline(ds0, pipelined) + assert ds.count() == 5 + ds = maybe_pipeline(ds0, pipelined) + assert sorted(ds.iter_rows()) == [0, 1, 2, 3, 4] + + pg = ray.util.placement_group([{"CPU": 1}]) + ray.get( + run.options( + scheduling_strategy=PlacementGroupSchedulingStrategy( + placement_group=pg, placement_group_capture_child_tasks=True + ) + ).remote() + ) + + +def test_dataset_lineage_serialization(shutdown_only): + ray.init() + ds = ray.data.range(10) + ds = ds.map(lambda x: x + 1) + ds = ds.map(lambda x: x + 1) + ds = ds.random_shuffle() + epoch = ds._get_epoch() + uuid = ds._get_uuid() + plan_uuid = ds._plan._dataset_uuid + + serialized_ds = ds.serialize_lineage() + # Confirm that the original Dataset was properly copied before clearing/mutating. + in_blocks = ds._plan._in_blocks + # Should not raise. + in_blocks._check_if_cleared() + assert isinstance(in_blocks, LazyBlockList) + assert in_blocks._block_partition_refs[0] is None + + ray.shutdown() + ray.init() + + ds = Dataset.deserialize_lineage(serialized_ds) + # Check Dataset state. + assert ds._get_epoch() == epoch + assert ds._get_uuid() == uuid + assert ds._plan._dataset_uuid == plan_uuid + # Check Dataset content. + assert ds.count() == 10 + assert sorted(ds.take()) == list(range(2, 12)) + + +def test_dataset_lineage_serialization_unsupported(shutdown_only): + ray.init() + # In-memory data sources not supported. + ds = ray.data.from_items(list(range(10))) + ds = ds.map(lambda x: x + 1) + ds = ds.map(lambda x: x + 1) + + with pytest.raises(ValueError): + ds.serialize_lineage() + + # In-memory data source unions not supported. + ds = ray.data.from_items(list(range(10))) + ds1 = ray.data.from_items(list(range(10, 20))) + ds2 = ds.union(ds1) + + with pytest.raises(ValueError): + ds2.serialize_lineage() + + # Post-lazy-read unions not supported. + ds = ray.data.range(10).map(lambda x: x + 1) + ds1 = ray.data.range(20).map(lambda x: 2 * x) + ds2 = ds.union(ds1) + + with pytest.raises(ValueError): + ds2.serialize_lineage() + + # Lazy read unions supported. + ds = ray.data.range(10) + ds1 = ray.data.range(20) + ds2 = ds.union(ds1) + + serialized_ds = ds2.serialize_lineage() + ds3 = Dataset.deserialize_lineage(serialized_ds) + assert ds3.take(30) == list(range(10)) + list(range(20)) + + # Zips not supported. + ds = ray.data.from_items(list(range(10))) + ds1 = ray.data.from_items(list(range(10, 20))) + ds2 = ds.zip(ds1) + + with pytest.raises(ValueError): + ds2.serialize_lineage() + + +@pytest.mark.parametrize("pipelined", [False, True]) +def test_basic(ray_start_regular_shared, pipelined): + ds0 = ray.data.range(5) + ds = maybe_pipeline(ds0, pipelined) + assert sorted(ds.map(lambda x: x + 1).take()) == [1, 2, 3, 4, 5] + ds = maybe_pipeline(ds0, pipelined) + assert ds.count() == 5 + ds = maybe_pipeline(ds0, pipelined) + assert sorted(ds.iter_rows()) == [0, 1, 2, 3, 4] + + +def test_range_table(ray_start_regular_shared): + ds = ray.data.range_table(10, parallelism=10) + assert ds.num_blocks() == 10 + assert ds.count() == 10 + assert ds.take() == [{"value": i} for i in range(10)] + + ds = ray.data.range_table(10, parallelism=2) + assert ds.num_blocks() == 2 + assert ds.count() == 10 + assert ds.take() == [{"value": i} for i in range(10)] + + +def test_empty_dataset(ray_start_regular_shared): + ds = ray.data.range(0) + assert ds.count() == 0 + assert ds.size_bytes() is None + assert ds.schema() is None + + ds = ray.data.range(1) + ds = ds.filter(lambda x: x > 1) + ds.fully_executed() + assert str(ds) == "Dataset(num_blocks=1, num_rows=0, schema=Unknown schema)" + + # Test map on empty dataset. + ds = ray.data.from_items([]) + ds = ds.map(lambda x: x) + ds.fully_executed() + assert ds.count() == 0 + + # Test filter on empty dataset. + ds = ray.data.from_items([]) + ds = ds.filter(lambda: True) + ds.fully_executed() + assert ds.count() == 0 + + +def test_schema(ray_start_regular_shared): + ds = ray.data.range(10, parallelism=10) + ds2 = ray.data.range_table(10, parallelism=10) + ds3 = ds2.repartition(5) + ds3.fully_executed() + ds4 = ds3.map(lambda x: {"a": "hi", "b": 1.0}).limit(5).repartition(1) + ds4.fully_executed() + assert str(ds) == "Dataset(num_blocks=10, num_rows=10, schema=)" + assert str(ds2) == "Dataset(num_blocks=10, num_rows=10, schema={value: int64})" + assert str(ds3) == "Dataset(num_blocks=5, num_rows=10, schema={value: int64})" + assert ( + str(ds4) == "Dataset(num_blocks=1, num_rows=5, schema={a: string, b: double})" + ) + + +def test_schema_lazy(ray_start_regular_shared): + ds = ray.data.range(100, parallelism=10) + # We do not kick off the read task by default. + assert ds._plan._in_blocks._num_computed() == 0 + schema = ds.schema() + assert schema == int + assert ds._plan._in_blocks._num_computed() == 1 + # Fetching the schema should not trigger execution of extra read tasks. + assert ds._plan.execute()._num_computed() == 1 + + +def test_count_lazy(ray_start_regular_shared): + ds = ray.data.range(100, parallelism=10) + # We do not kick off the read task by default. + assert ds._plan._in_blocks._num_computed() == 0 + assert ds.count() == 100 + # Getting number of rows should not trigger execution of any read tasks + # for ray.data.range(), as the number of rows is known beforehand. + assert ds._plan._in_blocks._num_computed() == 0 + + +def test_lazy_loading_exponential_rampup(ray_start_regular_shared): + ds = ray.data.range(100, parallelism=20) + + def check_num_computed(expected): + if ray.data.context.DatasetContext.get_current().use_streaming_executor: + # In streaing executor, ds.take() will not invoke partial execution + # in LazyBlocklist. + assert ds._plan.execute()._num_computed() == 0 + else: + assert ds._plan.execute()._num_computed() == expected + + check_num_computed(0) + assert ds.take(10) == list(range(10)) + check_num_computed(2) + assert ds.take(20) == list(range(20)) + check_num_computed(4) + assert ds.take(30) == list(range(30)) + check_num_computed(8) + assert ds.take(50) == list(range(50)) + check_num_computed(16) + assert ds.take(100) == list(range(100)) + check_num_computed(20) + + +def test_dataset_repr(ray_start_regular_shared): + ds = ray.data.range(10, parallelism=10) + assert repr(ds) == "Dataset(num_blocks=10, num_rows=10, schema=)" + ds = ds.map_batches(lambda x: x) + assert repr(ds) == ( + "MapBatches()\n" + "+- Dataset(num_blocks=10, num_rows=10, schema=)" + ) + ds = ds.filter(lambda x: x > 0) + assert repr(ds) == ( + "Filter\n" + "+- MapBatches()\n" + " +- Dataset(num_blocks=10, num_rows=10, schema=)" + ) + ds = ds.random_shuffle() + assert repr(ds) == ( + "RandomShuffle\n" + "+- Filter\n" + " +- MapBatches()\n" + " +- Dataset(num_blocks=10, num_rows=10, schema=)" + ) + ds.fully_executed() + assert repr(ds) == "Dataset(num_blocks=10, num_rows=9, schema=)" + ds = ds.map_batches(lambda x: x) + assert repr(ds) == ( + "MapBatches()\n" + "+- Dataset(num_blocks=10, num_rows=9, schema=)" + ) + ds1, ds2 = ds.split(2) + assert ( + repr(ds1) + == f"Dataset(num_blocks=5, num_rows={ds1.count()}, schema=)" + ) + assert ( + repr(ds2) + == f"Dataset(num_blocks=5, num_rows={ds2.count()}, schema=)" + ) + ds3 = ds1.union(ds2) + assert repr(ds3) == "Dataset(num_blocks=10, num_rows=9, schema=)" + ds = ds.zip(ds3) + assert repr(ds) == ( + "Zip\n" "+- Dataset(num_blocks=10, num_rows=9, schema=)" + ) + + def my_dummy_fn(x): + return x + + ds = ray.data.range(10, parallelism=10) + ds = ds.map_batches(my_dummy_fn) + assert repr(ds) == ( + "MapBatches(my_dummy_fn)\n" + "+- Dataset(num_blocks=10, num_rows=10, schema=)" + ) + + +@pytest.mark.parametrize("lazy", [False, True]) +def test_limit(ray_start_regular_shared, lazy): + ds = ray.data.range(100, parallelism=20) + if not lazy: + ds = ds.fully_executed() + for i in range(100): + assert ds.limit(i).take(200) == list(range(i)) + + +# NOTE: We test outside the power-of-2 range in order to ensure that we're not reading +# redundant files due to exponential ramp-up. +@pytest.mark.parametrize("limit,expected", [(10, 1), (20, 2), (30, 3), (60, 6)]) +def test_limit_no_redundant_read(ray_start_regular_shared, limit, expected): + # Test that dataset truncation eliminates redundant reads. + @ray.remote + class Counter: + def __init__(self): + self.count = 0 + + def increment(self): + self.count += 1 + + def get(self): + return self.count + + def reset(self): + self.count = 0 + + class CountingRangeDatasource(Datasource): + def __init__(self): + self.counter = Counter.remote() + + def prepare_read(self, parallelism, n): + def range_(i): + ray.get(self.counter.increment.remote()) + return [list(range(parallelism * i, parallelism * i + n))] + + return [ + ReadTask( + lambda i=i: range_(i), + BlockMetadata( + num_rows=n, + size_bytes=None, + schema=None, + input_files=None, + exec_stats=None, + ), + ) + for i in range(parallelism) + ] + + source = CountingRangeDatasource() + + ds = ray.data.read_datasource( + source, + parallelism=10, + n=10, + ) + ds2 = ds.limit(limit) + # Check content. + assert ds2.take(limit) == list(range(limit)) + # Check number of read tasks launched. + assert ray.get(source.counter.get.remote()) == expected + + +def test_limit_no_num_row_info(ray_start_regular_shared): + # Test that datasources with no number-of-rows metadata available are still able to + # be truncated, falling back to kicking off all read tasks. + class DumbOnesDatasource(Datasource): + def prepare_read(self, parallelism, n): + return parallelism * [ + ReadTask( + lambda: [[1] * n], + BlockMetadata( + num_rows=None, + size_bytes=None, + schema=None, + input_files=None, + exec_stats=None, + ), + ) + ] + + ds = ray.data.read_datasource(DumbOnesDatasource(), parallelism=10, n=10) + for i in range(1, 100): + assert ds.limit(i).take(100) == [1] * i + + +def test_convert_types(ray_start_regular_shared): + plain_ds = ray.data.range(1) + arrow_ds = plain_ds.map(lambda x: {"a": x}) + assert arrow_ds.take() == [{"a": 0}] + assert "ArrowRow" in arrow_ds.map(lambda x: str(type(x))).take()[0] + + arrow_ds = ray.data.range_table(1) + assert arrow_ds.map(lambda x: "plain_{}".format(x["value"])).take() == ["plain_0"] + # In streaming, we set batch_format to "default" (because calling + # ds.dataset_format() will still invoke bulk execution and we want + # to avoid that). As a result, it's receiving PandasRow (the defaut + # batch format), which unwraps [0] to plain 0. + if ray.data.context.DatasetContext.get_current().use_streaming_executor: + assert arrow_ds.map(lambda x: {"a": (x["value"],)}).take() == [{"a": 0}] + else: + assert arrow_ds.map(lambda x: {"a": (x["value"],)}).take() == [{"a": [0]}] + + +def test_from_items(ray_start_regular_shared): + ds = ray.data.from_items(["hello", "world"]) + assert ds.take() == ["hello", "world"] + + +@pytest.mark.parametrize("parallelism", list(range(1, 21))) +def test_from_items_parallelism(ray_start_regular_shared, parallelism): + # Test that specifying parallelism yields the expected number of blocks. + n = 20 + records = [{"a": i} for i in range(n)] + ds = ray.data.from_items(records, parallelism=parallelism) + out = ds.take_all() + assert out == records + assert ds.num_blocks() == parallelism + + +def test_from_items_parallelism_truncated(ray_start_regular_shared): + # Test that specifying parallelism greater than the number of items is truncated to + # the number of items. + n = 10 + parallelism = 20 + records = [{"a": i} for i in range(n)] + ds = ray.data.from_items(records, parallelism=parallelism) + out = ds.take_all() + assert out == records + assert ds.num_blocks() == n + + +def test_take_all(ray_start_regular_shared): + assert ray.data.range(5).take_all() == [0, 1, 2, 3, 4] + + with pytest.raises(ValueError): + assert ray.data.range(5).take_all(4) + + +def test_sliding_window(): + arr = list(range(10)) + + # Test all windows over this iterable. + window_sizes = list(range(1, len(arr) + 1)) + for window_size in window_sizes: + windows = list(_sliding_window(arr, window_size)) + assert len(windows) == len(arr) - window_size + 1 + assert all(len(window) == window_size for window in windows) + assert all( + list(window) == arr[i : i + window_size] for i, window in enumerate(windows) + ) + + # Test window size larger than iterable length. + windows = list(_sliding_window(arr, 15)) + assert len(windows) == 1 + assert list(windows[0]) == arr + + +def test_iter_rows(ray_start_regular_shared): + # Test simple rows. + n = 10 + ds = ray.data.range(n) + for row, k in zip(ds.iter_rows(), range(n)): + assert row == k + + # Test tabular rows. + t1 = pa.Table.from_pydict({"one": [1, 2, 3], "two": [2, 3, 4]}) + t2 = pa.Table.from_pydict({"one": [4, 5, 6], "two": [5, 6, 7]}) + t3 = pa.Table.from_pydict({"one": [7, 8, 9], "two": [8, 9, 10]}) + t4 = pa.Table.from_pydict({"one": [10, 11, 12], "two": [11, 12, 13]}) + ts = [t1, t2, t3, t4] + t = pa.concat_tables(ts) + ds = ray.data.from_arrow(ts) + + def to_pylist(table): + pydict = table.to_pydict() + names = table.schema.names + pylist = [ + {column: pydict[column][row] for column in names} + for row in range(table.num_rows) + ] + return pylist + + # Default ArrowRows. + for row, t_row in zip(ds.iter_rows(), to_pylist(t)): + assert isinstance(row, TableRow) + # In streaming, we set batch_format to "default" because calling + # ds.dataset_format() will still invoke bulk execution and we want + # to avoid that. As a result, it's receiving PandasRow (the defaut + # batch format). + if ray.data.context.DatasetContext.get_current().use_streaming_executor: + assert isinstance(row, PandasRow) + else: + assert isinstance(row, ArrowRow) + assert row == t_row + + # PandasRows after conversion. + pandas_ds = ds.map_batches(lambda x: x, batch_format="pandas") + df = t.to_pandas() + for row, (index, df_row) in zip(pandas_ds.iter_rows(), df.iterrows()): + assert isinstance(row, TableRow) + assert isinstance(row, PandasRow) + assert row == df_row.to_dict() + + # Prefetch. + for row, t_row in zip(ds.iter_rows(prefetch_blocks=1), to_pylist(t)): + assert isinstance(row, TableRow) + # In streaming, we set batch_format to "default" because calling + # ds.dataset_format() will still invoke bulk execution and we want + # to avoid that. As a result, it's receiving PandasRow (the defaut + # batch format). + if ray.data.context.DatasetContext.get_current().use_streaming_executor: + assert isinstance(row, PandasRow) + else: + assert isinstance(row, ArrowRow) + assert row == t_row + + +def test_iter_batches_basic(ray_start_regular_shared): + df1 = pd.DataFrame({"one": [1, 2, 3], "two": [2, 3, 4]}) + df2 = pd.DataFrame({"one": [4, 5, 6], "two": [5, 6, 7]}) + df3 = pd.DataFrame({"one": [7, 8, 9], "two": [8, 9, 10]}) + df4 = pd.DataFrame({"one": [10, 11, 12], "two": [11, 12, 13]}) + dfs = [df1, df2, df3, df4] + ds = ray.data.from_pandas(dfs) + + # Default. + for batch, df in zip(ds.iter_batches(batch_size=None, batch_format="pandas"), dfs): + assert isinstance(batch, pd.DataFrame) + assert batch.equals(df) + + # pyarrow.Table format. + for batch, df in zip(ds.iter_batches(batch_size=None, batch_format="pyarrow"), dfs): + assert isinstance(batch, pa.Table) + assert batch.equals(pa.Table.from_pandas(df)) + + # NumPy format. + for batch, df in zip(ds.iter_batches(batch_size=None, batch_format="numpy"), dfs): + assert isinstance(batch, dict) + assert list(batch.keys()) == ["one", "two"] + assert all(isinstance(col, np.ndarray) for col in batch.values()) + pd.testing.assert_frame_equal(pd.DataFrame(batch), df) + + # Numpy format (single column). + ds2 = ds.select_columns(["one"]) + for batch, df in zip(ds2.iter_batches(batch_size=None, batch_format="numpy"), dfs): + assert isinstance(batch, dict) + assert list(batch.keys()) == ["one"] + assert all(isinstance(col, np.ndarray) for col in batch.values()) + pd.testing.assert_frame_equal(pd.DataFrame(batch), df[["one"]]) + + # Test NumPy format on Arrow blocks. + ds2 = ds.map_batches(lambda b: b, batch_size=None, batch_format="pyarrow") + for batch, df in zip(ds2.iter_batches(batch_size=None, batch_format="numpy"), dfs): + assert isinstance(batch, dict) + assert list(batch.keys()) == ["one", "two"] + assert all(isinstance(col, np.ndarray) for col in batch.values()) + pd.testing.assert_frame_equal(pd.DataFrame(batch), df) + + # Test NumPy format on Arrow blocks (single column). + ds3 = ds2.select_columns(["one"]) + for batch, df in zip(ds3.iter_batches(batch_size=None, batch_format="numpy"), dfs): + assert isinstance(batch, dict) + assert list(batch.keys()) == ["one"] + assert all(isinstance(col, np.ndarray) for col in batch.values()) + pd.testing.assert_frame_equal(pd.DataFrame(batch), df[["one"]]) + + # Native format (deprecated). + for batch, df in zip(ds.iter_batches(batch_size=None, batch_format="native"), dfs): + assert BlockAccessor.for_block(batch).to_pandas().equals(df) + + # Default format. + for batch, df in zip(ds.iter_batches(batch_size=None, batch_format="default"), dfs): + assert BlockAccessor.for_block(batch).to_pandas().equals(df) + + # Batch size. + batch_size = 2 + batches = list(ds.iter_batches(batch_size=batch_size, batch_format="pandas")) + assert all(len(batch) == batch_size for batch in batches) + assert len(batches) == math.ceil( + (len(df1) + len(df2) + len(df3) + len(df4)) / batch_size + ) + assert pd.concat(batches, ignore_index=True).equals( + pd.concat(dfs, ignore_index=True) + ) + + # Batch size larger than block. + batch_size = 4 + batches = list(ds.iter_batches(batch_size=batch_size, batch_format="pandas")) + assert all(len(batch) == batch_size for batch in batches) + assert len(batches) == math.ceil( + (len(df1) + len(df2) + len(df3) + len(df4)) / batch_size + ) + assert pd.concat(batches, ignore_index=True).equals( + pd.concat(dfs, ignore_index=True) + ) + + # Batch size larger than dataset. + batch_size = 15 + batches = list(ds.iter_batches(batch_size=batch_size, batch_format="pandas")) + assert all(len(batch) == ds.count() for batch in batches) + assert len(batches) == 1 + assert pd.concat(batches, ignore_index=True).equals( + pd.concat(dfs, ignore_index=True) + ) + + # Batch size drop partial. + batch_size = 5 + batches = list( + ds.iter_batches(batch_size=batch_size, drop_last=True, batch_format="pandas") + ) + assert all(len(batch) == batch_size for batch in batches) + assert len(batches) == (len(df1) + len(df2) + len(df3) + len(df4)) // batch_size + assert pd.concat(batches, ignore_index=True).equals( + pd.concat(dfs, ignore_index=True)[:10] + ) + + # Batch size don't drop partial. + batch_size = 5 + batches = list( + ds.iter_batches(batch_size=batch_size, drop_last=False, batch_format="pandas") + ) + assert all(len(batch) == batch_size for batch in batches[:-1]) + assert len(batches[-1]) == (len(df1) + len(df2) + len(df3) + len(df4)) % batch_size + assert len(batches) == math.ceil( + (len(df1) + len(df2) + len(df3) + len(df4)) / batch_size + ) + assert pd.concat(batches, ignore_index=True).equals( + pd.concat(dfs, ignore_index=True) + ) + + # Prefetch. + batches = list( + ds.iter_batches(prefetch_blocks=1, batch_size=None, batch_format="pandas") + ) + assert len(batches) == len(dfs) + for batch, df in zip(batches, dfs): + assert isinstance(batch, pd.DataFrame) + assert batch.equals(df) + + batch_size = 2 + batches = list( + ds.iter_batches(prefetch_blocks=2, batch_size=batch_size, batch_format="pandas") + ) + assert all(len(batch) == batch_size for batch in batches) + assert len(batches) == math.ceil( + (len(df1) + len(df2) + len(df3) + len(df4)) / batch_size + ) + assert pd.concat(batches, ignore_index=True).equals( + pd.concat(dfs, ignore_index=True) + ) + + # Prefetch more than number of blocks. + batches = list( + ds.iter_batches( + prefetch_blocks=len(dfs), batch_size=None, batch_format="pandas" + ) + ) + assert len(batches) == len(dfs) + for batch, df in zip(batches, dfs): + assert isinstance(batch, pd.DataFrame) + assert batch.equals(df) + + # Prefetch with ray.wait. + context = DatasetContext.get_current() + old_config = context.actor_prefetcher_enabled + try: + context.actor_prefetcher_enabled = False + batches = list( + ds.iter_batches(prefetch_blocks=1, batch_size=None, batch_format="pandas") + ) + assert len(batches) == len(dfs) + for batch, df in zip(batches, dfs): + assert isinstance(batch, pd.DataFrame) + assert batch.equals(df) + finally: + context.actor_prefetcher_enabled = old_config + + +def test_iter_batches_empty_block(ray_start_regular_shared): + ds = ray.data.range(1).repartition(10) + assert list(ds.iter_batches(batch_size=None)) == [[0]] + assert list(ds.iter_batches(batch_size=1, local_shuffle_buffer_size=1)) == [[0]] + + +@pytest.mark.parametrize("pipelined", [False, True]) +@pytest.mark.parametrize("ds_format", ["arrow", "pandas", "simple"]) +def test_iter_batches_local_shuffle(shutdown_only, pipelined, ds_format): + # Input validation. + # Batch size must be given for local shuffle. + with pytest.raises(ValueError): + list( + ray.data.range(100).iter_batches( + batch_size=None, local_shuffle_buffer_size=10 + ) + ) + + def range(n, parallelism=200): + if ds_format == "simple": + ds = ray.data.range(n, parallelism=parallelism) + elif ds_format == "arrow": + ds = ray.data.range_table(n, parallelism=parallelism) + elif ds_format == "pandas": + ds = ray.data.range_table(n, parallelism=parallelism).map_batches( + lambda df: df, batch_size=None, batch_format="pandas" + ) + if pipelined: + pipe = ds.repeat(2) + return pipe + else: + return ds + + def to_row_dicts(batch): + if isinstance(batch, pd.DataFrame): + batch = batch.to_dict(orient="records") + return batch + + def unbatch(batches): + return [r for batch in batches for r in to_row_dicts(batch)] + + def sort(r): + if ds_format == "simple": + return sorted(r) + return sorted(r, key=lambda v: v["value"]) + + base = range(100).take_all() + + # Local shuffle. + r1 = unbatch( + range(100, parallelism=10).iter_batches( + batch_size=3, + local_shuffle_buffer_size=25, + ) + ) + r2 = unbatch( + range(100, parallelism=10).iter_batches( + batch_size=3, + local_shuffle_buffer_size=25, + ) + ) + # Check randomness of shuffle. + assert r1 != r2, (r1, r2) + assert r1 != base + assert r2 != base + # Check content. + assert sort(r1) == sort(base) + assert sort(r2) == sort(base) + + # Set seed. + r1 = unbatch( + range(100, parallelism=10).iter_batches( + batch_size=3, + local_shuffle_buffer_size=25, + local_shuffle_seed=0, + ) + ) + r2 = unbatch( + range(100, parallelism=10).iter_batches( + batch_size=3, + local_shuffle_buffer_size=25, + local_shuffle_seed=0, + ) + ) + # Check randomness of shuffle. + assert r1 == r2, (r1, r2) + assert r1 != base + # Check content. + assert sort(r1) == sort(base) + + # Single block. + r1 = unbatch( + range(100, parallelism=1).iter_batches( + batch_size=3, + local_shuffle_buffer_size=25, + ) + ) + r2 = unbatch( + range(100, parallelism=1).iter_batches( + batch_size=3, + local_shuffle_buffer_size=25, + ) + ) + # Check randomness of shuffle. + assert r1 != r2, (r1, r2) + assert r1 != base + assert r2 != base + # Check content. + assert sort(r1) == sort(base) + assert sort(r2) == sort(base) + + # Single-row blocks. + r1 = unbatch( + range(100, parallelism=100).iter_batches( + batch_size=3, + local_shuffle_buffer_size=25, + ) + ) + r2 = unbatch( + range(100, parallelism=100).iter_batches( + batch_size=3, + local_shuffle_buffer_size=25, + ) + ) + # Check randomness of shuffle. + assert r1 != r2, (r1, r2) + assert r1 != base + assert r2 != base + # Check content. + assert sort(r1) == sort(base) + assert sort(r2) == sort(base) + + # Buffer larger than dataset. + r1 = unbatch( + range(100, parallelism=10).iter_batches( + batch_size=3, + local_shuffle_buffer_size=200, + ) + ) + r2 = unbatch( + range(100, parallelism=10).iter_batches( + batch_size=3, + local_shuffle_buffer_size=200, + ) + ) + # Check randomness of shuffle. + assert r1 != r2, (r1, r2) + assert r1 != base + assert r2 != base + # Check content. + assert sort(r1) == sort(base) + assert sort(r2) == sort(base) + + # Batch size larger than block. + r1 = unbatch( + range(100, parallelism=20).iter_batches( + batch_size=12, + local_shuffle_buffer_size=25, + ) + ) + r2 = unbatch( + range(100, parallelism=20).iter_batches( + batch_size=12, + local_shuffle_buffer_size=25, + ) + ) + # Check randomness of shuffle. + assert r1 != r2, (r1, r2) + assert r1 != base + assert r2 != base + # Check content. + assert sort(r1) == sort(base) + assert sort(r2) == sort(base) + + # Batch size larger than dataset. + r1 = unbatch( + range(100, parallelism=10).iter_batches( + batch_size=200, + local_shuffle_buffer_size=400, + ) + ) + r2 = unbatch( + range(100, parallelism=10).iter_batches( + batch_size=200, + local_shuffle_buffer_size=400, + ) + ) + # Check randomness of shuffle. + assert r1 != r2, (r1, r2) + assert r1 != base + assert r2 != base + # Check content. + assert sort(r1) == sort(base) + assert sort(r2) == sort(base) + + # Drop partial batches. + r1 = unbatch( + range(100, parallelism=10).iter_batches( + batch_size=7, + local_shuffle_buffer_size=21, + drop_last=True, + ) + ) + r2 = unbatch( + range(100, parallelism=10).iter_batches( + batch_size=7, + local_shuffle_buffer_size=21, + drop_last=True, + ) + ) + # Check randomness of shuffle. + assert r1 != r2, (r1, r2) + assert r1 != base + assert r2 != base + # Check content. + # Check that partial batches were dropped. + assert len(r1) % 7 == 0 + assert len(r2) % 7 == 0 + tmp_base = base + if ds_format in ("arrow", "pandas"): + r1 = [tuple(r.items()) for r in r1] + r2 = [tuple(r.items()) for r in r2] + tmp_base = [tuple(r.items()) for r in base] + assert set(r1) <= set(tmp_base) + assert set(r2) <= set(tmp_base) + + # Test empty dataset. + ds = ray.data.from_items([]) + r1 = unbatch(ds.iter_batches(batch_size=2, local_shuffle_buffer_size=10)) + assert len(r1) == 0 + assert r1 == ds.take() + + +def test_iter_batches_grid(ray_start_regular_shared): + # Tests slicing, batch combining, and partial batch dropping logic over + # a grid of dataset, batching, and dropping configurations. + # Grid: num_blocks x num_rows_block_1 x ... x num_rows_block_N x + # batch_size x drop_last + seed = int(time.time()) + print(f"Seeding RNG for test_iter_batches_grid with: {seed}") + random.seed(seed) + max_num_blocks = 20 + max_num_rows_per_block = 20 + num_blocks_samples = 3 + block_sizes_samples = 3 + batch_size_samples = 3 + + for num_blocks in np.random.randint(1, max_num_blocks + 1, size=num_blocks_samples): + block_sizes_list = [ + np.random.randint(1, max_num_rows_per_block + 1, size=num_blocks) + for _ in range(block_sizes_samples) + ] + for block_sizes in block_sizes_list: + # Create the dataset with the given block sizes. + dfs = [] + running_size = 0 + for block_size in block_sizes: + dfs.append( + pd.DataFrame( + {"value": list(range(running_size, running_size + block_size))} + ) + ) + running_size += block_size + num_rows = running_size + ds = ray.data.from_pandas(dfs) + for batch_size in np.random.randint( + 1, num_rows + 1, size=batch_size_samples + ): + for drop_last in (False, True): + batches = list( + ds.iter_batches( + batch_size=batch_size, + drop_last=drop_last, + batch_format="pandas", + ) + ) + if num_rows % batch_size == 0 or not drop_last: + # Number of batches should be equal to + # num_rows / batch_size, rounded up. + assert len(batches) == math.ceil(num_rows / batch_size) + # Concatenated batches should equal the DataFrame + # representation of the entire dataset. + assert pd.concat(batches, ignore_index=True).equals( + ds.to_pandas() + ) + else: + # Number of batches should be equal to + # num_rows / batch_size, rounded down. + assert len(batches) == num_rows // batch_size + # Concatenated batches should equal the DataFrame + # representation of the dataset with the partial batch + # remainder sliced off. + assert pd.concat(batches, ignore_index=True).equals( + ds.to_pandas()[: batch_size * (num_rows // batch_size)] + ) + if num_rows % batch_size == 0 or drop_last: + assert all(len(batch) == batch_size for batch in batches) + else: + assert all(len(batch) == batch_size for batch in batches[:-1]) + assert len(batches[-1]) == num_rows % batch_size + + +def test_lazy_loading_iter_batches_exponential_rampup(ray_start_regular_shared): + ds = ray.data.range(32, parallelism=8) + expected_num_blocks = [1, 2, 4, 4, 8, 8, 8, 8] + for _, expected in zip(ds.iter_batches(batch_size=None), expected_num_blocks): + if ray.data.context.DatasetContext.get_current().use_streaming_executor: + # In streaming execution of ds.iter_batches(), there is no partial + # execution so _num_computed() in LazyBlocklist is 0. + assert ds._plan.execute()._num_computed() == 0 + else: + assert ds._plan.execute()._num_computed() == expected + + +def test_union(ray_start_regular_shared): + ds = ray.data.range(20, parallelism=10) + + # Test lazy union. + ds = ds.union(ds, ds, ds, ds) + assert ds.num_blocks() == 50 + assert ds.count() == 100 + assert ds.sum() == 950 + + ds = ds.union(ds) + assert ds.count() == 200 + assert ds.sum() == (950 * 2) + + # Test materialized union. + ds2 = ray.data.from_items([1, 2, 3, 4, 5]) + assert ds2.count() == 5 + ds2 = ds2.union(ds2) + assert ds2.count() == 10 + ds2 = ds2.union(ds) + assert ds2.count() == 210 + + +@pytest.mark.parametrize("pipelined", [False, True]) +def test_iter_tf_batches(ray_start_regular_shared, pipelined): + df1 = pd.DataFrame( + {"one": [1, 2, 3], "two": [1.0, 2.0, 3.0], "label": [1.0, 2.0, 3.0]} + ) + df2 = pd.DataFrame( + {"one": [4, 5, 6], "two": [4.0, 5.0, 6.0], "label": [4.0, 5.0, 6.0]} + ) + df3 = pd.DataFrame({"one": [7, 8], "two": [7.0, 8.0], "label": [7.0, 8.0]}) + df = pd.concat([df1, df2, df3]) + ds = ray.data.from_pandas([df1, df2, df3]) + ds = maybe_pipeline(ds, pipelined) + + num_epochs = 1 if pipelined else 2 + for _ in range(num_epochs): + iterations = [] + for batch in ds.iter_tf_batches(batch_size=3): + iterations.append( + np.stack((batch["one"], batch["two"], batch["label"]), axis=1) + ) + combined_iterations = np.concatenate(iterations) + np.testing.assert_array_equal(np.sort(df.values), np.sort(combined_iterations)) + + +@pytest.mark.parametrize("pipelined", [False, True]) +def test_iter_tf_batches_tensor_ds(ray_start_regular_shared, pipelined): + arr1 = np.arange(12).reshape((3, 2, 2)) + arr2 = np.arange(12, 24).reshape((3, 2, 2)) + arr = np.concatenate((arr1, arr2)) + ds = ray.data.from_numpy([arr1, arr2]) + ds = maybe_pipeline(ds, pipelined) + + num_epochs = 1 if pipelined else 2 + for _ in range(num_epochs): + iterations = [] + for batch in ds.iter_tf_batches(batch_size=2): + iterations.append(batch) + combined_iterations = np.concatenate(iterations) + np.testing.assert_array_equal(arr, combined_iterations) + + +def test_block_builder_for_block(ray_start_regular_shared): + # list + builder = BlockBuilder.for_block(list()) + builder.add_block([1, 2]) + assert builder.build() == [1, 2] + builder.add_block([3, 4]) + assert builder.build() == [1, 2, 3, 4] + + # pandas dataframe + builder = BlockBuilder.for_block(pd.DataFrame()) + b1 = pd.DataFrame({"A": [1], "B": ["a"]}) + builder.add_block(b1) + assert builder.build().equals(b1) + b2 = pd.DataFrame({"A": [2, 3], "B": ["c", "d"]}) + builder.add_block(b2) + expected = pd.DataFrame({"A": [1, 2, 3], "B": ["a", "c", "d"]}) + assert builder.build().equals(expected) + + # pyarrow table + builder = BlockBuilder.for_block(pa.Table.from_arrays(list())) + b1 = pa.Table.from_pydict({"A": [1], "B": ["a"]}) + builder.add_block(b1) + builder.build().equals(b1) + b2 = pa.Table.from_pydict({"A": [2, 3], "B": ["c", "d"]}) + builder.add_block(b2) + expected = pa.Table.from_pydict({"A": [1, 2, 3], "B": ["a", "c", "d"]}) + builder.build().equals(expected) + + # wrong type + with pytest.raises(TypeError): + BlockBuilder.for_block(str()) + + +@pytest.mark.parametrize("num_parts", [1, 30]) +@pytest.mark.parametrize("ds_format", ["arrow", "pandas"]) +def test_global_tabular_min(ray_start_regular_shared, ds_format, num_parts): + seed = int(time.time()) + print(f"Seeding RNG for test_global_arrow_min with: {seed}") + random.seed(seed) + xs = list(range(100)) + random.shuffle(xs) + + def _to_pandas(ds): + return ds.map_batches(lambda x: x, batch_size=None, batch_format="pandas") + + # Test built-in global min aggregation + ds = ray.data.from_items([{"A": x} for x in xs]).repartition(num_parts) + if ds_format == "pandas": + ds = _to_pandas(ds) + assert ds.min("A") == 0 + + # Test empty dataset + ds = ray.data.range_table(10) + if ds_format == "pandas": + ds = _to_pandas(ds) + assert ds.filter(lambda r: r["value"] > 10).min("value") is None + + # Test built-in global min aggregation with nans + nan_ds = ray.data.from_items([{"A": x} for x in xs] + [{"A": None}]).repartition( + num_parts + ) + if ds_format == "pandas": + nan_ds = _to_pandas(nan_ds) + assert nan_ds.min("A") == 0 + # Test ignore_nulls=False + assert nan_ds.min("A", ignore_nulls=False) is None + # Test all nans + nan_ds = ray.data.from_items([{"A": None}] * len(xs)).repartition(num_parts) + if ds_format == "pandas": + nan_ds = _to_pandas(nan_ds) + assert nan_ds.min("A") is None + assert nan_ds.min("A", ignore_nulls=False) is None + + +@pytest.mark.parametrize("num_parts", [1, 30]) +@pytest.mark.parametrize("ds_format", ["arrow", "pandas"]) +def test_global_tabular_max(ray_start_regular_shared, ds_format, num_parts): + seed = int(time.time()) + print(f"Seeding RNG for test_global_arrow_max with: {seed}") + random.seed(seed) + xs = list(range(100)) + random.shuffle(xs) + + def _to_pandas(ds): + return ds.map_batches(lambda x: x, batch_size=None, batch_format="pandas") + + # Test built-in global max aggregation + ds = ray.data.from_items([{"A": x} for x in xs]).repartition(num_parts) + if ds_format == "pandas": + ds = _to_pandas(ds) + assert ds.max("A") == 99 + + # Test empty dataset + ds = ray.data.range_table(10) + if ds_format == "pandas": + ds = _to_pandas(ds) + assert ds.filter(lambda r: r["value"] > 10).max("value") is None + + # Test built-in global max aggregation with nans + nan_ds = ray.data.from_items([{"A": x} for x in xs] + [{"A": None}]).repartition( + num_parts + ) + if ds_format == "pandas": + nan_ds = _to_pandas(nan_ds) + assert nan_ds.max("A") == 99 + # Test ignore_nulls=False + assert nan_ds.max("A", ignore_nulls=False) is None + # Test all nans + nan_ds = ray.data.from_items([{"A": None}] * len(xs)).repartition(num_parts) + if ds_format == "pandas": + nan_ds = _to_pandas(nan_ds) + assert nan_ds.max("A") is None + assert nan_ds.max("A", ignore_nulls=False) is None + + +@pytest.mark.parametrize("num_parts", [1, 30]) +@pytest.mark.parametrize("ds_format", ["arrow", "pandas"]) +def test_global_tabular_mean(ray_start_regular_shared, ds_format, num_parts): + seed = int(time.time()) + print(f"Seeding RNG for test_global_arrow_mean with: {seed}") + random.seed(seed) + xs = list(range(100)) + random.shuffle(xs) + + def _to_pandas(ds): + return ds.map_batches(lambda x: x, batch_size=None, batch_format="pandas") + + # Test built-in global mean aggregation + ds = ray.data.from_items([{"A": x} for x in xs]).repartition(num_parts) + if ds_format == "pandas": + ds = _to_pandas(ds) + assert ds.mean("A") == 49.5 + + # Test empty dataset + ds = ray.data.range_table(10) + if ds_format == "pandas": + ds = _to_pandas(ds) + assert ds.filter(lambda r: r["value"] > 10).mean("value") is None + + # Test built-in global mean aggregation with nans + nan_ds = ray.data.from_items([{"A": x} for x in xs] + [{"A": None}]).repartition( + num_parts + ) + if ds_format == "pandas": + nan_ds = _to_pandas(nan_ds) + assert nan_ds.mean("A") == 49.5 + # Test ignore_nulls=False + assert nan_ds.mean("A", ignore_nulls=False) is None + # Test all nans + nan_ds = ray.data.from_items([{"A": None}] * len(xs)).repartition(num_parts) + if ds_format == "pandas": + nan_ds = _to_pandas(nan_ds) + assert nan_ds.mean("A") is None + assert nan_ds.mean("A", ignore_nulls=False) is None + + +@pytest.mark.parametrize("num_parts", [1, 30]) +@pytest.mark.parametrize("ds_format", ["arrow", "pandas"]) +def test_global_tabular_std(ray_start_regular_shared, ds_format, num_parts): + seed = int(time.time()) + print(f"Seeding RNG for test_global_arrow_std with: {seed}") + random.seed(seed) + xs = list(range(100)) + random.shuffle(xs) + + def _to_arrow(ds): + return ds.map_batches(lambda x: x, batch_size=None, batch_format="pyarrow") + + def _to_pandas(ds): + return ds.map_batches(lambda x: x, batch_size=None, batch_format="pandas") + + # Test built-in global max aggregation + df = pd.DataFrame({"A": xs}) + ds = ray.data.from_pandas(df).repartition(num_parts) + if ds_format == "arrow": + ds = _to_arrow(ds) + assert math.isclose(ds.std("A"), df["A"].std()) + assert math.isclose(ds.std("A", ddof=0), df["A"].std(ddof=0)) + + # Test empty dataset + ds = ray.data.from_pandas(pd.DataFrame({"A": []})) + if ds_format == "arrow": + ds = _to_arrow(ds) + assert ds.std("A") is None + # Test edge cases + ds = ray.data.from_pandas(pd.DataFrame({"A": [3]})) + if ds_format == "arrow": + ds = _to_arrow(ds) + assert ds.std("A") == 0 + + # Test built-in global std aggregation with nans + nan_df = pd.DataFrame({"A": xs + [None]}) + nan_ds = ray.data.from_pandas(nan_df).repartition(num_parts) + if ds_format == "arrow": + nan_ds = _to_arrow(nan_ds) + assert math.isclose(nan_ds.std("A"), nan_df["A"].std()) + # Test ignore_nulls=False + assert nan_ds.std("A", ignore_nulls=False) is None + # Test all nans + nan_ds = ray.data.from_items([{"A": None}] * len(xs)).repartition(num_parts) + if ds_format == "pandas": + nan_ds = _to_pandas(nan_ds) + assert nan_ds.std("A") is None + assert nan_ds.std("A", ignore_nulls=False) is None + + +def test_column_name_type_check(ray_start_regular_shared): + df = pd.DataFrame({"1": np.random.rand(10), "a": np.random.rand(10)}) + ds = ray.data.from_pandas(df) + expected_str = "Dataset(num_blocks=1, num_rows=10, schema={1: float64, a: float64})" + assert str(ds) == expected_str, str(ds) + df = pd.DataFrame({1: np.random.rand(10), "a": np.random.rand(10)}) + with pytest.raises(ValueError): + ray.data.from_pandas(df) + + +def test_len(ray_start_regular_shared): + ds = ray.data.range(1) + with pytest.raises(AttributeError): + len(ds) + + +def test_simple_block_select(): + xs = list(range(100)) + block_accessor = BlockAccessor.for_block(xs) + + block = block_accessor.select([lambda x: x % 3]) + assert block == [x % 3 for x in xs] + + with pytest.raises(ValueError): + block = block_accessor.select(["foo"]) + + with pytest.raises(ValueError): + block = block_accessor.select([]) + + +def test_pandas_block_select(): + df = pd.DataFrame({"one": [10, 11, 12], "two": [11, 12, 13], "three": [14, 15, 16]}) + block_accessor = BlockAccessor.for_block(df) + + block = block_accessor.select(["two"]) + assert block.equals(df[["two"]]) + + block = block_accessor.select(["two", "one"]) + assert block.equals(df[["two", "one"]]) + + with pytest.raises(ValueError): + block = block_accessor.select([lambda x: x % 3, "two"]) + + +# NOTE: All tests above share a Ray cluster, while the tests below do not. These +# tests should only be carefully reordered to retain this invariant! + + +def test_unsupported_pyarrow_versions_check(shutdown_only, unsupported_pyarrow_version): + ray.shutdown() + + # Test that unsupported pyarrow versions cause an error to be raised upon the + # initial pyarrow use. + ray.init(runtime_env={"pip": [f"pyarrow=={unsupported_pyarrow_version}"]}) + + # Test Arrow-native creation APIs. + # Test range_table. + with pytest.raises(ImportError): + ray.data.range_table(10).take_all() + + # Test from_arrow. + with pytest.raises(ImportError): + ray.data.from_arrow(pa.table({"a": [1, 2]})) + + # Test read_parquet. + with pytest.raises(ImportError): + ray.data.read_parquet("example://iris.parquet").take_all() + + # Test from_numpy (we use Arrow for representing the tensors). + with pytest.raises(ImportError): + ray.data.from_numpy(np.arange(12).reshape((3, 2, 2))) + + +def test_unsupported_pyarrow_versions_check_disabled( + shutdown_only, + unsupported_pyarrow_version, + disable_pyarrow_version_check, +): + # Test that unsupported pyarrow versions DO NOT cause an error to be raised upon the + # initial pyarrow use when the version check is disabled. + ray.init( + runtime_env={ + "pip": [f"pyarrow=={unsupported_pyarrow_version}"], + "env_vars": {"RAY_DISABLE_PYARROW_VERSION_CHECK": "1"}, + }, + ) + + # Test Arrow-native creation APIs. + # Test range_table. + try: + ray.data.range_table(10).take_all() + except ImportError as e: + pytest.fail(f"_check_pyarrow_version failed unexpectedly: {e}") + + # Test from_arrow. + try: + ray.data.from_arrow(pa.table({"a": [1, 2]})) + except ImportError as e: + pytest.fail(f"_check_pyarrow_version failed unexpectedly: {e}") + + # Test read_parquet. + try: + ray.data.read_parquet("example://iris.parquet").take_all() + except ImportError as e: + pytest.fail(f"_check_pyarrow_version failed unexpectedly: {e}") + + # Test from_numpy (we use Arrow for representing the tensors). + try: + ray.data.from_numpy(np.arange(12).reshape((3, 2, 2))) + except ImportError as e: + pytest.fail(f"_check_pyarrow_version failed unexpectedly: {e}") + + +def test_read_write_local_node_ray_client(ray_start_cluster_enabled): + cluster = ray_start_cluster_enabled + cluster.add_node(num_cpus=4) + cluster.head_node._ray_params.ray_client_server_port = "10004" + cluster.head_node.start_ray_client_server() + address = "ray://localhost:10004" + + import tempfile + + data_path = tempfile.mkdtemp() + df = pd.DataFrame({"one": list(range(0, 10)), "two": list(range(10, 20))}) + path = os.path.join(data_path, "test.parquet") + df.to_parquet(path) + + # Read/write from Ray Client will result in error. + ray.init(address) + with pytest.raises(ValueError): + ds = ray.data.read_parquet("local://" + path).fully_executed() + ds = ray.data.from_pandas(df) + with pytest.raises(ValueError): + ds.write_parquet("local://" + data_path).fully_executed() + + +def test_read_warning_large_parallelism(ray_start_regular, propagate_logs, caplog): + with caplog.at_level(logging.WARNING, logger="ray.data.read_api"): + ray.data.range(5000, parallelism=5000).fully_executed() + assert ( + "The requested parallelism of 5000 is " + "more than 4x the number of available CPU slots in the cluster" in caplog.text + ), caplog.text + + +def test_read_write_local_node(ray_start_cluster): + cluster = ray_start_cluster + cluster.add_node( + resources={"bar:1": 100}, + num_cpus=10, + _system_config={"max_direct_call_object_size": 0}, + ) + cluster.add_node(resources={"bar:2": 100}, num_cpus=10) + cluster.add_node(resources={"bar:3": 100}, num_cpus=10) + + ray.init(cluster.address) + + import os + import tempfile + + data_path = tempfile.mkdtemp() + num_files = 5 + for idx in range(num_files): + df = pd.DataFrame( + {"one": list(range(idx, idx + 10)), "two": list(range(idx + 10, idx + 20))} + ) + path = os.path.join(data_path, f"test{idx}.parquet") + df.to_parquet(path) + + ctx = ray.data.context.DatasetContext.get_current() + ctx.read_write_local_node = True + + def check_dataset_is_local(ds): + blocks = ds.get_internal_block_refs() + assert len(blocks) == num_files + ray.wait(blocks, num_returns=len(blocks), fetch_local=False) + location_data = ray.experimental.get_object_locations(blocks) + locations = [] + for block in blocks: + locations.extend(location_data[block]["node_ids"]) + assert set(locations) == {ray.get_runtime_context().get_node_id()} + + local_path = "local://" + data_path + # Plain read. + ds = ray.data.read_parquet(local_path).fully_executed() + check_dataset_is_local(ds) + + # SPREAD scheduling got overridden when read local scheme. + ds = ray.data.read_parquet( + local_path, ray_remote_args={"scheduling_strategy": "SPREAD"} + ).fully_executed() + check_dataset_is_local(ds) + + # With fusion. + ds = ray.data.read_parquet(local_path).map(lambda x: x).fully_executed() + check_dataset_is_local(ds) + + # Write back to local scheme. + output = os.path.join(local_path, "test_read_write_local_node") + ds.write_parquet(output) + assert "1 nodes used" in ds.stats(), ds.stats() + ray.data.read_parquet(output).take_all() == ds.take_all() + + # Mixing paths of local and non-local scheme is invalid. + with pytest.raises(ValueError): + ds = ray.data.read_parquet( + [local_path + "/test1.parquet", data_path + "/test2.parquet"] + ).fully_executed() + with pytest.raises(ValueError): + ds = ray.data.read_parquet( + [local_path + "/test1.parquet", "example://iris.parquet"] + ).fully_executed() + with pytest.raises(ValueError): + ds = ray.data.read_parquet( + ["example://iris.parquet", local_path + "/test1.parquet"] + ).fully_executed() + + +@ray.remote +class Counter: + def __init__(self): + self.value = 0 + + def increment(self): + self.value += 1 + return self.value + + +class FlakyCSVDatasource(CSVDatasource): + def __init__(self): + self.counter = Counter.remote() + + def _read_stream(self, f: "pa.NativeFile", path: str, **reader_args): + count = self.counter.increment.remote() + if ray.get(count) == 1: + raise ValueError("oops") + else: + for block in CSVDatasource._read_stream(self, f, path, **reader_args): + yield block + + def _write_block(self, f: "pa.NativeFile", block: BlockAccessor, **writer_args): + count = self.counter.increment.remote() + if ray.get(count) == 1: + raise ValueError("oops") + else: + CSVDatasource._write_block(self, f, block, **writer_args) + + +def test_dataset_retry_exceptions(ray_start_regular, local_path): + df1 = pd.DataFrame({"one": [1, 2, 3], "two": ["a", "b", "c"]}) + path1 = os.path.join(local_path, "test1.csv") + df1.to_csv(path1, index=False, storage_options={}) + ds1 = ray.data.read_datasource(FlakyCSVDatasource(), parallelism=1, paths=path1) + ds1.write_datasource(FlakyCSVDatasource(), path=local_path, dataset_uuid="data") + assert df1.equals( + pd.read_csv(os.path.join(local_path, "data_000000.csv"), storage_options={}) + ) + + counter = Counter.remote() + + def flaky_mapper(x): + count = counter.increment.remote() + if ray.get(count) == 1: + raise ValueError("oops") + else: + return ray.get(count) + + assert sorted(ds1.map(flaky_mapper).take()) == [2, 3, 4] + + with pytest.raises(ValueError): + ray.data.read_datasource( + FlakyCSVDatasource(), + parallelism=1, + paths=path1, + ray_remote_args={"retry_exceptions": False}, + ).take() + + +def test_datasource(ray_start_regular): + source = ray.data.datasource.RandomIntRowDatasource() + assert len(ray.data.read_datasource(source, n=10, num_columns=2).take()) == 10 + source = ray.data.datasource.RangeDatasource() + assert ray.data.read_datasource(source, n=10).take() == list(range(10)) + + +def test_polars_lazy_import(shutdown_only): + import sys + + ctx = ray.data.context.DatasetContext.get_current() + + try: + original_use_polars = ctx.use_polars + ctx.use_polars = True + + num_items = 100 + parallelism = 4 + ray.init(num_cpus=4) + + @ray.remote + def f(should_import_polars): + # Sleep to spread the tasks. + time.sleep(1) + polars_imported = "polars" in sys.modules.keys() + return polars_imported == should_import_polars + + # We should not use polars for non-Arrow sort. + _ = ray.data.range(num_items, parallelism=parallelism).sort() + assert all(ray.get([f.remote(False) for _ in range(parallelism)])) + + a = range(100) + dfs = [] + partition_size = num_items // parallelism + for i in range(parallelism): + dfs.append( + pd.DataFrame({"a": a[i * partition_size : (i + 1) * partition_size]}) + ) + # At least one worker should have imported polars. + _ = ( + ray.data.from_pandas(dfs) + .map_batches(lambda t: t, batch_format="pyarrow", batch_size=None) + .sort(key="a") + .fully_executed() + ) + assert any(ray.get([f.remote(True) for _ in range(parallelism)])) + + finally: + ctx.use_polars = original_use_polars + + +def test_default_batch_format(shutdown_only): + ds = ray.data.range(100) + assert ds.default_batch_format() == list + + ds = ray.data.range_tensor(100) + assert ds.default_batch_format() == np.ndarray + + df = pd.DataFrame({"foo": ["a", "b"], "bar": [0, 1]}) + ds = ray.data.from_pandas(df) + assert ds.default_batch_format() == pd.DataFrame + + +def test_dataset_schema_after_read_stats(ray_start_cluster): + cluster = ray_start_cluster + cluster.add_node(num_cpus=1) + ray.init(cluster.address) + cluster.add_node(num_cpus=1, resources={"foo": 1}) + ds = ray.data.read_csv( + "example://iris.csv", ray_remote_args={"resources": {"foo": 1}} + ) + schema = ds.schema() + ds.stats() + assert schema == ds.schema() + + +def test_dataset_plan_as_string(ray_start_cluster): + ds = ray.data.read_parquet("example://iris.parquet") + assert ds._plan.get_plan_as_string() == ( + "Dataset(\n" + " num_blocks=1,\n" + " num_rows=150,\n" + " schema={\n" + " sepal.length: double,\n" + " sepal.width: double,\n" + " petal.length: double,\n" + " petal.width: double,\n" + " variety: string\n" + " }\n" + ")" + ) + for _ in range(5): + ds = ds.map_batches(lambda x: x) + assert ds._plan.get_plan_as_string() == ( + "MapBatches()\n" + "+- MapBatches()\n" + " +- MapBatches()\n" + " +- MapBatches()\n" + " +- MapBatches()\n" + " +- Dataset(\n" + " num_blocks=1,\n" + " num_rows=150,\n" + " schema={\n" + " sepal.length: double,\n" + " sepal.width: double,\n" + " petal.length: double,\n" + " petal.width: double,\n" + " variety: string\n" + " }\n" + " )" + ) + + +class LoggerWarningCalled(Exception): + """Custom exception used in test_warning_execute_with_no_cpu() and + test_nowarning_execute_with_cpu(). Raised when the `logger.warning` method + is called, so that we can kick out of `plan.execute()` by catching this Exception + and check logging was done properly.""" + + pass + + +def test_warning_execute_with_no_cpu(ray_start_cluster): + """Tests ExecutionPlan.execute() to ensure a warning is logged + when no CPU resources are available.""" + # Create one node with no CPUs to trigger the Dataset warning + ray.init(ray_start_cluster.address) + cluster = ray_start_cluster + cluster.add_node(num_cpus=0) + + logger = DatasetLogger("ray.data._internal.plan").get_logger() + with patch.object( + logger, + "warning", + side_effect=LoggerWarningCalled, + ) as mock_logger: + try: + ds = ray.data.range(10) + ds = ds.map_batches(lambda x: x) + ds.take() + except Exception as e: + if ray.data.context.DatasetContext.get_current().use_streaming_executor: + assert isinstance(e, ValueError) + assert "exceeds the execution limits ExecutionResources(cpu=0.0" in str( + e + ) + else: + assert isinstance(e, LoggerWarningCalled) + logger_args, logger_kwargs = mock_logger.call_args + assert ( + "Warning: The Ray cluster currently does not have " + in logger_args[0] + ) + + +def test_nowarning_execute_with_cpu(ray_start_cluster): + """Tests ExecutionPlan.execute() to ensure no warning is logged + when there are available CPU resources.""" + # Create one node with CPUs to avoid triggering the Dataset warning + ray.init(ray_start_cluster.address) + + logger = DatasetLogger("ray.data._internal.plan").get_logger() + with patch.object( + logger, + "warning", + side_effect=LoggerWarningCalled, + ) as mock_logger: + ds = ray.data.range(10) + ds = ds.map_batches(lambda x: x) + ds.take() + mock_logger.assert_not_called() + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main(["-v", __file__])) diff --git a/python/ray/data/tests/test_dataset_csv.py b/python/ray/data/tests/test_dataset_csv.py index 3a2362d4f15f2..4ca6fa23bbf1e 100644 --- a/python/ray/data/tests/test_dataset_csv.py +++ b/python/ray/data/tests/test_dataset_csv.py @@ -1,3 +1,4 @@ +import itertools import os import shutil from functools import partial @@ -24,6 +25,7 @@ PathPartitionFilter, ) from ray.data.datasource.file_based_datasource import ( + FILE_SIZE_FETCH_PARALLELIZATION_THRESHOLD, FileExtensionFilter, _unwrap_protocol, ) @@ -123,7 +125,7 @@ def test_csv_read(ray_start_regular_shared, fs, data_path, endpoint_url): ds = ray.data.read_csv(path, filesystem=fs, partitioning=None) df = pd.concat([df1, df2], ignore_index=True) dsdf = ds.to_pandas() - assert df.equals(dsdf) + pd.testing.assert_frame_equal(df, dsdf) if fs is None: shutil.rmtree(path) else: @@ -259,6 +261,157 @@ def test_csv_read_meta_provider( ) +@pytest.mark.parametrize( + "fs,data_path,endpoint_url", + [ + (None, lazy_fixture("local_path"), None), + (lazy_fixture("local_fs"), lazy_fixture("local_path"), None), + (lazy_fixture("s3_fs"), lazy_fixture("s3_path"), lazy_fixture("s3_server")), + ], +) +def test_csv_read_many_files_basic( + ray_start_regular_shared, + fs, + data_path, + endpoint_url, +): + if endpoint_url is None: + storage_options = {} + else: + storage_options = dict(client_kwargs=dict(endpoint_url=endpoint_url)) + + paths = [] + dfs = [] + num_dfs = 4 * FILE_SIZE_FETCH_PARALLELIZATION_THRESHOLD + for i in range(num_dfs): + df = pd.DataFrame({"one": list(range(i * 3, (i + 1) * 3))}) + dfs.append(df) + path = os.path.join(data_path, f"test_{i}.csv") + paths.append(path) + df.to_csv(path, index=False, storage_options=storage_options) + ds = ray.data.read_csv(paths, filesystem=fs) + + dsdf = ds.to_pandas() + df = pd.concat(dfs).reset_index(drop=True) + pd.testing.assert_frame_equal(df, dsdf) + + +@pytest.mark.parametrize( + "fs,data_path,endpoint_url", + [ + (None, lazy_fixture("local_path"), None), + (lazy_fixture("local_fs"), lazy_fixture("local_path"), None), + (lazy_fixture("s3_fs"), lazy_fixture("s3_path"), lazy_fixture("s3_server")), + ], +) +def test_csv_read_many_files_partitioned( + ray_start_regular_shared, + fs, + data_path, + endpoint_url, + write_partitioned_df, + assert_base_partitioned_ds, +): + if endpoint_url is None: + storage_options = {} + else: + storage_options = dict(client_kwargs=dict(endpoint_url=endpoint_url)) + + partition_keys = ["one"] + partition_path_encoder = PathPartitionEncoder.of( + base_dir=data_path, + field_names=partition_keys, + filesystem=fs, + ) + paths = [] + dfs = [] + num_dfs = FILE_SIZE_FETCH_PARALLELIZATION_THRESHOLD + num_rows = 6 * num_dfs + num_files = 2 * num_dfs + for i in range(num_dfs): + df = pd.DataFrame( + {"one": [1, 1, 1, 3, 3, 3], "two": list(range(6 * i, 6 * (i + 1)))} + ) + df_paths = write_partitioned_df( + df, + partition_keys, + partition_path_encoder, + partial(df_to_csv, storage_options=storage_options, index=False), + file_name_suffix=i, + ) + dfs.append(df) + paths.extend(df_paths) + + ds = ray.data.read_csv( + paths, + filesystem=fs, + partitioning=partition_path_encoder.scheme, + parallelism=num_files, + ) + + assert_base_partitioned_ds( + ds, + count=num_rows, + num_input_files=num_files, + num_rows=num_rows, + schema="{one: int64, two: int64}", + num_computed=num_files, + sorted_values=sorted( + itertools.chain.from_iterable( + list( + map(list, zip([1, 1, 1, 3, 3, 3], list(range(6 * i, 6 * (i + 1))))) + ) + for i in range(num_dfs) + ) + ), + ) + + +@pytest.mark.parametrize( + "fs,data_path,endpoint_url", + [ + (None, lazy_fixture("local_path"), None), + (lazy_fixture("local_fs"), lazy_fixture("local_path"), None), + (lazy_fixture("s3_fs"), lazy_fixture("s3_path"), lazy_fixture("s3_server")), + ], +) +def test_csv_read_many_files_diff_dirs( + ray_start_regular_shared, + fs, + data_path, + endpoint_url, +): + if endpoint_url is None: + storage_options = {} + else: + storage_options = dict(client_kwargs=dict(endpoint_url=endpoint_url)) + + dir1 = os.path.join(data_path, "dir1") + dir2 = os.path.join(data_path, "dir2") + if fs is None: + os.mkdir(dir1) + os.mkdir(dir2) + else: + fs.create_dir(_unwrap_protocol(dir1)) + fs.create_dir(_unwrap_protocol(dir2)) + + paths = [] + dfs = [] + num_dfs = 2 * FILE_SIZE_FETCH_PARALLELIZATION_THRESHOLD + for i, dir_path in enumerate([dir1, dir2]): + for j in range(num_dfs * i, num_dfs * (i + 1)): + df = pd.DataFrame({"one": list(range(3 * j, 3 * (j + 1)))}) + dfs.append(df) + path = os.path.join(dir_path, f"test_{j}.csv") + paths.append(path) + df.to_csv(path, index=False, storage_options=storage_options) + ds = ray.data.read_csv(paths, filesystem=fs) + + dsdf = ds.to_pandas() + df = pd.concat(dfs).reset_index(drop=True) + pd.testing.assert_frame_equal(df, dsdf) + + @pytest.mark.parametrize( "fs,data_path,endpoint_url", [ diff --git a/python/ray/data/tests/test_dataset_ecosystem.py b/python/ray/data/tests/test_dataset_ecosystem.py new file mode 100644 index 0000000000000..e3c19d130f0f6 --- /dev/null +++ b/python/ray/data/tests/test_dataset_ecosystem.py @@ -0,0 +1,145 @@ +import numpy as np +import pandas as pd +import pyarrow as pa +import pytest + +import ray +from ray.data.extensions.tensor_extension import ( + ArrowTensorArray, + ArrowTensorType, + TensorArray, + TensorDtype, +) +from ray.data.tests.conftest import * # noqa +from ray.tests.conftest import * # noqa + + +def test_from_dask(ray_start_regular_shared): + import dask.dataframe as dd + + df = pd.DataFrame({"one": list(range(100)), "two": list(range(100))}) + ddf = dd.from_pandas(df, npartitions=10) + ds = ray.data.from_dask(ddf) + dfds = ds.to_pandas() + assert df.equals(dfds) + + +@pytest.mark.parametrize("ds_format", ["pandas", "arrow"]) +def test_to_dask(ray_start_regular_shared, ds_format): + from ray.util.dask import ray_dask_get + + df1 = pd.DataFrame({"one": [1, 2, 3], "two": ["a", "b", "c"]}) + df2 = pd.DataFrame({"one": [4, 5, 6], "two": ["e", "f", "g"]}) + df = pd.concat([df1, df2]) + ds = ray.data.from_pandas([df1, df2]) + if ds_format == "arrow": + ds = ds.map_batches(lambda df: df, batch_format="pyarrow", batch_size=None) + ddf = ds.to_dask() + meta = ddf._meta + # Check metadata. + assert isinstance(meta, pd.DataFrame) + assert meta.empty + assert list(meta.columns) == ["one", "two"] + assert list(meta.dtypes) == [np.int64, object] + # Explicit Dask-on-Ray + assert df.equals(ddf.compute(scheduler=ray_dask_get)) + # Implicit Dask-on-Ray. + assert df.equals(ddf.compute()) + + # Explicit metadata. + df1["two"] = df1["two"].astype(pd.StringDtype()) + df2["two"] = df2["two"].astype(pd.StringDtype()) + df = pd.concat([df1, df2]) + ds = ray.data.from_pandas([df1, df2]) + if ds_format == "arrow": + ds = ds.map_batches(lambda df: df, batch_format="pyarrow", batch_size=None) + ddf = ds.to_dask( + meta=pd.DataFrame( + {"one": pd.Series(dtype=np.int16), "two": pd.Series(dtype=pd.StringDtype())} + ), + ) + meta = ddf._meta + # Check metadata. + assert isinstance(meta, pd.DataFrame) + assert meta.empty + assert list(meta.columns) == ["one", "two"] + assert list(meta.dtypes) == [np.int16, pd.StringDtype()] + # Explicit Dask-on-Ray + assert df.equals(ddf.compute(scheduler=ray_dask_get)) + # Implicit Dask-on-Ray. + assert df.equals(ddf.compute()) + + +def test_to_dask_tensor_column_cast_pandas(ray_start_regular_shared): + # Check that tensor column casting occurs when converting a Dataset to a Dask + # DataFrame. + data = np.arange(12).reshape((3, 2, 2)) + ctx = ray.data.context.DatasetContext.get_current() + original = ctx.enable_tensor_extension_casting + try: + ctx.enable_tensor_extension_casting = True + in_df = pd.DataFrame({"a": TensorArray(data)}) + ds = ray.data.from_pandas(in_df) + dtypes = ds.schema().types + assert len(dtypes) == 1 + assert isinstance(dtypes[0], TensorDtype) + out_df = ds.to_dask().compute() + assert out_df["a"].dtype.type is np.object_ + expected_df = pd.DataFrame({"a": list(data)}) + pd.testing.assert_frame_equal(out_df, expected_df) + finally: + ctx.enable_tensor_extension_casting = original + + +def test_to_dask_tensor_column_cast_arrow(ray_start_regular_shared): + # Check that tensor column casting occurs when converting a Dataset to a Dask + # DataFrame. + data = np.arange(12).reshape((3, 2, 2)) + ctx = ray.data.context.DatasetContext.get_current() + original = ctx.enable_tensor_extension_casting + try: + ctx.enable_tensor_extension_casting = True + in_table = pa.table({"a": ArrowTensorArray.from_numpy(data)}) + ds = ray.data.from_arrow(in_table) + dtype = ds.schema().field(0).type + assert isinstance(dtype, ArrowTensorType) + out_df = ds.to_dask().compute() + assert out_df["a"].dtype.type is np.object_ + expected_df = pd.DataFrame({"a": list(data)}) + pd.testing.assert_frame_equal(out_df, expected_df) + finally: + ctx.enable_tensor_extension_casting = original + + +def test_from_modin(ray_start_regular_shared): + import modin.pandas as mopd + + df = pd.DataFrame( + {"one": list(range(100)), "two": list(range(100))}, + ) + modf = mopd.DataFrame(df) + ds = ray.data.from_modin(modf) + dfds = ds.to_pandas() + assert df.equals(dfds) + + +def test_to_modin(ray_start_regular_shared): + # create two modin dataframes + # one directly from a pandas dataframe, and + # another from ray.dataset created from the original pandas dataframe + # + import modin.pandas as mopd + + df = pd.DataFrame( + {"one": list(range(100)), "two": list(range(100))}, + ) + modf1 = mopd.DataFrame(df) + ds = ray.data.from_pandas([df]) + modf2 = ds.to_modin() + assert modf1.equals(modf2) + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main(["-v", __file__])) diff --git a/python/ray/data/tests/test_dataset_image.py b/python/ray/data/tests/test_dataset_image.py index b22fa3b378477..8d495b02bf4a7 100644 --- a/python/ray/data/tests/test_dataset_image.py +++ b/python/ray/data/tests/test_dataset_image.py @@ -1,5 +1,4 @@ import os -import time from typing import Dict import numpy as np @@ -194,16 +193,9 @@ def test_dynamic_block_split(ray_start_regular_shared): # Verify dynamic block splitting taking effect to generate more blocks. assert ds.num_blocks() == 3 - # NOTE: Need to wait for 1 second before checking stats, because we report - # stats to stats actors asynchronously when returning the blocks metadata. - # TODO(chengsu): clean it up after refactoring lazy block list. - time.sleep(1) - assert "3 blocks executed" in ds.stats() - # Test union of same datasets union_ds = ds.union(ds, ds, ds).fully_executed() assert union_ds.num_blocks() == 12 - assert "3 blocks executed" in union_ds.stats() finally: ctx.target_max_block_size = target_max_block_size ctx.block_splitting_enabled = block_splitting_enabled diff --git a/python/ray/data/tests/test_dataset_map.py b/python/ray/data/tests/test_dataset_map.py new file mode 100644 index 0000000000000..b082182e08efe --- /dev/null +++ b/python/ray/data/tests/test_dataset_map.py @@ -0,0 +1,925 @@ +import itertools +import math +import os +import signal +import time +from typing import Iterator + +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +import ray +from ray._private.test_utils import wait_for_condition +from ray.data.block import BlockAccessor +from ray.data.context import DatasetContext +from ray.data.tests.conftest import * # noqa +from ray.tests.conftest import * # noqa + + +def maybe_pipeline(ds, enabled): + if enabled: + return ds.window(blocks_per_window=1) + else: + return ds + + +@pytest.mark.parametrize("pipelined", [False, True]) +def test_basic_actors(shutdown_only, pipelined): + ray.init(num_cpus=6) + n = 5 + ds = ray.data.range(n) + ds = maybe_pipeline(ds, pipelined) + assert sorted(ds.map(lambda x: x + 1, compute="actors").take()) == list( + range(1, n + 1) + ) + + # Should still work even if num actors > num cpus. + ds = ray.data.range(n) + ds = maybe_pipeline(ds, pipelined) + assert sorted( + ds.map(lambda x: x + 1, compute=ray.data.ActorPoolStrategy(4, 4)).take() + ) == list(range(1, n + 1)) + + # Test setting custom max inflight tasks. + ds = ray.data.range(10, parallelism=5) + ds = maybe_pipeline(ds, pipelined) + assert sorted( + ds.map( + lambda x: x + 1, + compute=ray.data.ActorPoolStrategy(max_tasks_in_flight_per_actor=3), + ).take() + ) == list(range(1, 11)) + + # Test invalid max tasks inflight arg. + with pytest.raises(ValueError): + ray.data.range(10).map( + lambda x: x, + compute=ray.data.ActorPoolStrategy(max_tasks_in_flight_per_actor=0), + ) + + # Test min no more than max check. + with pytest.raises(ValueError): + ray.data.range(10).map(lambda x: x, compute=ray.data.ActorPoolStrategy(8, 4)) + + +def test_callable_classes(shutdown_only): + ray.init(num_cpus=2) + ds = ray.data.range(10, parallelism=10) + + class StatefulFn: + def __init__(self): + self.num_reuses = 0 + + def __call__(self, x): + r = self.num_reuses + self.num_reuses += 1 + return r + + # Need to specify compute explicitly. + with pytest.raises(ValueError): + ds.map(StatefulFn).take() + + # Need to specify actor compute strategy. + with pytest.raises(ValueError): + ds.map(StatefulFn, compute="tasks").take() + + # Need to specify compute explicitly. + with pytest.raises(ValueError): + ds.flat_map(StatefulFn).take() + + # Need to specify actor compute strategy. + with pytest.raises(ValueError): + ds.flat_map(StatefulFn, compute="tasks") + + # Need to specify compute explicitly. + with pytest.raises(ValueError): + ds.filter(StatefulFn).take() + + # Need to specify actor compute strategy. + with pytest.raises(ValueError): + ds.filter(StatefulFn, compute="tasks") + + # map + actor_reuse = ds.map(StatefulFn, compute="actors").take() + assert sorted(actor_reuse) == list(range(10)), actor_reuse + + class StatefulFn: + def __init__(self): + self.num_reuses = 0 + + def __call__(self, x): + r = self.num_reuses + self.num_reuses += 1 + return [r] + + # flat map + actor_reuse = ds.flat_map(StatefulFn, compute="actors").take() + assert sorted(actor_reuse) == list(range(10)), actor_reuse + + # map batches + actor_reuse = ds.map_batches(StatefulFn, batch_size=1, compute="actors").take() + assert sorted(actor_reuse) == list(range(10)), actor_reuse + + class StatefulFn: + def __init__(self): + self.num_reuses = 0 + + def __call__(self, x): + r = self.num_reuses + self.num_reuses += 1 + return r > 0 + + # filter + actor_reuse = ds.filter(StatefulFn, compute="actors").take() + assert len(actor_reuse) == 9, actor_reuse + + +def test_transform_failure(shutdown_only): + ray.init(num_cpus=2) + ds = ray.data.from_items([0, 10], parallelism=2) + + def mapper(x): + time.sleep(x) + raise ValueError("oops") + return x + + with pytest.raises(ray.exceptions.RayTaskError): + ds.map(mapper).fully_executed() + + +def test_flat_map_generator(ray_start_regular_shared): + ds = ray.data.range(3) + + def map_generator(item: int) -> Iterator[int]: + for _ in range(2): + yield item + 1 + + assert sorted(ds.flat_map(map_generator).take()) == [1, 1, 2, 2, 3, 3] + + +def test_add_column(ray_start_regular_shared): + ds = ray.data.range(5).add_column("foo", lambda x: 1) + assert ds.take(1) == [{"value": 0, "foo": 1}] + + ds = ray.data.range_table(5).add_column("foo", lambda x: x["value"] + 1) + assert ds.take(1) == [{"value": 0, "foo": 1}] + + ds = ray.data.range_table(5).add_column("value", lambda x: x["value"] + 1) + assert ds.take(2) == [{"value": 1}, {"value": 2}] + + with pytest.raises(ValueError): + ds = ray.data.range(5).add_column("value", 0) + + +def test_drop_columns(ray_start_regular_shared, tmp_path): + df = pd.DataFrame({"col1": [1, 2, 3], "col2": [2, 3, 4], "col3": [3, 4, 5]}) + ds1 = ray.data.from_pandas(df) + ds1.write_parquet(str(tmp_path)) + ds2 = ray.data.read_parquet(str(tmp_path)) + + for ds in [ds1, ds2]: + assert ds.drop_columns(["col2"]).take(1) == [{"col1": 1, "col3": 3}] + assert ds.drop_columns(["col1", "col3"]).take(1) == [{"col2": 2}] + assert ds.drop_columns([]).take(1) == [{"col1": 1, "col2": 2, "col3": 3}] + assert ds.drop_columns(["col1", "col2", "col3"]).take(1) == [{}] + assert ds.drop_columns(["col1", "col1", "col2", "col1"]).take(1) == [ + {"col3": 3} + ] + # Test dropping non-existent column + with pytest.raises(KeyError): + ds.drop_columns(["dummy_col", "col1", "col2"]).fully_executed() + + +def test_select_columns(ray_start_regular_shared): + # Test pandas and arrow + df = pd.DataFrame({"col1": [1, 2, 3], "col2": [2, 3, 4], "col3": [3, 4, 5]}) + ds1 = ray.data.from_pandas(df) + assert ds1.dataset_format() == "pandas" + + ds2 = ds1.map_batches(lambda pa: pa, batch_size=1, batch_format="pyarrow") + assert ds2.dataset_format() == "arrow" + + for each_ds in [ds1, ds2]: + assert each_ds.select_columns(cols=[]).take(1) == [{}] + assert each_ds.select_columns(cols=["col1", "col2", "col3"]).take(1) == [ + {"col1": 1, "col2": 2, "col3": 3} + ] + assert each_ds.select_columns(cols=["col1", "col2"]).take(1) == [ + {"col1": 1, "col2": 2} + ] + assert each_ds.select_columns(cols=["col2", "col1"]).take(1) == [ + {"col1": 1, "col2": 2} + ] + # Test selecting columns with duplicates + assert each_ds.select_columns(cols=["col1", "col2", "col2"]).schema().names == [ + "col1", + "col2", + "col2", + ] + # Test selecting a column that is not in the dataset schema + with pytest.raises(KeyError): + each_ds.select_columns(cols=["col1", "col2", "dummy_col"]).fully_executed() + + # Test simple + ds3 = ray.data.range(10) + assert ds3.dataset_format() == "simple" + with pytest.raises(ValueError): + ds3.select_columns(cols=[]).fully_executed() + + +def test_map_batches_basic(ray_start_regular_shared, tmp_path, restore_dataset_context): + ctx = DatasetContext.get_current() + ctx.execution_options.preserve_order = True + + # Test input validation + ds = ray.data.range(5) + with pytest.raises(ValueError): + ds.map_batches(lambda x: x + 1, batch_format="pyarrow", batch_size=-1).take() + + # Set up. + df = pd.DataFrame({"one": [1, 2, 3], "two": [2, 3, 4]}) + table = pa.Table.from_pandas(df) + pq.write_table(table, os.path.join(tmp_path, "test1.parquet")) + + # Test pandas + ds = ray.data.read_parquet(str(tmp_path)) + ds2 = ds.map_batches(lambda df: df + 1, batch_size=1, batch_format="pandas") + assert ds2.dataset_format() == "pandas" + ds_list = ds2.take() + values = [s["one"] for s in ds_list] + assert values == [2, 3, 4] + values = [s["two"] for s in ds_list] + assert values == [3, 4, 5] + + # Test Pyarrow + ds = ray.data.read_parquet(str(tmp_path)) + ds2 = ds.map_batches(lambda pa: pa, batch_size=1, batch_format="pyarrow") + assert ds2.dataset_format() == "arrow" + ds_list = ds2.take() + values = [s["one"] for s in ds_list] + assert values == [1, 2, 3] + values = [s["two"] for s in ds_list] + assert values == [2, 3, 4] + + # Test batch + size = 300 + ds = ray.data.range(size) + ds2 = ds.map_batches(lambda df: df + 1, batch_size=17, batch_format="pandas") + assert ds2.dataset_format() == "pandas" + ds_list = ds2.take_all() + for i in range(size): + # The pandas column is "value", and it originally has rows from 0~299. + # After the map batch, it should have 1~300. + row = ds_list[i] + assert row["value"] == i + 1 + assert ds.count() == 300 + + # Test the lambda returns different types than the batch_format + # pandas => list block + ds = ray.data.read_parquet(str(tmp_path)) + ds2 = ds.map_batches(lambda df: [1], batch_size=1) + assert ds2.dataset_format() == "simple" + ds_list = ds2.take() + assert ds_list == [1, 1, 1] + assert ds.count() == 3 + + # pyarrow => list block + ds = ray.data.read_parquet(str(tmp_path)) + ds2 = ds.map_batches(lambda df: [1], batch_size=1, batch_format="pyarrow") + assert ds2.dataset_format() == "simple" + ds_list = ds2.take() + assert ds_list == [1, 1, 1] + assert ds.count() == 3 + + # Test the wrong return value raises an exception. + ds = ray.data.read_parquet(str(tmp_path)) + with pytest.raises(ValueError): + ds_list = ds.map_batches( + lambda df: 1, batch_size=2, batch_format="pyarrow" + ).take() + + +def test_map_batches_extra_args(shutdown_only, tmp_path): + ray.shutdown() + ray.init(num_cpus=2) + + def put(x): + # We only support automatic deref in the legacy backend. + if DatasetContext.get_current().new_execution_backend: + return x + else: + return ray.put(x) + + # Test input validation + ds = ray.data.range(5) + + class Foo: + def __call__(self, df): + return df + + with pytest.raises(ValueError): + # CallableClass not supported for task compute strategy, which is the default. + ds.map_batches(Foo) + + with pytest.raises(ValueError): + # CallableClass not supported for task compute strategy. + ds.map_batches(Foo, compute="tasks") + + with pytest.raises(ValueError): + # fn_constructor_args and fn_constructor_kwargs only supported for actor + # compute strategy. + ds.map_batches( + lambda x: x, + compute="tasks", + fn_constructor_args=(1,), + fn_constructor_kwargs={"a": 1}, + ) + + with pytest.raises(ValueError): + # fn_constructor_args and fn_constructor_kwargs only supported for callable + # class UDFs. + ds.map_batches( + lambda x: x, + compute="actors", + fn_constructor_args=(1,), + fn_constructor_kwargs={"a": 1}, + ) + + # Set up. + df = pd.DataFrame({"one": [1, 2, 3], "two": [2, 3, 4]}) + table = pa.Table.from_pandas(df) + pq.write_table(table, os.path.join(tmp_path, "test1.parquet")) + + # Test extra UDF args. + # Test positional. + def udf(batch, a): + assert a == 1 + return batch + a + + ds = ray.data.read_parquet(str(tmp_path)) + ds2 = ds.map_batches( + udf, + batch_size=1, + batch_format="pandas", + fn_args=(put(1),), + ) + assert ds2.dataset_format() == "pandas" + ds_list = ds2.take() + values = [s["one"] for s in ds_list] + assert values == [2, 3, 4] + values = [s["two"] for s in ds_list] + assert values == [3, 4, 5] + + # Test kwargs. + def udf(batch, b=None): + assert b == 2 + return b * batch + + ds = ray.data.read_parquet(str(tmp_path)) + ds2 = ds.map_batches( + udf, + batch_size=1, + batch_format="pandas", + fn_kwargs={"b": put(2)}, + ) + assert ds2.dataset_format() == "pandas" + ds_list = ds2.take() + values = [s["one"] for s in ds_list] + assert values == [2, 4, 6] + values = [s["two"] for s in ds_list] + assert values == [4, 6, 8] + + # Test both. + def udf(batch, a, b=None): + assert a == 1 + assert b == 2 + return b * batch + a + + ds = ray.data.read_parquet(str(tmp_path)) + ds2 = ds.map_batches( + udf, + batch_size=1, + batch_format="pandas", + fn_args=(put(1),), + fn_kwargs={"b": put(2)}, + ) + assert ds2.dataset_format() == "pandas" + ds_list = ds2.take() + values = [s["one"] for s in ds_list] + assert values == [3, 5, 7] + values = [s["two"] for s in ds_list] + assert values == [5, 7, 9] + + # Test constructor UDF args. + # Test positional. + class CallableFn: + def __init__(self, a): + assert a == 1 + self.a = a + + def __call__(self, x): + return x + self.a + + ds = ray.data.read_parquet(str(tmp_path)) + ds2 = ds.map_batches( + CallableFn, + batch_size=1, + batch_format="pandas", + compute="actors", + fn_constructor_args=(put(1),), + ) + assert ds2.dataset_format() == "pandas" + ds_list = ds2.take() + values = [s["one"] for s in ds_list] + assert values == [2, 3, 4] + values = [s["two"] for s in ds_list] + assert values == [3, 4, 5] + + # Test kwarg. + class CallableFn: + def __init__(self, b=None): + assert b == 2 + self.b = b + + def __call__(self, x): + return self.b * x + + ds = ray.data.read_parquet(str(tmp_path)) + ds2 = ds.map_batches( + CallableFn, + batch_size=1, + batch_format="pandas", + compute="actors", + fn_constructor_kwargs={"b": put(2)}, + ) + assert ds2.dataset_format() == "pandas" + ds_list = ds2.take() + values = [s["one"] for s in ds_list] + assert values == [2, 4, 6] + values = [s["two"] for s in ds_list] + assert values == [4, 6, 8] + + # Test both. + class CallableFn: + def __init__(self, a, b=None): + assert a == 1 + assert b == 2 + self.a = a + self.b = b + + def __call__(self, x): + return self.b * x + self.a + + ds = ray.data.read_parquet(str(tmp_path)) + ds2 = ds.map_batches( + CallableFn, + batch_size=1, + batch_format="pandas", + compute="actors", + fn_constructor_args=(put(1),), + fn_constructor_kwargs={"b": put(2)}, + ) + assert ds2.dataset_format() == "pandas" + ds_list = ds2.take() + values = [s["one"] for s in ds_list] + assert values == [3, 5, 7] + values = [s["two"] for s in ds_list] + assert values == [5, 7, 9] + + # Test callable chain. + ds = ray.data.read_parquet(str(tmp_path)) + fn_constructor_args = (put(1),) + fn_constructor_kwargs = {"b": put(2)} + ds2 = ( + ds.lazy() + .map_batches( + CallableFn, + batch_size=1, + batch_format="pandas", + compute="actors", + fn_constructor_args=fn_constructor_args, + fn_constructor_kwargs=fn_constructor_kwargs, + ) + .map_batches( + CallableFn, + batch_size=1, + batch_format="pandas", + compute="actors", + fn_constructor_args=fn_constructor_args, + fn_constructor_kwargs=fn_constructor_kwargs, + ) + ) + assert ds2.dataset_format() == "pandas" + ds_list = ds2.take() + values = [s["one"] for s in ds_list] + assert values == [7, 11, 15] + values = [s["two"] for s in ds_list] + assert values == [11, 15, 19] + + # Test function + callable chain. + ds = ray.data.read_parquet(str(tmp_path)) + fn_constructor_args = (put(1),) + fn_constructor_kwargs = {"b": put(2)} + ds2 = ( + ds.lazy() + .map_batches( + lambda df, a, b=None: b * df + a, + batch_size=1, + batch_format="pandas", + compute="actors", + fn_args=(put(1),), + fn_kwargs={"b": put(2)}, + ) + .map_batches( + CallableFn, + batch_size=1, + batch_format="pandas", + compute="actors", + fn_constructor_args=fn_constructor_args, + fn_constructor_kwargs=fn_constructor_kwargs, + ) + ) + assert ds2.dataset_format() == "pandas" + ds_list = ds2.take() + values = [s["one"] for s in ds_list] + assert values == [7, 11, 15] + values = [s["two"] for s in ds_list] + assert values == [11, 15, 19] + + +def test_map_batches_generator(ray_start_regular_shared, tmp_path): + # Set up. + df = pd.DataFrame({"one": [1, 2, 3], "two": [2, 3, 4]}) + table = pa.Table.from_pandas(df) + pq.write_table(table, os.path.join(tmp_path, "test1.parquet")) + + def pandas_generator(batch: pd.DataFrame) -> Iterator[pd.DataFrame]: + for i in range(len(batch)): + yield batch.iloc[[i]] + 1 + + ds = ray.data.read_parquet(str(tmp_path)) + ds2 = ds.map_batches(pandas_generator, batch_size=1, batch_format="pandas") + assert ds2.dataset_format() == "pandas" + ds_list = ds2.take() + values = sorted([s["one"] for s in ds_list]) + assert values == [2, 3, 4] + values = sorted([s["two"] for s in ds_list]) + assert values == [3, 4, 5] + + def fail_generator(batch): + for i in range(len(batch)): + yield i + + # Test the wrong return value raises an exception. + ds = ray.data.read_parquet(str(tmp_path)) + with pytest.raises(ValueError): + ds_list = ds.map_batches( + fail_generator, batch_size=2, batch_format="pyarrow" + ).take() + + +def test_map_batches_actors_preserves_order(shutdown_only): + ray.shutdown() + ray.init(num_cpus=2) + # Test that actor compute model preserves block order. + ds = ray.data.range(10, parallelism=5) + assert ds.map_batches(lambda x: x, compute="actors").take() == list(range(10)) + + +@pytest.mark.parametrize( + "num_rows,num_blocks,batch_size", + [ + (10, 5, 2), + (10, 1, 10), + (12, 3, 2), + ], +) +def test_map_batches_batch_mutation( + ray_start_regular_shared, num_rows, num_blocks, batch_size, restore_dataset_context +): + ctx = DatasetContext.get_current() + ctx.execution_options.preserve_order = True + + # Test that batch mutation works without encountering a read-only error (e.g. if the + # batch is a zero-copy view on data in the object store). + def mutate(df): + df["value"] += 1 + return df + + ds = ray.data.range_table(num_rows, parallelism=num_blocks).repartition(num_blocks) + # Convert to Pandas blocks. + ds = ds.map_batches(lambda df: df, batch_format="pandas", batch_size=None) + + # Apply UDF that mutates the batches. + ds = ds.map_batches(mutate, batch_size=batch_size) + assert [row["value"] for row in ds.iter_rows()] == list(range(1, num_rows + 1)) + + +@pytest.mark.parametrize( + "num_rows,num_blocks,batch_size", + [ + (10, 5, 2), + (10, 1, 10), + (12, 3, 2), + ], +) +def test_map_batches_batch_zero_copy( + ray_start_regular_shared, num_rows, num_blocks, batch_size +): + # Test that batches are zero-copy read-only views when zero_copy_batch=True. + def mutate(df): + # Check that batch is read-only. + assert not df.values.flags.writeable + df["value"] += 1 + return df + + ds = ray.data.range_table(num_rows, parallelism=num_blocks).repartition(num_blocks) + # Convert to Pandas blocks. + ds = ds.map_batches(lambda df: df, batch_format="pandas", batch_size=None) + ds.fully_executed() + + # Apply UDF that mutates the batches, which should fail since the batch is + # read-only. + with pytest.raises(ValueError, match="tried to mutate a zero-copy read-only batch"): + ds = ds.map_batches(mutate, batch_size=batch_size, zero_copy_batch=True) + ds.fully_executed() + + +BLOCK_BUNDLING_TEST_CASES = [ + (block_size, batch_size) + for batch_size in range(1, 8) + for block_size in range(1, 2 * batch_size + 1) +] + + +@pytest.mark.parametrize("block_size,batch_size", BLOCK_BUNDLING_TEST_CASES) +def test_map_batches_block_bundling_auto( + ray_start_regular_shared, block_size, batch_size +): + # Ensure that we test at least 2 batches worth of blocks. + num_blocks = max(10, 2 * batch_size // block_size) + ds = ray.data.range(num_blocks * block_size, parallelism=num_blocks) + # Confirm that we have the expected number of initial blocks. + assert ds.num_blocks() == num_blocks + + # Blocks should be bundled up to the batch size. + ds1 = ds.map_batches(lambda x: x, batch_size=batch_size).fully_executed() + assert ds1.num_blocks() == math.ceil(num_blocks / max(batch_size // block_size, 1)) + + # Blocks should not be bundled up when batch_size is not specified. + ds2 = ds.map_batches(lambda x: x).fully_executed() + assert ds2.num_blocks() == num_blocks + + +@pytest.mark.parametrize( + "block_sizes,batch_size,expected_num_blocks", + [ + ([1, 2], 3, 1), + ([2, 2, 1], 3, 2), + ([1, 2, 3, 4], 4, 3), + ([3, 1, 1, 3], 4, 2), + ([2, 4, 1, 8], 4, 4), + ([1, 1, 1, 1], 4, 1), + ([1, 0, 3, 2], 4, 2), + ([4, 4, 4, 4], 4, 4), + ], +) +def test_map_batches_block_bundling_skewed_manual( + ray_start_regular_shared, block_sizes, batch_size, expected_num_blocks +): + num_blocks = len(block_sizes) + ds = ray.data.from_pandas( + [pd.DataFrame({"a": [1] * block_size}) for block_size in block_sizes] + ) + # Confirm that we have the expected number of initial blocks. + assert ds.num_blocks() == num_blocks + ds = ds.map_batches(lambda x: x, batch_size=batch_size).fully_executed() + + # Blocks should be bundled up to the batch size. + assert ds.num_blocks() == expected_num_blocks + + +BLOCK_BUNDLING_SKEWED_TEST_CASES = [ + (block_sizes, batch_size) + for batch_size in range(1, 4) + for num_blocks in range(1, batch_size + 1) + for block_sizes in itertools.product( + range(1, 2 * batch_size + 1), repeat=num_blocks + ) +] + + +@pytest.mark.parametrize("block_sizes,batch_size", BLOCK_BUNDLING_SKEWED_TEST_CASES) +def test_map_batches_block_bundling_skewed_auto( + ray_start_regular_shared, block_sizes, batch_size +): + num_blocks = len(block_sizes) + ds = ray.data.from_pandas( + [pd.DataFrame({"a": [1] * block_size}) for block_size in block_sizes] + ) + # Confirm that we have the expected number of initial blocks. + assert ds.num_blocks() == num_blocks + ds = ds.map_batches(lambda x: x, batch_size=batch_size).fully_executed() + curr = 0 + num_out_blocks = 0 + for block_size in block_sizes: + if curr > 0 and curr + block_size > batch_size: + num_out_blocks += 1 + curr = 0 + curr += block_size + if curr > 0: + num_out_blocks += 1 + + # Blocks should be bundled up to the batch size. + assert ds.num_blocks() == num_out_blocks + + +def test_map_with_mismatched_columns(ray_start_regular_shared): + def bad_fn(row): + if row > 5: + return {"a": "hello1"} + else: + return {"b": "hello1"} + + def good_fn(row): + if row > 5: + return {"a": "hello1", "b": "hello2"} + else: + return {"b": "hello2", "a": "hello1"} + + ds = ray.data.range(10, parallelism=1) + error_message = "Current row has different columns compared to previous rows." + with pytest.raises(ValueError) as e: + ds.map(bad_fn).fully_executed() + assert error_message in str(e.value) + ds_map = ds.map(good_fn) + assert ds_map.take() == [{"a": "hello1", "b": "hello2"} for _ in range(10)] + + +def test_map_batches_preserve_empty_blocks(ray_start_regular_shared): + ds = ray.data.range(10, parallelism=10) + ds = ds.map_batches(lambda x: []) + ds = ds.map_batches(lambda x: x) + assert ds.num_blocks() == 10, ds + + +def test_map_batches_combine_empty_blocks(ray_start_regular_shared): + xs = [x % 3 for x in list(range(100))] + + # ds1 has 1 block which contains 100 rows. + ds1 = ray.data.from_items(xs).repartition(1).sort().map_batches(lambda x: x) + assert ds1._block_num_rows() == [100] + + # ds2 has 30 blocks, but only 3 of them are non-empty + ds2 = ( + ray.data.from_items(xs) + .repartition(30) + .sort() + .map_batches(lambda x: x, batch_size=1) + ) + assert len(ds2._block_num_rows()) == 3 + count = sum(1 for x in ds2._block_num_rows() if x > 0) + assert count == 3 + + # The number of partitions should not affect the map_batches() result. + assert ds1.take_all() == ds2.take_all() + + +def test_random_sample(ray_start_regular_shared): + import math + + def ensure_sample_size_close(dataset, sample_percent=0.5): + r1 = ds.random_sample(sample_percent) + assert math.isclose( + r1.count(), int(ds.count() * sample_percent), rel_tol=2, abs_tol=2 + ) + + ds = ray.data.range(10, parallelism=2) + ensure_sample_size_close(ds) + + ds = ray.data.range_table(10, parallelism=2) + ensure_sample_size_close(ds) + + ds = ray.data.range_tensor(5, parallelism=2, shape=(2, 2)) + ensure_sample_size_close(ds) + + # imbalanced datasets + ds1 = ray.data.range(1, parallelism=1) + ds2 = ray.data.range(2, parallelism=1) + ds3 = ray.data.range(3, parallelism=1) + # noinspection PyTypeChecker + ds = ds1.union(ds2).union(ds3) + ensure_sample_size_close(ds) + # Small datasets + ds1 = ray.data.range(5, parallelism=5) + ensure_sample_size_close(ds1) + + +def test_random_sample_checks(ray_start_regular_shared): + with pytest.raises(ValueError): + # Cannot sample -1 + ray.data.range(1).random_sample(-1) + with pytest.raises(ValueError): + # Cannot sample from empty dataset + ray.data.range(0).random_sample(0.2) + with pytest.raises(ValueError): + # Cannot sample fraction > 1 + ray.data.range(1).random_sample(10) + + +# NOTE: All tests above share a Ray cluster, while the tests below do not. These +# tests should only be carefully reordered to retain this invariant! + + +def test_actor_pool_strategy_apply_interrupt(shutdown_only): + """Test that _apply kills the actor pool if an interrupt is raised.""" + ray.shutdown() + + ray.init(include_dashboard=False, num_cpus=1) + + cpus = ray.available_resources()["CPU"] + ds = ray.data.range(5, parallelism=5) + aps = ray.data.ActorPoolStrategy(max_size=5) + blocks = ds._plan.execute() + + # Start some actors, the first one sends a SIGINT, emulating a KeyboardInterrupt + def test_func(block): + for i, _ in enumerate(BlockAccessor.for_block(block).iter_rows()): + if i == 0: + os.kill(os.getpid(), signal.SIGINT) + else: + time.sleep(1000) + return block + + # No need to test ActorPoolStrategy in new execution backend. + if not DatasetContext.get_current().new_execution_backend: + with pytest.raises(ray.exceptions.RayTaskError): + aps._apply(test_func, {}, blocks, False) + + # Check that all actors have been killed by counting the available CPUs + wait_for_condition(lambda: (ray.available_resources().get("CPU", 0) == cpus)) + + +def test_actor_pool_strategy_default_num_actors(shutdown_only): + def f(x): + import time + + time.sleep(1) + return x + + num_cpus = 5 + ray.init(num_cpus=num_cpus) + compute_strategy = ray.data.ActorPoolStrategy() + ray.data.range(10, parallelism=10).map_batches( + f, batch_size=1, compute=compute_strategy + ).fully_executed() + + # The new execution backend is not using the ActorPoolStrategy under + # the hood, so the expectation here applies only to the old backend. + # TODO(https://github.com/ray-project/ray/issues/31723): we should check + # the num of workers once we have autoscaling in new execution backend. + if not DatasetContext.get_current().new_execution_backend: + expected_max_num_workers = math.ceil( + num_cpus * (1 / compute_strategy.ready_to_total_workers_ratio) + ) + assert ( + compute_strategy.num_workers >= num_cpus + and compute_strategy.num_workers <= expected_max_num_workers + ), "Number of actors is out of the expected bound" + + +def test_actor_pool_strategy_bundles_to_max_actors(shutdown_only): + """Tests that blocks are bundled up to the specified max number of actors.""" + + def f(x): + return x + + max_size = 2 + compute_strategy = ray.data.ActorPoolStrategy(max_size=max_size) + ds = ( + ray.data.range(10, parallelism=10) + .map_batches(f, batch_size=None, compute=compute_strategy) + .fully_executed() + ) + + # TODO(https://github.com/ray-project/ray/issues/31723): implement the feature + # of capping bundle size by actor pool size, and then re-enable this test. + if not DatasetContext.get_current().new_execution_backend: + assert f"{max_size}/{max_size} blocks" in ds.stats() + + # Check batch size is still respected. + ds = ( + ray.data.range(10, parallelism=10) + .map_batches(f, batch_size=10, compute=compute_strategy) + .fully_executed() + ) + + assert "1/1 blocks" in ds.stats() + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main(["-v", __file__])) diff --git a/python/ray/data/tests/test_dataset_numpy.py b/python/ray/data/tests/test_dataset_numpy.py index d882315ce72b2..b36c549812321 100644 --- a/python/ray/data/tests/test_dataset_numpy.py +++ b/python/ray/data/tests/test_dataset_numpy.py @@ -122,8 +122,11 @@ def test_numpy_roundtrip(ray_start_regular_shared, fs, data_path): ds.write_numpy(data_path, filesystem=fs) ds = ray.data.read_numpy(data_path, filesystem=fs) assert str(ds) == ( - "Dataset(num_blocks=2, num_rows=?, " - "schema={__value__: ArrowTensorType(shape=(1,), dtype=int64)})" + "Dataset(\n" + " num_blocks=2,\n" + " num_rows=?,\n" + " schema={__value__: ArrowTensorType(shape=(1,), dtype=int64)}\n" + ")" ) np.testing.assert_equal(ds.take(2), [np.array([0]), np.array([1])]) @@ -134,8 +137,11 @@ def test_numpy_read(ray_start_regular_shared, tmp_path): np.save(os.path.join(path, "test.npy"), np.expand_dims(np.arange(0, 10), 1)) ds = ray.data.read_numpy(path) assert str(ds) == ( - "Dataset(num_blocks=1, num_rows=10, " - "schema={__value__: ArrowTensorType(shape=(1,), dtype=int64)})" + "Dataset(\n" + " num_blocks=1,\n" + " num_rows=10,\n" + " schema={__value__: ArrowTensorType(shape=(1,), dtype=int64)}\n" + ")" ) np.testing.assert_equal(ds.take(2), [np.array([0]), np.array([1])]) @@ -147,8 +153,11 @@ def test_numpy_read(ray_start_regular_shared, tmp_path): assert ds.num_blocks() == 1 assert ds.count() == 10 assert str(ds) == ( - "Dataset(num_blocks=1, num_rows=10, " - "schema={__value__: ArrowTensorType(shape=(1,), dtype=int64)})" + "Dataset(\n" + " num_blocks=1,\n" + " num_rows=10,\n" + " schema={__value__: ArrowTensorType(shape=(1,), dtype=int64)}\n" + ")" ) assert [v.item() for v in ds.take(2)] == [0, 1] @@ -160,8 +169,11 @@ def test_numpy_read_meta_provider(ray_start_regular_shared, tmp_path): np.save(path, np.expand_dims(np.arange(0, 10), 1)) ds = ray.data.read_numpy(path, meta_provider=FastFileMetadataProvider()) assert str(ds) == ( - "Dataset(num_blocks=1, num_rows=10, " - "schema={__value__: ArrowTensorType(shape=(1,), dtype=int64)})" + "Dataset(\n" + " num_blocks=1,\n" + " num_rows=10,\n" + " schema={__value__: ArrowTensorType(shape=(1,), dtype=int64)}\n" + ")" ) np.testing.assert_equal(ds.take(2), [np.array([0]), np.array([1])]) diff --git a/python/ray/data/tests/test_dataset_pandas.py b/python/ray/data/tests/test_dataset_pandas.py index 75e33872b0e80..b8064e34354fd 100644 --- a/python/ray/data/tests/test_dataset_pandas.py +++ b/python/ray/data/tests/test_dataset_pandas.py @@ -7,7 +7,6 @@ from ray.data.extensions import ( TensorDtype, - TensorArray, ArrowTensorType, ArrowTensorArray, ) @@ -112,14 +111,18 @@ def test_to_pandas_tensor_column_cast_pandas(ray_start_regular_shared): original = ctx.enable_tensor_extension_casting try: ctx.enable_tensor_extension_casting = True - in_df = pd.DataFrame({"a": TensorArray(data)}) + in_df = pd.DataFrame({"a": [data]}) ds = ray.data.from_pandas(in_df) dtypes = ds.schema().types assert len(dtypes) == 1 + # Tensor column should be automatically cast to Tensor extension. assert isinstance(dtypes[0], TensorDtype) + # Original df should not be changed. + assert not isinstance(in_df.dtypes[0], TensorDtype) out_df = ds.to_pandas() + # Column should be cast back to object dtype when returning back to user. assert out_df["a"].dtype.type is np.object_ - expected_df = pd.DataFrame({"a": list(data)}) + expected_df = pd.DataFrame({"a": [data]}) pd.testing.assert_frame_equal(out_df, expected_df) finally: ctx.enable_tensor_extension_casting = original diff --git a/python/ray/data/tests/test_dataset_parquet.py b/python/ray/data/tests/test_dataset_parquet.py index 999cef8f56ed1..d0af09650484a 100644 --- a/python/ray/data/tests/test_dataset_parquet.py +++ b/python/ray/data/tests/test_dataset_parquet.py @@ -14,9 +14,11 @@ DefaultFileMetadataProvider, DefaultParquetMetadataProvider, ) +from ray.data.datasource.parquet_base_datasource import ParquetBaseDatasource from ray.data.datasource.parquet_datasource import ( PARALLELIZE_META_FETCH_THRESHOLD, _ParquetDatasourceReader, + ParquetDatasource, ) from ray.data.datasource.file_based_datasource import _unwrap_protocol from ray.data.datasource.parquet_datasource import ( @@ -444,15 +446,21 @@ def test_parquet_read_partitioned(ray_start_regular_shared, fs, data_path): assert ds.schema() is not None input_files = ds.input_files() assert len(input_files) == 2, input_files - assert ( - str(ds) == "Dataset(num_blocks=2, num_rows=6, " - "schema={two: string, " - "one: dictionary})" + assert str(ds) == ( + "Dataset(\n" + " num_blocks=2,\n" + " num_rows=6,\n" + " schema={two: string, " + "one: dictionary}\n" + ")" ), ds - assert ( - repr(ds) == "Dataset(num_blocks=2, num_rows=6, " - "schema={two: string, " - "one: dictionary})" + assert repr(ds) == ( + "Dataset(\n" + " num_blocks=2,\n" + " num_rows=6,\n" + " schema={two: string, " + "one: dictionary}\n" + ")" ), ds check_num_computed(ds, 1, 1) @@ -929,6 +937,57 @@ def test_parquet_reader_batch_size(ray_start_regular_shared, tmp_path): assert ds.count() == 1000 +def test_parquet_datasource_names(ray_start_regular_shared): + assert ParquetBaseDatasource().get_name() == "ParquetBulk" + assert ParquetDatasource().get_name() == "Parquet" + + +# NOTE: All tests above share a Ray cluster, while the tests below do not. These +# tests should only be carefully reordered to retain this invariant! + + +def test_parquet_read_spread(ray_start_cluster, tmp_path): + ray.shutdown() + cluster = ray_start_cluster + cluster.add_node( + resources={"bar:1": 100}, + num_cpus=10, + _system_config={"max_direct_call_object_size": 0}, + ) + cluster.add_node(resources={"bar:2": 100}, num_cpus=10) + cluster.add_node(resources={"bar:3": 100}, num_cpus=0) + + ray.init(cluster.address) + + @ray.remote + def get_node_id(): + return ray.get_runtime_context().get_node_id() + + node1_id = ray.get(get_node_id.options(resources={"bar:1": 1}).remote()) + node2_id = ray.get(get_node_id.options(resources={"bar:2": 1}).remote()) + + data_path = str(tmp_path) + df1 = pd.DataFrame({"one": list(range(100)), "two": list(range(100, 200))}) + path1 = os.path.join(data_path, "test1.parquet") + df1.to_parquet(path1) + df2 = pd.DataFrame({"one": list(range(300, 400)), "two": list(range(400, 500))}) + path2 = os.path.join(data_path, "test2.parquet") + df2.to_parquet(path2) + + ds = ray.data.read_parquet(data_path) + + # Force reads. + blocks = ds.get_internal_block_refs() + assert len(blocks) == 2 + + ray.wait(blocks, num_returns=len(blocks), fetch_local=False) + location_data = ray.experimental.get_object_locations(blocks) + locations = [] + for block in blocks: + locations.extend(location_data[block]["node_ids"]) + assert set(locations) == {node1_id, node2_id} + + if __name__ == "__main__": import sys diff --git a/python/ray/data/tests/test_dataset_tensor.py b/python/ray/data/tests/test_dataset_tensor.py index e95066fae5079..4d0ca0f6d8eea 100644 --- a/python/ray/data/tests/test_dataset_tensor.py +++ b/python/ray/data/tests/test_dataset_tensor.py @@ -23,8 +23,11 @@ def test_tensors_basic(ray_start_regular_shared): tensor_shape = (3, 5) ds = ray.data.range_tensor(6, shape=tensor_shape, parallelism=6) assert str(ds) == ( - "Dataset(num_blocks=6, num_rows=6, " - "schema={__value__: ArrowTensorType(shape=(3, 5), dtype=int64)})" + "Dataset(\n" + " num_blocks=6,\n" + " num_rows=6,\n" + " schema={__value__: ArrowTensorType(shape=(3, 5), dtype=int64)}\n" + ")" ) assert ds.size_bytes() == 5 * 3 * 6 * 8 @@ -273,8 +276,11 @@ def test_tensors_inferred_from_map(ray_start_regular_shared): ds = ray.data.range(10, parallelism=10).map(lambda _: np.ones((4, 4))) ds.fully_executed() assert str(ds) == ( - "Dataset(num_blocks=10, num_rows=10, " - "schema={__value__: ArrowTensorType(shape=(4, 4), dtype=double)})" + "Dataset(\n" + " num_blocks=10,\n" + " num_rows=10,\n" + " schema={__value__: ArrowTensorType(shape=(4, 4), dtype=double)}\n" + ")" ) # Test map_batches. @@ -283,8 +289,11 @@ def test_tensors_inferred_from_map(ray_start_regular_shared): ) ds.fully_executed() assert str(ds) == ( - "Dataset(num_blocks=4, num_rows=24, " - "schema={__value__: ArrowTensorType(shape=(4, 4), dtype=double)})" + "Dataset(\n" + " num_blocks=4,\n" + " num_rows=24,\n" + " schema={__value__: ArrowTensorType(shape=(4, 4), dtype=double)}\n" + ")" ) # Test flat_map. @@ -293,8 +302,11 @@ def test_tensors_inferred_from_map(ray_start_regular_shared): ) ds.fully_executed() assert str(ds) == ( - "Dataset(num_blocks=10, num_rows=20, " - "schema={__value__: ArrowTensorType(shape=(4, 4), dtype=double)})" + "Dataset(\n" + " num_blocks=10,\n" + " num_rows=20,\n" + " schema={__value__: ArrowTensorType(shape=(4, 4), dtype=double)}\n" + ")" ) # Test map_batches ndarray column. @@ -303,8 +315,11 @@ def test_tensors_inferred_from_map(ray_start_regular_shared): ) ds.fully_executed() assert str(ds) == ( - "Dataset(num_blocks=4, num_rows=24, " - "schema={a: TensorDtype(shape=(4, 4), dtype=float64)})" + "Dataset(\n" + " num_blocks=4,\n" + " num_rows=24,\n" + " schema={a: TensorDtype(shape=(4, 4), dtype=float64)}\n" + ")" ) ds = ray.data.range(16, parallelism=4).map_batches( @@ -313,8 +328,11 @@ def test_tensors_inferred_from_map(ray_start_regular_shared): ) ds.fully_executed() assert str(ds) == ( - "Dataset(num_blocks=4, num_rows=16, " - "schema={a: TensorDtype(shape=(None, None), dtype=float64)})" + "Dataset(\n" + " num_blocks=4,\n" + " num_rows=16,\n" + " schema={a: TensorDtype(shape=(None, None), dtype=float64)}\n" + ")" ) diff --git a/python/ray/data/tests/test_dataset_tf.py b/python/ray/data/tests/test_dataset_tf.py index 2be4a34922cc7..799c595478990 100644 --- a/python/ray/data/tests/test_dataset_tf.py +++ b/python/ray/data/tests/test_dataset_tf.py @@ -7,7 +7,6 @@ from ray.air import session from ray.air.config import ScalingConfig from ray.air.constants import TENSOR_COLUMN_NAME -from ray.data.extensions import TensorArray from ray.data.preprocessors import Concatenator from ray.train.tensorflow import TensorflowTrainer @@ -139,7 +138,7 @@ def test_element_spec_pipeline(self): def test_element_spec_shape_with_ragged_tensors(self, batch_size): df = pd.DataFrame( { - "spam": TensorArray([np.zeros([32, 32, 3]), np.zeros([64, 64, 3])]), + "spam": [np.zeros([32, 32, 3]), np.zeros([64, 64, 3])], "ham": [0, 0], } ) diff --git a/python/ray/data/tests/test_dynamic_block_split.py b/python/ray/data/tests/test_dynamic_block_split.py index 252e27c30fa0c..1b58747d21116 100644 --- a/python/ray/data/tests/test_dynamic_block_split.py +++ b/python/ray/data/tests/test_dynamic_block_split.py @@ -1,5 +1,8 @@ +import time + import numpy as np import pandas as pd +import pyarrow as pa import pytest import ray @@ -7,6 +10,7 @@ from ray.data.block import BlockMetadata from ray.data.context import DatasetContext from ray.data.datasource import Datasource +from ray.data.datasource.csv_datasource import CSVDatasource from ray.data.datasource.datasource import ReadTask, Reader from ray.tests.conftest import * # noqa @@ -15,12 +19,14 @@ # Data source generates random bytes data class RandomBytesDatasource(Datasource): def create_reader(self, **read_args): - return RandomBytesReader(read_args["num_blocks"], read_args["block_size"]) + return RandomBytesReader( + read_args["num_blocks_per_task"], read_args["block_size"] + ) class RandomBytesReader(Reader): - def __init__(self, num_blocks: int, block_size: int): - self.num_blocks = num_blocks + def __init__(self, num_blocks_per_task: int, block_size: int): + self.num_blocks_per_task = num_blocks_per_task self.block_size = block_size def estimate_inmemory_data_size(self): @@ -28,15 +34,15 @@ def estimate_inmemory_data_size(self): def get_read_tasks(self, parallelism: int): def _blocks_generator(): - for _ in range(self.num_blocks): + for _ in range(self.num_blocks_per_task): yield pd.DataFrame({"one": [np.random.bytes(self.block_size)]}) return parallelism * [ ReadTask( lambda: _blocks_generator(), BlockMetadata( - num_rows=self.num_blocks, - size_bytes=self.num_blocks * self.block_size, + num_rows=self.num_blocks_per_task, + size_bytes=self.num_blocks_per_task * self.block_size, schema=None, input_files=None, exec_stats=None, @@ -45,6 +51,46 @@ def _blocks_generator(): ] +class SlowCSVDatasource(CSVDatasource): + def _read_stream(self, f: "pa.NativeFile", path: str, **reader_args): + for block in CSVDatasource._read_stream(self, f, path, **reader_args): + time.sleep(3) + yield block + + +# Tests that we don't block on exponential rampup when doing bulk reads. +# https://github.com/ray-project/ray/issues/20625 +@pytest.mark.parametrize("block_split", [False, True]) +def test_bulk_lazy_eval_split_mode(shutdown_only, block_split, tmp_path): + # Defensively shutdown Ray for the first test here to make sure there + # is no existing Ray cluster. + ray.shutdown() + + ray.init(num_cpus=8) + ctx = ray.data.context.DatasetContext.get_current() + + try: + original = ctx.block_splitting_enabled + + ray.data.range(8, parallelism=8).write_csv(str(tmp_path)) + if not block_split: + # Setting infinite block size effectively disables block splitting. + ctx.target_max_block_size = float("inf") + ds = ray.data.read_datasource( + SlowCSVDatasource(), parallelism=8, paths=str(tmp_path) + ) + + start = time.time() + ds.map(lambda x: x) + delta = time.time() - start + + print("full read time", delta) + # Should run in ~3 seconds. It takes >9 seconds if bulk read is broken. + assert delta < 8, delta + finally: + ctx.block_splitting_enabled = original + + def test_enable_in_ray_client(ray_start_cluster_enabled): cluster = ray_start_cluster_enabled cluster.add_node(num_cpus=4) @@ -82,65 +128,70 @@ def test_enable_in_ray_client(ray_start_cluster_enabled): ], ) def test_dataset( - ray_start_regular_shared, + shutdown_only, enable_dynamic_block_splitting, target_max_block_size, compute, ): - # Test 10 blocks from 10 tasks, each block is 1024 bytes. - num_blocks = 10 + ray.shutdown() + # We need at least 2 CPUs to run a actorpool streaming + ray.init(num_cpus=2) + # Test 10 tasks, each task returning 10 blocks, each block has 1 row and each + # row has 1024 bytes. + num_blocks_per_task = 10 block_size = 1024 num_tasks = 10 ds = ray.data.read_datasource( RandomBytesDatasource(), parallelism=num_tasks, - num_blocks=num_blocks, + num_blocks_per_task=num_blocks_per_task, block_size=block_size, ) + # Note the following calls to ds will not fully execute it. assert ds.schema() is not None - assert ds.count() == num_blocks * num_tasks + assert ds.count() == num_blocks_per_task * num_tasks assert ds.num_blocks() == num_tasks - assert ds.size_bytes() >= 0.7 * block_size * num_blocks * num_tasks + assert ds.size_bytes() >= 0.7 * block_size * num_blocks_per_task * num_tasks map_ds = ds.map_batches(lambda x: x, compute=compute) map_ds.fully_executed() assert map_ds.num_blocks() == num_tasks map_ds = ds.map_batches( - lambda x: x, batch_size=num_blocks * num_tasks, compute=compute + lambda x: x, batch_size=num_blocks_per_task * num_tasks, compute=compute ) map_ds.fully_executed() assert map_ds.num_blocks() == 1 map_ds = ds.map(lambda x: x, compute=compute) map_ds.fully_executed() - assert map_ds.num_blocks() == num_blocks * num_tasks + assert map_ds.num_blocks() == num_blocks_per_task * num_tasks ds_list = ds.split(5) assert len(ds_list) == 5 for new_ds in ds_list: - assert new_ds.num_blocks() == num_blocks * num_tasks / 5 + assert new_ds.num_blocks() == num_blocks_per_task * num_tasks / 5 train, test = ds.train_test_split(test_size=0.25) - assert train.num_blocks() == num_blocks * num_tasks * 0.75 - assert test.num_blocks() == num_blocks * num_tasks * 0.25 + assert train.num_blocks() == num_blocks_per_task * num_tasks * 0.75 + assert test.num_blocks() == num_blocks_per_task * num_tasks * 0.25 new_ds = ds.union(ds, ds) assert new_ds.num_blocks() == num_tasks * 3 new_ds.fully_executed() - assert new_ds.num_blocks() == num_blocks * num_tasks * 3 + assert new_ds.num_blocks() == num_blocks_per_task * num_tasks * 3 new_ds = ds.random_shuffle() assert new_ds.num_blocks() == num_tasks new_ds = ds.randomize_block_order() assert new_ds.num_blocks() == num_tasks - assert ds.groupby("one").count().count() == num_blocks * num_tasks + assert ds.groupby("one").count().count() == num_blocks_per_task * num_tasks new_ds = ds.zip(ds) new_ds.fully_executed() - assert new_ds.num_blocks() == num_blocks * num_tasks + assert new_ds.num_blocks() == num_blocks_per_task * num_tasks assert len(ds.take(5)) == 5 - assert len(ds.take_all()) == num_blocks * num_tasks + assert len(ds.take_all()) == num_blocks_per_task * num_tasks for batch in ds.iter_batches(batch_size=10): assert len(batch) == 10 @@ -148,15 +199,16 @@ def test_dataset( def test_dataset_pipeline( ray_start_regular_shared, enable_dynamic_block_splitting, target_max_block_size ): - # Test 10 blocks from 10 tasks, each block is 1024 bytes. - num_blocks = 10 + # Test 10 tasks, each task returning 10 blocks, each block has 1 row and each + # row has 1024 bytes. + num_blocks_per_task = 10 block_size = 1024 num_tasks = 10 ds = ray.data.read_datasource( RandomBytesDatasource(), parallelism=num_tasks, - num_blocks=num_blocks, + num_blocks_per_task=num_blocks_per_task, block_size=block_size, ) dsp = ds.window(blocks_per_window=2) @@ -166,7 +218,7 @@ def test_dataset_pipeline( result_batches = list(ds.iter_batches(batch_size=5)) for batch in result_batches: assert len(batch) == 5 - assert len(result_batches) == num_blocks * num_tasks / 5 + assert len(result_batches) == num_blocks_per_task * num_tasks / 5 dsp = ds.window(blocks_per_window=2) assert dsp._length == num_tasks / 2 @@ -178,40 +230,42 @@ def test_dataset_pipeline( def test_filter( ray_start_regular_shared, enable_dynamic_block_splitting, target_max_block_size ): - # Test 10 blocks from 1 task, each block is 1024 bytes. - num_blocks = 10 + # Test 10 tasks, each task returning 10 blocks, each block has 1 row and each + # row has 1024 bytes. + num_blocks_per_task = 10 block_size = 1024 ds = ray.data.read_datasource( RandomBytesDatasource(), parallelism=1, - num_blocks=num_blocks, + num_blocks_per_task=num_blocks_per_task, block_size=block_size, ) ds = ds.filter(lambda _: True) ds.fully_executed() - assert ds.count() == num_blocks - assert ds.num_blocks() == num_blocks + assert ds.count() == num_blocks_per_task + assert ds.num_blocks() == num_blocks_per_task ds = ds.filter(lambda _: False) ds.fully_executed() assert ds.count() == 0 - assert ds.num_blocks() == num_blocks + assert ds.num_blocks() == num_blocks_per_task def test_lazy_block_list( shutdown_only, enable_dynamic_block_splitting, target_max_block_size ): - # Test 10 blocks from 10 tasks, each block is 1024 bytes. - num_blocks = 10 + # Test 10 tasks, each task returning 10 blocks, each block has 1 row and each + # row has 1024 bytes. + num_blocks_per_task = 10 block_size = 1024 num_tasks = 10 ds = ray.data.read_datasource( RandomBytesDatasource(), parallelism=num_tasks, - num_blocks=num_blocks, + num_blocks_per_task=num_blocks_per_task, block_size=block_size, ) ds.schema() @@ -231,18 +285,18 @@ def test_lazy_block_list( assert len(cached_metadata) == num_tasks for i, block_metadata in enumerate(cached_metadata): if i == 0: - assert len(block_metadata) == num_blocks + assert len(block_metadata) == num_blocks_per_task for m in block_metadata: assert m.num_rows == 1 else: assert block_metadata is None - assert len(metadata) == num_tasks - 1 + num_blocks + assert len(metadata) == num_tasks - 1 + num_blocks_per_task for i, block_metadata in enumerate(metadata): - if i < num_blocks: + if i < num_blocks_per_task: assert block_metadata.num_rows == 1 assert block_metadata.schema is not None else: - assert block_metadata.num_rows == num_blocks + assert block_metadata.num_rows == num_blocks_per_task assert block_metadata.schema is None # Check APIs of LazyBlockList @@ -255,12 +309,12 @@ def test_lazy_block_list( assert len(block_lists[0]._block_partition_refs) == 2 assert len(block_lists[0]._cached_metadata) == 2 - block_lists = block_list.split_by_bytes(block_size * num_blocks * 2) + block_lists = block_list.split_by_bytes(block_size * num_blocks_per_task * 2) assert len(block_lists) == num_tasks / 2 assert len(block_lists[0]._block_partition_refs) == 2 assert len(block_lists[0]._cached_metadata) == 2 - new_block_list = block_list.truncate_by_rows(num_blocks * 3) + new_block_list = block_list.truncate_by_rows(num_blocks_per_task * 3) assert len(new_block_list._block_partition_refs) == 3 assert len(new_block_list._cached_metadata) == 3 @@ -275,7 +329,7 @@ def test_lazy_block_list( assert len(new_block_list._cached_metadata) == num_tasks output_blocks = block_list.get_blocks_with_metadata() - assert len(output_blocks) == num_tasks * num_blocks + assert len(output_blocks) == num_tasks * num_blocks_per_task for _, metadata in output_blocks: assert metadata.num_rows == 1 for _, metadata in block_list.iter_blocks_with_metadata(): @@ -291,10 +345,10 @@ def test_lazy_block_list( assert all(map(lambda ref: ref is None, block_list._block_partition_meta_refs)) assert len(cached_metadata) == num_tasks for block_metadata in cached_metadata: - assert len(block_metadata) == num_blocks + assert len(block_metadata) == num_blocks_per_task for m in block_metadata: assert m.num_rows == 1 - assert len(metadata) == num_tasks * num_blocks + assert len(metadata) == num_tasks * num_blocks_per_task for block_metadata in metadata: assert block_metadata.num_rows == 1 assert block_metadata.schema is not None @@ -302,7 +356,7 @@ def test_lazy_block_list( def test_read_large_data(ray_start_cluster, enable_dynamic_block_splitting): # Test 20G input with single task - num_blocks = 20 + num_blocks_per_task = 20 block_size = 1024 * 1024 * 1024 cluster = ray_start_cluster @@ -316,12 +370,12 @@ def foo(batch): ds = ray.data.read_datasource( RandomBytesDatasource(), parallelism=1, - num_blocks=num_blocks, + num_blocks_per_task=num_blocks_per_task, block_size=block_size, ) ds = ds.map_batches(foo, batch_size=None) - assert ds.count() == num_blocks + assert ds.count() == num_blocks_per_task if __name__ == "__main__": diff --git a/python/ray/data/tests/test_execution_optimizer.py b/python/ray/data/tests/test_execution_optimizer.py index e47ee3a16be5d..fe903813149a2 100644 --- a/python/ray/data/tests/test_execution_optimizer.py +++ b/python/ray/data/tests/test_execution_optimizer.py @@ -1,8 +1,10 @@ +import itertools import pytest import ray from ray.data._internal.execution.operators.map_operator import MapOperator from ray.data._internal.execution.operators.all_to_all_operator import AllToAllOperator +from ray.data._internal.execution.operators.zip_operator import ZipOperator from ray.data._internal.execution.operators.input_data_buffer import InputDataBuffer from ray.data._internal.logical.interfaces import LogicalPlan from ray.data._internal.logical.optimizers import PhysicalOptimizer @@ -20,6 +22,7 @@ Filter, FlatMap, ) +from ray.data._internal.logical.operators.n_ary_operator import Zip from ray.data._internal.planner.planner import Planner from ray.data.aggregate import Count from ray.data.datasource.parquet_datasource import ParquetDatasource @@ -594,6 +597,33 @@ def test_aggregate_e2e( assert row.as_pydict() == {"value": idx, "count()": 1} +def test_zip_operator(ray_start_regular_shared, enable_optimizer): + planner = Planner() + read_op1 = Read(ParquetDatasource()) + read_op2 = Read(ParquetDatasource()) + op = Zip(read_op1, read_op2) + plan = LogicalPlan(op) + physical_op = planner.plan(plan).dag + + assert op.name == "Zip" + assert isinstance(physical_op, ZipOperator) + assert len(physical_op.input_dependencies) == 2 + assert isinstance(physical_op.input_dependencies[0], MapOperator) + assert isinstance(physical_op.input_dependencies[1], MapOperator) + + +@pytest.mark.parametrize( + "num_blocks1,num_blocks2", + list(itertools.combinations_with_replacement(range(1, 12), 2)), +) +def test_zip_e2e(ray_start_regular_shared, enable_optimizer, num_blocks1, num_blocks2): + n = 12 + ds1 = ray.data.range(n, parallelism=num_blocks1) + ds2 = ray.data.range(n, parallelism=num_blocks2).map(lambda x: x + 1) + ds = ds1.zip(ds2) + assert ds.take() == list(zip(range(n), range(1, n + 1))) + + def test_streaming_executor( ray_start_regular_shared, enable_optimizer, diff --git a/python/ray/data/tests/test_metadata_provider.py b/python/ray/data/tests/test_metadata_provider.py index b9d0f9ef3713d..f8d7ab6c79688 100644 --- a/python/ray/data/tests/test_metadata_provider.py +++ b/python/ray/data/tests/test_metadata_provider.py @@ -1,14 +1,26 @@ +from functools import partial +import logging +import os import pytest import posixpath +from unittest.mock import patch import urllib.parse -import os -import logging import pyarrow as pa +from pyarrow.fs import LocalFileSystem import pandas as pd import pyarrow.parquet as pq from pytest_lazyfixture import lazy_fixture -from ray.data.datasource.file_based_datasource import _resolve_paths_and_filesystem +from ray.data.datasource.file_based_datasource import ( + FILE_SIZE_FETCH_PARALLELIZATION_THRESHOLD, + _resolve_paths_and_filesystem, + _unwrap_protocol, +) +from ray.data.datasource.file_meta_provider import ( + _get_file_infos_serial, + _get_file_infos_common_path_prefix, + _get_file_infos_parallel, +) from ray.tests.conftest import * # noqa from ray.data.datasource import ( @@ -18,11 +30,16 @@ DefaultFileMetadataProvider, DefaultParquetMetadataProvider, FastFileMetadataProvider, + PathPartitionEncoder, ) from ray.data.tests.conftest import * # noqa +def df_to_csv(dataframe, path, **kwargs): + dataframe.to_csv(path, **kwargs) + + def _get_parquet_file_meta_size_bytes(file_metas): return sum( sum(m.row_group(i).total_byte_size for i in range(m.num_row_groups)) @@ -125,7 +142,9 @@ def test_default_parquet_metadata_provider(fs, data_path): ), ], ) -def test_default_file_metadata_provider(caplog, fs, data_path, endpoint_url): +def test_default_file_metadata_provider( + propagate_logs, caplog, fs, data_path, endpoint_url +): storage_options = ( {} if endpoint_url is None @@ -144,8 +163,12 @@ def test_default_file_metadata_provider(caplog, fs, data_path, endpoint_url): df2.to_csv(path2, index=False, storage_options=storage_options) meta_provider = DefaultFileMetadataProvider() - with caplog.at_level(logging.WARNING): - file_paths, file_sizes = meta_provider.expand_paths(paths, fs) + with caplog.at_level(logging.WARNING), patch( + "ray.data.datasource.file_meta_provider._get_file_infos_serial", + wraps=_get_file_infos_serial, + ) as mock_get: + file_paths, file_sizes = map(list, zip(*meta_provider.expand_paths(paths, fs))) + mock_get.assert_called_once_with(paths, fs) assert "meta_provider=FastFileMetadataProvider()" in caplog.text assert file_paths == paths expected_file_sizes = _get_file_sizes_bytes(paths, fs) @@ -164,6 +187,195 @@ def test_default_file_metadata_provider(caplog, fs, data_path, endpoint_url): assert meta.schema is None +@pytest.mark.parametrize( + "fs,data_path,endpoint_url", + [ + (lazy_fixture("local_fs"), lazy_fixture("local_path"), None), + (lazy_fixture("s3_fs"), lazy_fixture("s3_path"), lazy_fixture("s3_server")), + ], +) +def test_default_file_metadata_provider_many_files_basic( + propagate_logs, + caplog, + fs, + data_path, + endpoint_url, +): + if endpoint_url is None: + storage_options = {} + else: + storage_options = dict(client_kwargs=dict(endpoint_url=endpoint_url)) + + paths = [] + dfs = [] + num_dfs = 4 * FILE_SIZE_FETCH_PARALLELIZATION_THRESHOLD + for i in range(num_dfs): + df = pd.DataFrame({"one": list(range(i * 3, (i + 1) * 3))}) + dfs.append(df) + path = os.path.join(data_path, f"test_{i}.csv") + paths.append(path) + df.to_csv(path, index=False, storage_options=storage_options) + paths, fs = _resolve_paths_and_filesystem(paths, fs) + + meta_provider = DefaultFileMetadataProvider() + if isinstance(fs, LocalFileSystem): + patcher = patch( + "ray.data.datasource.file_meta_provider._get_file_infos_serial", + wraps=_get_file_infos_serial, + ) + else: + patcher = patch( + "ray.data.datasource.file_meta_provider._get_file_infos_common_path_prefix", + wraps=_get_file_infos_common_path_prefix, + ) + with caplog.at_level(logging.WARNING), patcher as mock_get: + file_paths, file_sizes = map(list, zip(*meta_provider.expand_paths(paths, fs))) + if isinstance(fs, LocalFileSystem): + mock_get.assert_called_once_with(paths, fs) + else: + mock_get.assert_called_once_with(paths, _unwrap_protocol(data_path), fs) + assert "meta_provider=FastFileMetadataProvider()" in caplog.text + assert file_paths == paths + expected_file_sizes = _get_file_sizes_bytes(paths, fs) + assert file_sizes == expected_file_sizes + + +@pytest.mark.parametrize( + "fs,data_path,endpoint_url", + [ + (lazy_fixture("local_fs"), lazy_fixture("local_path"), None), + (lazy_fixture("s3_fs"), lazy_fixture("s3_path"), lazy_fixture("s3_server")), + ], +) +def test_default_file_metadata_provider_many_files_partitioned( + propagate_logs, + caplog, + fs, + data_path, + endpoint_url, + write_partitioned_df, + assert_base_partitioned_ds, +): + if endpoint_url is None: + storage_options = {} + else: + storage_options = dict(client_kwargs=dict(endpoint_url=endpoint_url)) + + partition_keys = ["one"] + partition_path_encoder = PathPartitionEncoder.of( + base_dir=data_path, + field_names=partition_keys, + filesystem=fs, + ) + paths = [] + dfs = [] + num_dfs = FILE_SIZE_FETCH_PARALLELIZATION_THRESHOLD + for i in range(num_dfs): + df = pd.DataFrame( + {"one": [1, 1, 1, 3, 3, 3], "two": list(range(6 * i, 6 * (i + 1)))} + ) + df_paths = write_partitioned_df( + df, + partition_keys, + partition_path_encoder, + partial(df_to_csv, storage_options=storage_options, index=False), + file_name_suffix=i, + ) + dfs.append(df) + paths.extend(df_paths) + paths, fs = _resolve_paths_and_filesystem(paths, fs) + partitioning = partition_path_encoder.scheme + + meta_provider = DefaultFileMetadataProvider() + if isinstance(fs, LocalFileSystem): + patcher = patch( + "ray.data.datasource.file_meta_provider._get_file_infos_serial", + wraps=_get_file_infos_serial, + ) + else: + patcher = patch( + "ray.data.datasource.file_meta_provider._get_file_infos_common_path_prefix", + wraps=_get_file_infos_common_path_prefix, + ) + with caplog.at_level(logging.WARNING), patcher as mock_get: + file_paths, file_sizes = map( + list, zip(*meta_provider.expand_paths(paths, fs, partitioning)) + ) + if isinstance(fs, LocalFileSystem): + mock_get.assert_called_once_with(paths, fs) + else: + mock_get.assert_called_once_with( + paths, + _unwrap_protocol(partitioning.base_dir), + fs, + ) + assert "meta_provider=FastFileMetadataProvider()" in caplog.text + assert file_paths == paths + expected_file_sizes = _get_file_sizes_bytes(paths, fs) + assert file_sizes == expected_file_sizes + + +@pytest.mark.parametrize( + "fs,data_path,endpoint_url", + [ + (lazy_fixture("local_fs"), lazy_fixture("local_path"), None), + (lazy_fixture("s3_fs"), lazy_fixture("s3_path"), lazy_fixture("s3_server")), + ], +) +def test_default_file_metadata_provider_many_files_diff_dirs( + ray_start_regular, + propagate_logs, + caplog, + fs, + data_path, + endpoint_url, +): + if endpoint_url is None: + storage_options = {} + else: + storage_options = dict(client_kwargs=dict(endpoint_url=endpoint_url)) + + dir1 = os.path.join(data_path, "dir1") + dir2 = os.path.join(data_path, "dir2") + if fs is None: + os.mkdir(dir1) + os.mkdir(dir2) + else: + fs.create_dir(_unwrap_protocol(dir1)) + fs.create_dir(_unwrap_protocol(dir2)) + + paths = [] + dfs = [] + num_dfs = 2 * FILE_SIZE_FETCH_PARALLELIZATION_THRESHOLD + for i, dir_path in enumerate([dir1, dir2]): + for j in range(num_dfs * i, num_dfs * (i + 1)): + df = pd.DataFrame({"one": list(range(3 * j, 3 * (j + 1)))}) + dfs.append(df) + path = os.path.join(dir_path, f"test_{j}.csv") + paths.append(path) + df.to_csv(path, index=False, storage_options=storage_options) + paths, fs = _resolve_paths_and_filesystem(paths, fs) + + meta_provider = DefaultFileMetadataProvider() + if isinstance(fs, LocalFileSystem): + patcher = patch( + "ray.data.datasource.file_meta_provider._get_file_infos_serial", + wraps=_get_file_infos_serial, + ) + else: + patcher = patch( + "ray.data.datasource.file_meta_provider._get_file_infos_parallel", + wraps=_get_file_infos_parallel, + ) + with caplog.at_level(logging.WARNING), patcher as mock_get: + file_paths, file_sizes = map(list, zip(*meta_provider.expand_paths(paths, fs))) + mock_get.assert_called_once_with(paths, fs) + assert "meta_provider=FastFileMetadataProvider()" in caplog.text + assert file_paths == paths + expected_file_sizes = _get_file_sizes_bytes(paths, fs) + assert file_sizes == expected_file_sizes + + @pytest.mark.parametrize( "fs,data_path,endpoint_url", [ @@ -182,7 +394,9 @@ def test_default_file_metadata_provider(caplog, fs, data_path, endpoint_url): ), ], ) -def test_fast_file_metadata_provider(caplog, fs, data_path, endpoint_url): +def test_fast_file_metadata_provider( + propagate_logs, caplog, fs, data_path, endpoint_url +): storage_options = ( {} if endpoint_url is None @@ -202,7 +416,7 @@ def test_fast_file_metadata_provider(caplog, fs, data_path, endpoint_url): meta_provider = FastFileMetadataProvider() with caplog.at_level(logging.WARNING): - file_paths, file_sizes = meta_provider.expand_paths(paths, fs) + file_paths, file_sizes = map(list, zip(*meta_provider.expand_paths(paths, fs))) assert "meta_provider=DefaultFileMetadataProvider()" in caplog.text assert file_paths == paths assert len(file_sizes) == len(file_paths) diff --git a/python/ray/data/tests/test_mongo_dataset.py b/python/ray/data/tests/test_mongo_dataset.py index d87d83421003f..92a7f1735fa75 100644 --- a/python/ray/data/tests/test_mongo_dataset.py +++ b/python/ray/data/tests/test_mongo_dataset.py @@ -78,8 +78,11 @@ def test_read_write_mongo(ray_start_regular_shared, start_mongo): ) assert ds._block_num_rows() == [3, 2] assert str(ds) == ( - "Dataset(num_blocks=2, num_rows=5, " - "schema={float_field: double, int_field: int32})" + "Dataset(\n" + " num_blocks=2,\n" + " num_rows=5,\n" + " schema={float_field: double, int_field: int32}\n" + ")" ) assert df.equals(ds.to_pandas()) @@ -93,8 +96,12 @@ def test_read_write_mongo(ray_start_regular_shared, start_mongo): ) assert ds._block_num_rows() == [3, 2] assert str(ds) == ( - "Dataset(num_blocks=2, num_rows=5, schema={_id: fixed_size_binary[12], " - "float_field: double, int_field: int32})" + "Dataset(\n" + " num_blocks=2,\n" + " num_rows=5,\n" + " schema={_id: fixed_size_binary[12], float_field: double, " + "int_field: int32}\n" + ")" ) assert df.equals(ds.drop_columns(["_id"]).to_pandas()) @@ -108,8 +115,12 @@ def test_read_write_mongo(ray_start_regular_shared, start_mongo): ) assert ds._block_num_rows() == [2, 1] assert str(ds) == ( - "Dataset(num_blocks=2, num_rows=3, schema={_id: fixed_size_binary[12], " - "float_field: double, int_field: int32})" + "Dataset(\n" + " num_blocks=2,\n" + " num_rows=3,\n" + " schema={_id: fixed_size_binary[12], float_field: double, " + "int_field: int32}\n" + ")" ) df[df["int_field"] < 3].equals(ds.drop_columns(["_id"]).to_pandas()) @@ -120,8 +131,12 @@ def test_read_write_mongo(ray_start_regular_shared, start_mongo): collection=foo_collection, ) assert str(ds) == ( - "Dataset(num_blocks=5, num_rows=5, schema={_id: fixed_size_binary[12], " - "float_field: double, int_field: int32})" + "Dataset(\n" + " num_blocks=5,\n" + " num_rows=5,\n" + " schema={_id: fixed_size_binary[12], float_field: double, " + "int_field: int32}\n" + ")" ) assert df.equals(ds.drop_columns(["_id"]).to_pandas()) @@ -133,8 +148,12 @@ def test_read_write_mongo(ray_start_regular_shared, start_mongo): parallelism=1000, ) assert str(ds) == ( - "Dataset(num_blocks=5, num_rows=5, schema={_id: fixed_size_binary[12], " - "float_field: double, int_field: int32})" + "Dataset(\n" + " num_blocks=5,\n" + " num_rows=5,\n" + " schema={_id: fixed_size_binary[12], float_field: double, " + "int_field: int32}\n" + ")" ) assert df.equals(ds.drop_columns(["_id"]).to_pandas()) @@ -192,8 +211,11 @@ def test_mongo_datasource(ray_start_regular_shared, start_mongo): ).fully_executed() assert ds._block_num_rows() == [3, 2] assert str(ds) == ( - "Dataset(num_blocks=2, num_rows=5, " - "schema={float_field: double, int_field: int32})" + "Dataset(\n" + " num_blocks=2,\n" + " num_rows=5,\n" + " schema={float_field: double, int_field: int32}\n" + ")" ) assert df.equals(ds.to_pandas()) @@ -208,8 +230,12 @@ def test_mongo_datasource(ray_start_regular_shared, start_mongo): ).fully_executed() assert ds._block_num_rows() == [3, 2] assert str(ds) == ( - "Dataset(num_blocks=2, num_rows=5, " - "schema={_id: fixed_size_binary[12], float_field: double, int_field: int32})" + "Dataset(\n" + " num_blocks=2,\n" + " num_rows=5,\n" + " schema={_id: fixed_size_binary[12], float_field: double, " + "int_field: int32}\n" + ")" ) assert df.equals(ds.drop_columns(["_id"]).to_pandas()) @@ -221,8 +247,12 @@ def test_mongo_datasource(ray_start_regular_shared, start_mongo): collection=foo_collection, ).fully_executed() assert str(ds) == ( - "Dataset(num_blocks=5, num_rows=5, schema={_id: fixed_size_binary[12], " - "float_field: double, int_field: int32})" + "Dataset(\n" + " num_blocks=5,\n" + " num_rows=5,\n" + " schema={_id: fixed_size_binary[12], float_field: double, " + "int_field: int32}\n" + ")" ) assert df.equals(ds.drop_columns(["_id"]).to_pandas()) @@ -235,8 +265,12 @@ def test_mongo_datasource(ray_start_regular_shared, start_mongo): collection=foo_collection, ) assert str(ds) == ( - "Dataset(num_blocks=5, num_rows=5, schema={_id: fixed_size_binary[12], " - "float_field: double, int_field: int32})" + "Dataset(\n" + " num_blocks=5,\n" + " num_rows=5,\n" + " schema={_id: fixed_size_binary[12], float_field: double, " + "int_field: int32}\n" + ")" ) assert df.equals(ds.drop_columns(["_id"]).to_pandas()) @@ -251,8 +285,12 @@ def test_mongo_datasource(ray_start_regular_shared, start_mongo): ) assert ds._block_num_rows() == [2, 1] assert str(ds) == ( - "Dataset(num_blocks=2, num_rows=3, schema={_id: fixed_size_binary[12], " - "float_field: double, int_field: int32})" + "Dataset(\n" + " num_blocks=2,\n" + " num_rows=3,\n" + " schema={_id: fixed_size_binary[12], float_field: double, " + "int_field: int32}\n" + ")" ) df[df["int_field"] < 3].equals(ds.drop_columns(["_id"]).to_pandas()) diff --git a/python/ray/data/tests/test_object_gc.py b/python/ray/data/tests/test_object_gc.py index 67c627ce3afc4..95697649d5c62 100644 --- a/python/ray/data/tests/test_object_gc.py +++ b/python/ray/data/tests/test_object_gc.py @@ -1,5 +1,3 @@ -import time - import pytest import ray @@ -7,12 +5,13 @@ from ray.tests.conftest import * # noqa -def check_no_spill(ctx, pipe, prefetch_blocks: int = 0): - # Run .iter_batches() for 10 secs, and we expect no object spilling. - end_time = time.time() + 10 - for batch in pipe.iter_batches(batch_size=None, prefetch_blocks=prefetch_blocks): - if time.time() > end_time: - break +def check_no_spill(ctx, pipe): + # Run up to 10 epochs of the pipeline to stress test that + # no spilling will happen. + max_epoch = 10 + for p in pipe.iter_epochs(max_epoch): + for _ in p.iter_batches(batch_size=None): + pass meminfo = memory_summary(ctx.address_info["address"], stats_only=True) assert "Spilled" not in meminfo, meminfo @@ -24,10 +23,7 @@ def test_iter_batches_no_spilling_upon_no_transformation(shutdown_only): ds = ray.data.range_tensor(500, shape=(80, 80, 4), parallelism=100) check_no_spill(ctx, ds.repeat()) - check_no_spill(ctx, ds.repeat(), 5) - check_no_spill(ctx, ds.window(blocks_per_window=20)) - check_no_spill(ctx, ds.window(blocks_per_window=20), 5) def test_iter_batches_no_spilling_upon_rewindow(shutdown_only): @@ -39,9 +35,6 @@ def test_iter_batches_no_spilling_upon_rewindow(shutdown_only): check_no_spill( ctx, ds.window(blocks_per_window=20).repeat().rewindow(blocks_per_window=10) ) - check_no_spill( - ctx, ds.window(blocks_per_window=20).repeat().rewindow(blocks_per_window=10), 5 - ) def test_iter_batches_no_spilling_upon_prior_transformation(shutdown_only): @@ -52,11 +45,8 @@ def test_iter_batches_no_spilling_upon_prior_transformation(shutdown_only): # Repeat, with transformation prior to the pipeline. check_no_spill(ctx, ds.map_batches(lambda x: x).repeat()) - check_no_spill(ctx, ds.map_batches(lambda x: x).repeat(), 5) - # Window, with transformation prior to the pipeline. check_no_spill(ctx, ds.map_batches(lambda x: x).window(blocks_per_window=20)) - check_no_spill(ctx, ds.map_batches(lambda x: x).window(blocks_per_window=20), 5) def test_iter_batches_no_spilling_upon_post_transformation(shutdown_only): @@ -67,11 +57,8 @@ def test_iter_batches_no_spilling_upon_post_transformation(shutdown_only): # Repeat, with transformation post the pipeline creation. check_no_spill(ctx, ds.repeat().map_batches(lambda x: x, batch_size=5)) - check_no_spill(ctx, ds.repeat().map_batches(lambda x: x, batch_size=5), 5) - # Window, with transformation post the pipeline creation. check_no_spill(ctx, ds.window(blocks_per_window=20).map_batches(lambda x: x)) - check_no_spill(ctx, ds.window(blocks_per_window=20).map_batches(lambda x: x), 5) def test_iter_batches_no_spilling_upon_transformations(shutdown_only): @@ -87,14 +74,6 @@ def test_iter_batches_no_spilling_upon_transformations(shutdown_only): .repeat() .map_batches(lambda x: x, batch_size=5), ) - check_no_spill( - ctx, - ds.map_batches(lambda x: x, batch_size=5) - .repeat() - .map_batches(lambda x: x, batch_size=5), - 5, - ) - # Window, with transformation before and post the pipeline. check_no_spill( ctx, @@ -102,13 +81,6 @@ def test_iter_batches_no_spilling_upon_transformations(shutdown_only): .window(blocks_per_window=20) .map_batches(lambda x: x), ) - check_no_spill( - ctx, - ds.map_batches(lambda x: x) - .window(blocks_per_window=20) - .map_batches(lambda x: x), - 5, - ) def test_iter_batches_no_spilling_upon_shuffle(shutdown_only): @@ -118,10 +90,7 @@ def test_iter_batches_no_spilling_upon_shuffle(shutdown_only): ds = ray.data.range_tensor(500, shape=(80, 80, 4), parallelism=100) check_no_spill(ctx, ds.repeat().random_shuffle_each_window()) - check_no_spill(ctx, ds.repeat().random_shuffle_each_window(), 5) - check_no_spill(ctx, ds.window(blocks_per_window=20).random_shuffle_each_window()) - check_no_spill(ctx, ds.window(blocks_per_window=20).random_shuffle_each_window(), 5) def test_pipeline_splitting_has_no_spilling(shutdown_only): diff --git a/python/ray/data/tests/test_operators.py b/python/ray/data/tests/test_operators.py index 104e5d4223155..0fb9b8813d516 100644 --- a/python/ray/data/tests/test_operators.py +++ b/python/ray/data/tests/test_operators.py @@ -261,6 +261,48 @@ def test_split_operator_random(ray_start_regular_shared, equal, random_seed): assert sum(len(output_splits[i]) for i in range(3)) == num_inputs, output_splits +def test_split_operator_locality_hints(ray_start_regular_shared): + input_op = InputDataBuffer(make_ref_bundles([[i] for i in range(10)])) + op = OutputSplitter(input_op, 2, equal=False, locality_hints=["node1", "node2"]) + + def get_fake_loc(item): + if item in [0, 1, 4, 5, 8]: + return "node1" + else: + return "node2" + + def get_bundle_loc(bundle): + return get_fake_loc(ray.get(bundle.blocks[0][0])[0]) + + op._get_location = get_bundle_loc + + # Feed data and implement streaming exec. + output_splits = collections.defaultdict(list) + op.start(ExecutionOptions()) + while input_op.has_next(): + op.add_input(input_op.get_next(), 0) + op.inputs_done() + while op.has_next(): + ref = op.get_next() + assert ref.owns_blocks, ref + for block, _ in ref.blocks: + output_splits[ref.output_split_idx].extend(ray.get(block)) + + total = 0 + for i in range(2): + if i == 0: + node = "node1" + else: + node = "node2" + split = output_splits[i] + for item in split: + assert get_fake_loc(item) == node + total += 1 + + assert total == 10, total + assert "10 locality hits, 0 misses" in op.progress_str() + + def test_map_operator_actor_locality_stats(ray_start_regular_shared): # Create with inputs. input_op = InputDataBuffer(make_ref_bundles([[i] for i in range(100)])) diff --git a/python/ray/data/tests/test_optimize.py b/python/ray/data/tests/test_optimize.py index 1b74355edad81..27b12f8a4845b 100644 --- a/python/ray/data/tests/test_optimize.py +++ b/python/ray/data/tests/test_optimize.py @@ -612,7 +612,7 @@ def __call__(self, x): pipe, 1, [ - "ReadParquetBulk->MapBatches(CallableFn)->MapBatches(CallableFn)", + "ReadParquet->MapBatches(CallableFn)->MapBatches(CallableFn)", ], ) @@ -648,7 +648,7 @@ def __call__(self, x): pipe, 1, [ - "ReadParquetBulk->MapBatches()->MapBatches(CallableFn)", + "ReadParquet->MapBatches()->MapBatches(CallableFn)", ], ) diff --git a/python/ray/data/tests/test_raydp_dataset.py b/python/ray/data/tests/test_raydp_dataset.py index b759fc65ee148..ca9ae69dc7377 100644 --- a/python/ray/data/tests/test_raydp_dataset.py +++ b/python/ray/data/tests/test_raydp_dataset.py @@ -4,6 +4,7 @@ import torch +# RayDP tests require Ray Java. Make sure ray jar is built before running this test. @pytest.fixture(scope="function") def spark(request): ray.init(num_cpus=2, include_dashboard=False) diff --git a/python/ray/data/tests/test_split.py b/python/ray/data/tests/test_split.py index 8d3949d20c742..9c31835481536 100644 --- a/python/ray/data/tests/test_split.py +++ b/python/ray/data/tests/test_split.py @@ -779,6 +779,31 @@ def test_train_test_split(ray_start_regular_shared): ds.train_test_split(test_size=9) +def test_split_is_not_disruptive(ray_start_cluster): + ray.shutdown() + ds = ray.data.range(100, parallelism=10).map_batches(lambda x: x).lazy() + + def verify_integrity(splits): + for dss in splits: + for batch in dss.iter_batches(): + pass + for batch in ds.iter_batches(): + pass + + # No block splitting invovled: split 10 even blocks into 2 groups. + verify_integrity(ds.split(2, equal=True)) + # Block splitting invovled: split 10 even blocks into 3 groups. + verify_integrity(ds.split(3, equal=True)) + + # Same as above but having tranforms post converting to lazy. + verify_integrity(ds.map_batches(lambda x: x).split(2, equal=True)) + verify_integrity(ds.map_batches(lambda x: x).split(3, equal=True)) + + # Same as above but having in-place tranforms post converting to lazy. + verify_integrity(ds.randomize_block_order().split(2, equal=True)) + verify_integrity(ds.randomize_block_order().split(3, equal=True)) + + if __name__ == "__main__": import sys diff --git a/python/ray/data/tests/test_stats.py b/python/ray/data/tests/test_stats.py index 8b458bc88e287..6fcb22b4259e0 100644 --- a/python/ray/data/tests/test_stats.py +++ b/python/ray/data/tests/test_stats.py @@ -5,8 +5,9 @@ import pytest import ray -from ray.data._internal.stats import DatasetStats +from ray.data._internal.stats import _StatsActor, DatasetStats from ray.data._internal.dataset_logger import DatasetLogger +from ray.data.block import BlockMetadata from ray.data.context import DatasetContext from ray.tests.conftest import * # noqa @@ -160,6 +161,9 @@ def test_dataset_stats_basic(ray_start_regular_shared, enable_auto_log_stats): Dataset iterator time breakdown: * In ray.wait(): T * In ray.get(): T +* Num blocks local: Z +* Num blocks remote: Z +* Num blocks unknown location: N * In next_batch(): T * In format_batch(): T * In user code: T @@ -288,7 +292,7 @@ def test_dataset_stats_read_parquet(ray_start_regular_shared, tmp_path): if context.new_execution_backend: assert ( stats - == """Stage N ReadParquetBulk->map: N/N blocks executed in T + == """Stage N ReadParquet->map: N/N blocks executed in T * Remote wall time: T min, T max, T mean, T total * Remote cpu time: T min, T max, T mean, T total * Peak heap memory usage (MiB): N min, N max, N mean @@ -912,6 +916,9 @@ def test_streaming_stats_full(ray_start_regular_shared, restore_dataset_context) Dataset iterator time breakdown: * In ray.wait(): T * In ray.get(): T +* Num blocks local: Z +* Num blocks remote: Z +* Num blocks unknown location: N * In next_batch(): T * In format_batch(): T * In user code: T @@ -920,6 +927,47 @@ def test_streaming_stats_full(ray_start_regular_shared, restore_dataset_context) ) +# NOTE: All tests above share a Ray cluster, while the tests below do not. These +# tests should only be carefully reordered to retain this invariant! + + +def test_stats_actor_cap_num_stats(ray_start_cluster): + actor = _StatsActor.remote(3) + metadatas = [] + task_idx = 0 + for uuid in range(3): + metadatas.append( + BlockMetadata( + num_rows=uuid, + size_bytes=None, + schema=None, + input_files=None, + exec_stats=None, + ) + ) + num_stats = uuid + 1 + actor.record_start.remote(uuid) + assert ray.get(actor._get_stats_dict_size.remote()) == ( + num_stats, + num_stats - 1, + num_stats - 1, + ) + actor.record_task.remote(uuid, task_idx, [metadatas[-1]]) + assert ray.get(actor._get_stats_dict_size.remote()) == ( + num_stats, + num_stats, + num_stats, + ) + for uuid in range(3): + assert ray.get(actor.get.remote(uuid))[0][task_idx] == [metadatas[uuid]] + # Add the fourth stats to exceed the limit. + actor.record_start.remote(3) + # The first stats (with uuid=0) should have been purged. + assert ray.get(actor.get.remote(0))[0] == {} + # The start_time has 3 entries because we just added it above with record_start(). + assert ray.get(actor._get_stats_dict_size.remote()) == (3, 2, 2) + + if __name__ == "__main__": import sys diff --git a/python/ray/data/tests/test_streaming_executor.py b/python/ray/data/tests/test_streaming_executor.py index c593bce813978..cf1f9f6d5eff4 100644 --- a/python/ray/data/tests/test_streaming_executor.py +++ b/python/ray/data/tests/test_streaming_executor.py @@ -125,7 +125,7 @@ def test_select_operator_to_run(): o3.num_active_work_refs = MagicMock(return_value=2) o3.internal_queue_size = MagicMock(return_value=0) assert select_operator_to_run(topo, NO_USAGE, ExecutionResources(), True) == o2 - # nternal queue size is added to num active tasks. + # Internal queue size is added to num active tasks. o3.num_active_work_refs = MagicMock(return_value=0) o3.internal_queue_size = MagicMock(return_value=2) assert select_operator_to_run(topo, NO_USAGE, ExecutionResources(), True) == o2 @@ -136,6 +136,10 @@ def test_select_operator_to_run(): o2.internal_queue_size = MagicMock(return_value=2) assert select_operator_to_run(topo, NO_USAGE, ExecutionResources(), True) == o3 + # Test prioritization of nothrottle ops. + o2.throttling_disabled = MagicMock(return_value=True) + assert select_operator_to_run(topo, NO_USAGE, ExecutionResources(), True) == o2 + def test_dispatch_next_task(): inputs = make_ref_bundles([[x] for x in range(20)]) @@ -302,17 +306,36 @@ def test_configure_output_locality(): o2, compute_strategy=ray.data.ActorPoolStrategy(1, 1), ) + # No locality. build_streaming_topology(o3, ExecutionOptions(locality_with_output=False)) assert o2._ray_remote_args.get("scheduling_strategy") is None assert o3._ray_remote_args.get("scheduling_strategy") == "SPREAD" + + # Current node locality. build_streaming_topology(o3, ExecutionOptions(locality_with_output=True)) - assert isinstance( - o2._ray_remote_args["scheduling_strategy"], NodeAffinitySchedulingStrategy - ) - assert isinstance( - o3._ray_remote_args["scheduling_strategy"], - NodeAffinitySchedulingStrategy, + s1 = o2._get_runtime_ray_remote_args()["scheduling_strategy"] + assert isinstance(s1, NodeAffinitySchedulingStrategy) + assert s1.node_id == ray.get_runtime_context().get_node_id() + s2 = o3._get_runtime_ray_remote_args()["scheduling_strategy"] + assert isinstance(s2, NodeAffinitySchedulingStrategy) + assert s2.node_id == ray.get_runtime_context().get_node_id() + + # Multi node locality. + build_streaming_topology( + o3, ExecutionOptions(locality_with_output=["node1", "node2"]) ) + s1a = o2._get_runtime_ray_remote_args()["scheduling_strategy"] + s1b = o2._get_runtime_ray_remote_args()["scheduling_strategy"] + s1c = o2._get_runtime_ray_remote_args()["scheduling_strategy"] + assert s1a.node_id == "node1" + assert s1b.node_id == "node2" + assert s1c.node_id == "node1" + s2a = o3._get_runtime_ray_remote_args()["scheduling_strategy"] + s2b = o3._get_runtime_ray_remote_args()["scheduling_strategy"] + s2c = o3._get_runtime_ray_remote_args()["scheduling_strategy"] + assert s2a.node_id == "node1" + assert s2b.node_id == "node2" + assert s2c.node_id == "node1" def test_calculate_topology_usage(): @@ -381,6 +404,31 @@ def test_execution_allowed_downstream_aware_memory_throttling(): ) +def test_execution_allowed_nothrottle(): + op = InputDataBuffer([]) + op.incremental_resource_usage = MagicMock(return_value=ExecutionResources()) + # Above global. + assert not _execution_allowed( + op, + TopologyResourceUsage( + ExecutionResources(object_store_memory=1000), + {op: DownstreamMemoryInfo(1, 1000)}, + ), + ExecutionResources(object_store_memory=900), + ) + + # Throttling disabled. + op.throttling_disabled = MagicMock(return_value=True) + assert _execution_allowed( + op, + TopologyResourceUsage( + ExecutionResources(object_store_memory=1000), + {op: DownstreamMemoryInfo(1, 1000)}, + ), + ExecutionResources(object_store_memory=900), + ) + + if __name__ == "__main__": import sys diff --git a/python/ray/data/tests/test_streaming_integration.py b/python/ray/data/tests/test_streaming_integration.py index 0520f1409be90..69b8ae72c5770 100644 --- a/python/ray/data/tests/test_streaming_integration.py +++ b/python/ray/data/tests/test_streaming_integration.py @@ -1,4 +1,5 @@ import pytest +import threading import time from typing import List, Any @@ -16,6 +17,7 @@ from ray.data._internal.execution.operators.all_to_all_operator import AllToAllOperator from ray.data._internal.execution.operators.map_operator import MapOperator from ray.data._internal.execution.operators.input_data_buffer import InputDataBuffer +from ray.data._internal.execution.operators.output_splitter import OutputSplitter from ray.data._internal.execution.util import make_ref_bundles from ray._private.test_utils import wait_for_condition from ray.data.tests.conftest import * # noqa @@ -55,6 +57,82 @@ def reverse_sort(inputs: List[RefBundle], ctx): assert output == expected, (output, expected) +def test_output_split_e2e(ray_start_10_cpus_shared): + executor = StreamingExecutor(ExecutionOptions()) + inputs = make_ref_bundles([[x] for x in range(20)]) + o1 = InputDataBuffer(inputs) + o2 = OutputSplitter(o1, 2, equal=True) + it = executor.execute(o2) + + class Consume(threading.Thread): + def __init__(self, idx): + self.idx = idx + self.out = [] + super().__init__() + + def run(self): + while True: + try: + self.out.append(it.get_next(output_split_idx=self.idx)) + except Exception as e: + print(e) + raise + + c0 = Consume(0) + c1 = Consume(1) + c0.start() + c1.start() + c0.join() + c1.join() + assert len(c0.out) == 10, c0.out + assert len(c1.out) == 10, c0.out + + +def test_streaming_split_e2e(ray_start_10_cpus_shared): + def get_lengths(*iterators): + lengths = [] + for it in iterators: + x = 0 + for batch in it.iter_batches(): + x += len(batch) + lengths.append(x) + lengths.sort() + return lengths + + ds = ray.data.range(1000) + ( + i1, + i2, + ) = ds.streaming_split(2, equal=True) + lengths = get_lengths(i1, i2) + assert lengths == [500, 500], lengths + + ds = ray.data.range(1) + ( + i1, + i2, + ) = ds.streaming_split(2, equal=True) + lengths = get_lengths(i1, i2) + assert lengths == [0, 0], lengths + + ds = ray.data.range(1) + ( + i1, + i2, + ) = ds.streaming_split(2, equal=False) + lengths = get_lengths(i1, i2) + assert lengths == [0, 1], lengths + + ds = ray.data.range(1000, parallelism=10) + i1, i2, i3 = ds.streaming_split(3, equal=True) + lengths = get_lengths(i1, i2, i3) + assert lengths == [333, 333, 333], lengths + + i1, i2, i3 = ds.streaming_split(3, equal=False) + lengths = get_lengths(i1, i2, i3) + assert lengths == [300, 300, 400], lengths + + def test_e2e_option_propagation(ray_start_10_cpus_shared, restore_dataset_context): DatasetContext.get_current().new_execution_backend = True DatasetContext.get_current().use_streaming_executor = True diff --git a/python/ray/data/tests/test_transform_pyarrow.py b/python/ray/data/tests/test_transform_pyarrow.py index 68b5a91594a0f..3ff1e29d32632 100644 --- a/python/ray/data/tests/test_transform_pyarrow.py +++ b/python/ray/data/tests/test_transform_pyarrow.py @@ -1,7 +1,12 @@ +import os + import numpy as np +import pandas as pd import pyarrow as pa import pytest +import ray +from ray.data.block import BlockAccessor from ray.data.extensions import ( ArrowTensorArray, ArrowTensorType, @@ -286,6 +291,109 @@ def test_unify_schemas(): ) +def test_arrow_block_select(): + df = pd.DataFrame({"one": [10, 11, 12], "two": [11, 12, 13], "three": [14, 15, 16]}) + table = pa.Table.from_pandas(df) + block_accessor = BlockAccessor.for_block(table) + + block = block_accessor.select(["two"]) + assert block.schema == pa.schema([("two", pa.int64())]) + assert block.to_pandas().equals(df[["two"]]) + + block = block_accessor.select(["two", "one"]) + assert block.schema == pa.schema([("two", pa.int64()), ("one", pa.int64())]) + assert block.to_pandas().equals(df[["two", "one"]]) + + with pytest.raises(ValueError): + block = block_accessor.select([lambda x: x % 3, "two"]) + + +def test_arrow_block_slice_copy(): + # Test that ArrowBlock slicing properly copies the underlying Arrow + # table. + def check_for_copy(table1, table2, a, b, is_copy): + expected_slice = table1.slice(a, b - a) + assert table2.equals(expected_slice) + assert table2.schema == table1.schema + assert table1.num_columns == table2.num_columns + for col1, col2 in zip(table1.columns, table2.columns): + assert col1.num_chunks == col2.num_chunks + for chunk1, chunk2 in zip(col1.chunks, col2.chunks): + bufs1 = chunk1.buffers() + bufs2 = chunk2.buffers() + expected_offset = 0 if is_copy else a + assert chunk2.offset == expected_offset + assert len(chunk2) == b - a + if is_copy: + assert bufs2[1].address != bufs1[1].address + else: + assert bufs2[1].address == bufs1[1].address + + n = 20 + df = pd.DataFrame( + {"one": list(range(n)), "two": ["a"] * n, "three": [np.nan] + [1.5] * (n - 1)} + ) + table = pa.Table.from_pandas(df) + a, b = 5, 10 + block_accessor = BlockAccessor.for_block(table) + + # Test with copy. + table2 = block_accessor.slice(a, b, True) + check_for_copy(table, table2, a, b, is_copy=True) + + # Test without copy. + table2 = block_accessor.slice(a, b, False) + check_for_copy(table, table2, a, b, is_copy=False) + + +def test_arrow_block_slice_copy_empty(): + # Test that ArrowBlock slicing properly copies the underlying Arrow + # table when the table is empty. + df = pd.DataFrame({"one": []}) + table = pa.Table.from_pandas(df) + a, b = 0, 0 + expected_slice = table.slice(a, b - a) + block_accessor = BlockAccessor.for_block(table) + + # Test with copy. + table2 = block_accessor.slice(a, b, True) + assert table2.equals(expected_slice) + assert table2.schema == table.schema + assert table2.num_rows == 0 + + # Test without copy. + table2 = block_accessor.slice(a, b, False) + assert table2.equals(expected_slice) + assert table2.schema == table.schema + assert table2.num_rows == 0 + + +def test_convert_to_pyarrow(ray_start_regular_shared, tmp_path): + ds = ray.data.range(100) + assert ds.to_dask().sum().compute()[0] == 4950 + path = os.path.join(tmp_path, "test_parquet_dir") + os.mkdir(path) + ds.write_parquet(path) + assert ray.data.read_parquet(path).count() == 100 + + +def test_pyarrow(ray_start_regular_shared): + ds = ray.data.range_table(5) + assert ds.map(lambda x: {"b": x["value"] + 2}).take() == [ + {"b": 2}, + {"b": 3}, + {"b": 4}, + {"b": 5}, + {"b": 6}, + ] + assert ds.map(lambda x: {"b": x["value"] + 2}).filter( + lambda x: x["b"] % 2 == 0 + ).take() == [{"b": 2}, {"b": 4}, {"b": 6}] + assert ds.filter(lambda x: x["value"] == 0).flat_map( + lambda x: [{"b": x["value"] + 2}, {"b": x["value"] + 20}] + ).take() == [{"b": 2}, {"b": 20}] + + if __name__ == "__main__": import sys diff --git a/python/ray/experimental/state/common.py b/python/ray/experimental/state/common.py index 4042312a735fd..c6fc8a8eaf0a4 100644 --- a/python/ray/experimental/state/common.py +++ b/python/ray/experimental/state/common.py @@ -564,6 +564,8 @@ class TaskState(StateSchema): start_time_ms: Optional[int] = state_column(detail=True, filterable=False) #: The time when the task is finished or failed. A Unix timestamp in ms. end_time_ms: Optional[int] = state_column(detail=True, filterable=False) + #: Task error type. + error_type: Optional[str] = state_column(detail=False, filterable=False) @dataclass(init=True) diff --git a/python/ray/serve/_private/common.py b/python/ray/serve/_private/common.py index 69f0f570c7691..c14493b488d38 100644 --- a/python/ray/serve/_private/common.py +++ b/python/ray/serve/_private/common.py @@ -311,7 +311,18 @@ def __post_init__(self): ] ) ) + + # RunningReplicaInfo class set frozen=True, this is the hacky way to set + # new attribute for the class. object.__setattr__(self, "_hash", hash_val) def __hash__(self): return self._hash + + def __eq__(self, other): + return all( + [ + isinstance(other, RunningReplicaInfo), + self._hash == other._hash, + ] + ) diff --git a/python/ray/serve/schema.py b/python/ray/serve/schema.py index a5672fa4cca9a..1e3d9703df5e8 100644 --- a/python/ray/serve/schema.py +++ b/python/ray/serve/schema.py @@ -251,7 +251,7 @@ class ServeApplicationSchema(BaseModel, extra=Extra.forbid): "Application name, the name should be unique within the serve instance" ), ) - route_prefix: str = Field( + route_prefix: Optional[str] = Field( default="/", description=( "Route prefix for HTTP requests. If not provided, it will use" @@ -260,7 +260,6 @@ class ServeApplicationSchema(BaseModel, extra=Extra.forbid): ), ) import_path: str = Field( - default=None, description=( "An import path to a bound deployment node. Should be of the " 'form "module.submodule_1...submodule_n.' @@ -477,6 +476,35 @@ class ServeDeploySchema(BaseModel, extra=Extra.forbid): description=("The set of Serve applications to run on the Ray cluster."), ) + @validator("applications") + def application_names_unique(cls, v): + # Ensure there are no duplicate applications listed + names = [app.name for app in v] + duplicates = {f'"{name}"' for name in names if names.count(name) > 1} + if len(duplicates): + apps_str = ("application " if len(duplicates) == 1 else "applications ") + ( + ", ".join(duplicates) + ) + raise ValueError( + f"Found multiple configs for {apps_str}. Please remove all duplicates." + ) + return v + + @validator("applications") + def application_routes_unique(cls, v): + # Ensure each application with a non-null route prefix has unique route prefixes + routes = [app.route_prefix for app in v if app.route_prefix is not None] + duplicates = {f'"{route}"' for route in routes if routes.count(route) > 1} + if len(duplicates): + routes_str = ( + "route prefix " if len(duplicates) == 1 else "route prefixes " + ) + (", ".join(duplicates)) + raise ValueError( + f"Found duplicate applications for {routes_str}. Please ensure each " + "application's route_prefix is unique." + ) + return v + @staticmethod def get_empty_schema_dict() -> Dict: """Returns an empty deploy schema dictionary. diff --git a/python/ray/serve/tests/test_router.py b/python/ray/serve/tests/test_router.py index 80520a064a848..422a245acfd7c 100644 --- a/python/ray/serve/tests/test_router.py +++ b/python/ray/serve/tests/test_router.py @@ -3,6 +3,7 @@ controller or the actual replica wrapper, use mock if necessary. """ import asyncio +import copy import pytest @@ -118,6 +119,23 @@ async def num_queries(self): await asyncio.sleep(0.2) assert not third_ref_pending_task.done() + # Let's make sure in flight queries is 1 for each replica. + assert len(rs.in_flight_queries[replicas[0]]) == 1 + assert len(rs.in_flight_queries[replicas[1]]) == 1 + + # Let's copy a new RunningReplicaInfo object and update the router + cur_replicas_info = list(rs.in_flight_queries.keys()) + replicas = copy.deepcopy(cur_replicas_info) + assert id(replicas[0].actor_handle) != id(cur_replicas_info[0].actor_handle) + assert replicas[0].replica_tag == cur_replicas_info[0].replica_tag + assert id(replicas[1].actor_handle) != id(cur_replicas_info[1].actor_handle) + assert replicas[1].replica_tag == cur_replicas_info[1].replica_tag + rs.update_running_replicas(replicas) + + # Let's make sure in flight queries is 1 for each replica even if replicas update + assert len(rs.in_flight_queries[replicas[0]]) == 1 + assert len(rs.in_flight_queries[replicas[1]]) == 1 + # Let's unblock the two replicas await signal.send.remote() assert await first_ref == "DONE" diff --git a/python/ray/serve/tests/test_schema.py b/python/ray/serve/tests/test_schema.py index 86bde26856f48..05da6629b8cae 100644 --- a/python/ray/serve/tests/test_schema.py +++ b/python/ray/serve/tests/test_schema.py @@ -18,6 +18,7 @@ DeploymentSchema, ServeApplicationSchema, ServeStatusSchema, + ServeDeploySchema, serve_status_to_schema, ) from ray.util.accelerators.accelerators import NVIDIA_TESLA_V100, NVIDIA_TESLA_P4 @@ -617,6 +618,11 @@ def test_remove_app_name_from_deployment_names(self): with pytest.raises(AssertionError): config.remove_app_name_from_deployment_names() + def test_serve_application_import_path_required(self): + # If no import path is specified, this should not parse successfully + with pytest.raises(ValidationError): + ServeApplicationSchema.parse_obj({"host": "127.0.0.1", "port": 8000}) + class TestServeDeploySchema: def test_serve_application_to_deploy_config(self): @@ -647,6 +653,90 @@ def test_serve_application_to_deploy_config(self): "applications": [app_config_dict], } + def test_deploy_config_duplicate_apps(self): + deploy_config_dict = { + "host": "127.0.0.1", + "port": 8000, + "applications": [ + { + "name": "app1", + "route_prefix": "/alice", + "import_path": "module.graph", + }, + { + "name": "app2", + "route_prefix": "/charlie", + "import_path": "module.graph", + }, + ], + } + ServeDeploySchema.parse_obj(deploy_config_dict) + + # Duplicate app1 + deploy_config_dict["applications"].append( + {"name": "app1", "route_prefix": "/bob", "import_path": "module.graph"}, + ) + with pytest.raises(ValidationError) as e: + ServeDeploySchema.parse_obj(deploy_config_dict) + assert "app1" in str(e.value) and "app2" not in str(e.value) + + # Duplicate app2 + deploy_config_dict["applications"].append( + {"name": "app2", "route_prefix": "/david", "import_path": "module.graph"} + ) + with pytest.raises(ValidationError) as e: + ServeDeploySchema.parse_obj(deploy_config_dict) + assert "app1" in str(e.value) and "app2" in str(e.value) + + def test_deploy_config_duplicate_routes1(self): + """Test that apps with duplicate route prefixes raises validation error""" + deploy_config_dict = { + "host": "127.0.0.1", + "port": 8000, + "applications": [ + { + "name": "app1", + "route_prefix": "/alice", + "import_path": "module.graph", + }, + {"name": "app2", "route_prefix": "/bob", "import_path": "module.graph"}, + ], + } + ServeDeploySchema.parse_obj(deploy_config_dict) + + # Duplicate route prefix /alice + deploy_config_dict["applications"].append( + {"name": "app3", "route_prefix": "/alice", "import_path": "module.graph"}, + ) + with pytest.raises(ValidationError) as e: + ServeDeploySchema.parse_obj(deploy_config_dict) + assert "alice" in str(e.value) and "bob" not in str(e.value) + + # Duplicate route prefix /bob + deploy_config_dict["applications"].append( + {"name": "app4", "route_prefix": "/bob", "import_path": "module.graph"}, + ) + with pytest.raises(ValidationError) as e: + ServeDeploySchema.parse_obj(deploy_config_dict) + assert "alice" in str(e.value) and "bob" in str(e.value) + + def test_deploy_config_duplicate_routes2(self): + """Test that multiple apps with route_prefix set to None parses with no error""" + deploy_config_dict = { + "host": "127.0.0.1", + "port": 8000, + "applications": [ + { + "name": "app1", + "route_prefix": "/app1", + "import_path": "module.graph", + }, + {"name": "app2", "route_prefix": None, "import_path": "module.graph"}, + {"name": "app3", "route_prefix": None, "import_path": "module.graph"}, + ], + } + ServeDeploySchema.parse_obj(deploy_config_dict) + class TestServeStatusSchema: def get_valid_serve_status_schema(self): diff --git a/python/ray/serve/tests/test_standalone2.py b/python/ray/serve/tests/test_standalone2.py index 94136b23b3f83..96ea3b21f6585 100644 --- a/python/ray/serve/tests/test_standalone2.py +++ b/python/ray/serve/tests/test_standalone2.py @@ -1204,6 +1204,72 @@ async def waiter(*args): serve.shutdown() +@pytest.mark.parametrize( + "ray_instance", + [ + { + "LISTEN_FOR_CHANGE_REQUEST_TIMEOUT_S_LOWER_BOUND": "1", + "LISTEN_FOR_CHANGE_REQUEST_TIMEOUT_S_UPPER_BOUND": "2", + }, + ], + indirect=True, +) +def test_long_poll_timeout_with_max_concurrent_queries(ray_instance): + """Test max_concurrent_queries can be honorded with long poll timeout + + issue: https://github.com/ray-project/ray/issues/32652 + """ + + signal_actor = SignalActor.remote() + + @serve.deployment(max_concurrent_queries=1) + async def f(): + await signal_actor.wait.remote() + return "hello" + + handle = serve.run(f.bind()) + first_ref = handle.remote() + + # Clear all the internal longpoll client objects within handle + # long poll client will receive new updates from long poll host, + # this is to simulate the longpoll timeout + object_snapshots1 = handle.router.long_poll_client.object_snapshots + handle.router.long_poll_client._reset() + wait_for_condition( + lambda: len(handle.router.long_poll_client.object_snapshots) > 0, timeout=10 + ) + object_snapshots2 = handle.router.long_poll_client.object_snapshots + + # Check object snapshots between timeout interval + assert object_snapshots1.keys() == object_snapshots2.keys() + assert len(object_snapshots1.keys()) == 1 + key = list(object_snapshots1.keys())[0] + assert ( + object_snapshots1[key][0].actor_handle != object_snapshots2[key][0].actor_handle + ) + assert ( + object_snapshots1[key][0].actor_handle._actor_id + == object_snapshots2[key][0].actor_handle._actor_id + ) + + # Make sure the inflight queries still one + assert len(handle.router._replica_set.in_flight_queries) == 1 + key = list(handle.router._replica_set.in_flight_queries.keys())[0] + assert len(handle.router._replica_set.in_flight_queries[key]) == 1 + + # Make sure the first request is being run. + replicas = list(handle.router._replica_set.in_flight_queries.keys()) + assert len(handle.router._replica_set.in_flight_queries[replicas[0]]) == 1 + # First ref should be still ongoing + with pytest.raises(ray.exceptions.GetTimeoutError): + ray.get(first_ref, timeout=1) + # Unblock the first request. + signal_actor.send.remote() + assert ray.get(first_ref) == "hello" + + serve.shutdown() + + def test_shutdown_remote(start_and_shutdown_ray_cli_function): """Check that serve.shutdown() works on a remote Ray cluster.""" diff --git a/python/ray/tests/spark/test_utils.py b/python/ray/tests/spark/test_utils.py index d9c7e570483f6..864e71ca48327 100644 --- a/python/ray/tests/spark/test_utils.py +++ b/python/ray/tests/spark/test_utils.py @@ -8,6 +8,7 @@ _calc_mem_per_ray_worker_node, _get_avail_mem_per_ray_worker_node, ) +from ray.util.spark.cluster_init import _convert_ray_node_options pytestmark = pytest.mark.skipif( not sys.platform.startswith("linux"), @@ -86,6 +87,16 @@ def test_get_avail_mem_per_ray_worker_node(monkeypatch): ) == (280000, 120000, None, None) +def test_convert_ray_node_options(): + assert _convert_ray_node_options( + { + "cluster_name": "aBc", + "disable_usage_stats": None, + "include_dashboard": False, + } + ) == ["--cluster-name=aBc", "--disable-usage-stats", "--include-dashboard=False"] + + if __name__ == "__main__": if os.environ.get("PARALLEL_CI"): sys.exit(pytest.main(["-n", "auto", "--boxed", "-vs", __file__])) diff --git a/python/ray/tests/test_failure_4.py b/python/ray/tests/test_failure_4.py index ab1394b3ac6aa..70e1c292847b9 100644 --- a/python/ray/tests/test_failure_4.py +++ b/python/ray/tests/test_failure_4.py @@ -7,6 +7,7 @@ import psutil import pytest from grpc._channel import _InactiveRpcError +from ray._private.state_api_test_utils import verify_failed_task import ray import ray._private.ray_constants as ray_constants @@ -555,13 +556,20 @@ def func(): # The lease request should wait inside raylet # since there is no available resources. - ret = func.remote() + ret = func.options(name="task-local-raylet-dead").remote() # Waiting for the lease request to reach raylet. time.sleep(1) head.kill_raylet() with pytest.raises(LocalRayletDiedError): ray.get(ret) + # Check the task failure states for observability. + wait_for_condition( + verify_failed_task, + name="task-local-raylet-dead", + error_type="LOCAL_RAYLET_DIED", + ) + def test_locality_aware_scheduling_for_dead_nodes(shutdown_only): """Test that locality-ware scheduling can handle dead nodes.""" diff --git a/python/ray/tests/test_memory_pressure.py b/python/ray/tests/test_memory_pressure.py index 9b3eb86430237..0148d2cd727ad 100644 --- a/python/ray/tests/test_memory_pressure.py +++ b/python/ray/tests/test_memory_pressure.py @@ -14,6 +14,7 @@ import numpy as np from ray._private.utils import get_system_memory from ray._private.utils import get_used_memory +from ray.experimental.state.api import list_tasks from ray.experimental.state.state_manager import StateDataSourceClient @@ -339,7 +340,7 @@ async def test_task_oom_logs_error(ray_with_memory_monitor): bytes_to_alloc = get_additional_bytes_to_reach_memory_usage_pct(1) with pytest.raises(ray.exceptions.OutOfMemoryError) as _: ray.get( - allocate_memory.options(max_retries=0).remote( + allocate_memory.options(max_retries=0, name="allocate_memory").remote( allocate_bytes=bytes_to_alloc, allocate_interval_s=0, post_allocate_sleep_s=1000, @@ -355,8 +356,16 @@ async def test_task_oom_logs_error(ray_with_memory_monitor): verified = True assert verified - # TODO(clarng): verify task info once state_api_client.get_task_info - # returns the crashed task. + def verify_oom_task_error(): + tasks = list_tasks(filters=[("name", "=", "allocate_memory")]) + print(tasks) + assert len(tasks) == 1, "no retries should be expected." + assert tasks[0]["state"] == "FAILED" + assert tasks[0]["error_type"] == "OUT_OF_MEMORY" + return True + + wait_for_condition(verify_oom_task_error) + # TODO(clarng): verify log info once state api can dump log info diff --git a/python/ray/tests/test_multiprocessing.py b/python/ray/tests/test_multiprocessing.py index 07051b1ef3659..7a82f39933f5c 100644 --- a/python/ray/tests/test_multiprocessing.py +++ b/python/ray/tests/test_multiprocessing.py @@ -6,7 +6,6 @@ import time import random from collections import defaultdict -import warnings import queue import math @@ -509,26 +508,17 @@ def f(args): result_iter.next() -@pytest.mark.filterwarnings( - "default:Passing a non-iterable argument:ray.util.annotations.RayDeprecationWarning" -) -def test_warn_on_non_iterable_imap_or_imap_unordered(pool): +def test_imap_fail_on_non_iterable(pool): def fn(_): pass non_iterable = 3 - with warnings.catch_warnings(record=True) as w: + with pytest.raises(TypeError, match="object is not iterable"): pool.imap(fn, non_iterable) - assert any( - "Passing a non-iterable argument" in str(warning.message) for warning in w - ) - with warnings.catch_warnings(record=True) as w: + with pytest.raises(TypeError, match="object is not iterable"): pool.imap_unordered(fn, non_iterable) - assert any( - "Passing a non-iterable argument" in str(warning.message) for warning in w - ) @pytest.mark.parametrize("use_iter", [True, False]) diff --git a/python/ray/tests/test_task_events.py b/python/ray/tests/test_task_events.py index 6311c9111ac38..cb3e88c1876a5 100644 --- a/python/ray/tests/test_task_events.py +++ b/python/ray/tests/test_task_events.py @@ -3,7 +3,9 @@ import pytest import threading import time - +from ray._private.state_api_test_utils import verify_failed_task +from ray.exceptions import RuntimeEnvSetupError +from ray.runtime_env import RuntimeEnv import ray from ray.experimental.state.common import ListApiOptions, StateResource from ray._private.test_utils import ( @@ -82,6 +84,172 @@ def verify(): ) +def test_failed_task_error(shutdown_only): + ray.init(_system_config=_SYSTEM_CONFIG) + + # Test failed task with TASK_EXECUTION_EXCEPTION + @ray.remote + def fail(x=None): + if x is not None: + time.sleep(x) + raise ValueError("fail is expected to failed") + + with pytest.raises(ray.exceptions.RayTaskError): + ray.get(fail.options(name="fail").remote()) + + wait_for_condition( + verify_failed_task, name="fail", error_type="TASK_EXECUTION_EXCEPTION" + ) + + # Test canceled tasks with TASK_CANCELLED + @ray.remote + def sleep(): + time.sleep(999) + + with pytest.raises(ray.exceptions.TaskCancelledError): + t = sleep.options(name="sleep-cancel").remote() + ray.cancel(t) + ray.get(t) + + wait_for_condition( + verify_failed_task, name="sleep-cancel", error_type="TASK_CANCELLED" + ) + + # Test task failed when worker killed :WORKER_DIED + @ray.remote(max_retries=0) + def die(): + exit(1) + + with pytest.raises(ray.exceptions.WorkerCrashedError): + ray.get(die.options(name="die-worker").remote()) + + wait_for_condition(verify_failed_task, name="die-worker", error_type="WORKER_DIED") + + # Test actor task failed with actor dead: ACTOR_DIED + @ray.remote + class Actor: + def f(self): + time.sleep(999) + + a = Actor.remote() + with pytest.raises(ray.exceptions.RayActorError): + ray.kill(a) + ray.get(a.f.options(name="actor-killed").remote()) + + wait_for_condition(verify_failed_task, name="actor-killed", error_type="ACTOR_DIED") + + +def test_failed_task_failed_due_to_node_failure(ray_start_cluster): + cluster = ray_start_cluster + cluster.add_node(num_cpus=1) + ray.init(address=cluster.address) + node = cluster.add_node(num_cpus=2) + + driver_script = """ +import ray +ray.init("auto") + +@ray.remote(num_cpus=2, max_retries=0) +def sleep(): + import time + time.sleep(999) + +x = sleep.options(name="node-killed").remote() +ray.get(x) + """ + + run_string_as_driver_nonblocking(driver_script) + + def driver_running(): + t = list_tasks(filters=[("name", "=", "node-killed")]) + return len(t) > 0 + + wait_for_condition(driver_running) + + # Kill the node + cluster.remove_node(node) + + wait_for_condition(verify_failed_task, name="node-killed", error_type="NODE_DIED") + + +def test_failed_task_unschedulable(shutdown_only): + ray.init(num_cpus=1, _system_config=_SYSTEM_CONFIG) + + node_id = ray.get_runtime_context().get_node_id() + policy = ray.util.scheduling_strategies.NodeAffinitySchedulingStrategy( + node_id=node_id, + soft=False, + ) + + @ray.remote + def task(): + pass + + task.options( + scheduling_strategy=policy, + name="task-unschedulable", + num_cpus=2, + ).remote() + + wait_for_condition( + verify_failed_task, + name="task-unschedulable", + error_type="TASK_UNSCHEDULABLE_ERROR", + ) + + +def test_failed_task_removed_placement_group(shutdown_only, monkeypatch): + ray.init(num_cpus=2, _system_config=_SYSTEM_CONFIG) + from ray.util.placement_group import placement_group, remove_placement_group + from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy + + pg = placement_group([{"CPU": 2}]) + ray.get(pg.ready()) + + @ray.remote(num_cpus=2) + def sleep(): + time.sleep(999) + + with monkeypatch.context() as m: + m.setenv( + "RAY_testing_asio_delay_us", + "NodeManagerService.grpc_server.RequestWorkerLease=3000000:3000000", + ) + + sleep.options( + scheduling_strategy=PlacementGroupSchedulingStrategy(placement_group=pg), + name="task-pg-removed", + max_retries=0, + ).remote() + + remove_placement_group(pg) + + wait_for_condition( + verify_failed_task, + name="task-pg-removed", + error_type="TASK_PLACEMENT_GROUP_REMOVED", + ) + + +def test_failed_task_runtime_env_setup(shutdown_only): + @ray.remote + def f(): + pass + + bad_env = RuntimeEnv(conda={"dependencies": ["_this_does_not_exist"]}) + with pytest.raises( + RuntimeEnvSetupError, + match="ResolvePackageNotFound", + ): + ray.get(f.options(runtime_env=bad_env, name="task-runtime-env-failed").remote()) + + wait_for_condition( + verify_failed_task, + name="task-runtime-env-failed", + error_type="RUNTIME_ENV_SETUP_FAILED", + ) + + def test_fault_tolerance_parent_failed(shutdown_only): ray.init(num_cpus=4, _system_config=_SYSTEM_CONFIG) diff --git a/python/ray/train/_internal/backend_executor.py b/python/ray/train/_internal/backend_executor.py index 6cebbc9331f70..6cc713fd67e0e 100644 --- a/python/ray/train/_internal/backend_executor.py +++ b/python/ray/train/_internal/backend_executor.py @@ -111,6 +111,15 @@ def start( actor_cls_kwargs=train_cls_kwargs, placement_group=placement_group, ) + # Hack to avoid OOMs. + # This is just a temporary solution for Train loading entire checkpoints + # into memory by ensuring that the rank 0 worker is on the same node as + # trainable, thus allowing for lazy checkpoint transfer to be used. + # See https://github.com/ray-project/ray/issues/33073 + # for more context. + # TODO remove + if self._trial_info and self._trial_info.driver_ip: + self.worker_group._move_workers_with_ip_to_front(self._trial_info.driver_ip) try: if initialization_hook: self._initialization_hook = initialization_hook diff --git a/python/ray/train/_internal/worker_group.py b/python/ray/train/_internal/worker_group.py index 07bdff2807011..ea4ee58be129d 100644 --- a/python/ray/train/_internal/worker_group.py +++ b/python/ray/train/_internal/worker_group.py @@ -345,5 +345,26 @@ def add_workers(self, num_workers: int): for i in range(len(new_actors)): self.workers.append(Worker(actor=new_actors[i], metadata=metadata[i])) + def _move_workers_with_ip_to_front(self, ip): + # Hack to avoid OOMs. + # This is just a temporary solution for Train loading entire checkpoints + # into memory by ensuring that the rank 0 worker is on the same node as + # trainable, thus allowing for lazy checkpoint transfer to be used. + # See https://github.com/ray-project/ray/issues/33073 + # for more context. + # TODO remove + workers_with_ip = [] + indices_to_remove = set() + for i, worker in enumerate(self.workers): + if worker.metadata.node_ip == ip: + workers_with_ip.append(worker) + indices_to_remove.add(i) + if workers_with_ip: + self.workers = workers_with_ip + [ + worker + for i, worker in enumerate(self.workers) + if i not in indices_to_remove + ] + def __len__(self): return len(self.workers) diff --git a/python/ray/train/batch_predictor.py b/python/ray/train/batch_predictor.py index ea1219f0fac97..6b0457919e56d 100644 --- a/python/ray/train/batch_predictor.py +++ b/python/ray/train/batch_predictor.py @@ -36,6 +36,7 @@ def __init__( self._predictor_cls = predictor_cls self._predictor_kwargs = predictor_kwargs self._override_preprocessor: Optional[Preprocessor] = None + self._override_preprocessor_set = False def __repr__(self): return ( @@ -98,7 +99,7 @@ def _predict_pandas(self, df, **kwargs) -> "pd.DataFrame": def get_preprocessor(self) -> Preprocessor: """Get the preprocessor to use prior to executing predictions.""" - if self._override_preprocessor: + if self._override_preprocessor_set: return self._override_preprocessor return self._checkpoint.get_preprocessor() @@ -106,6 +107,7 @@ def get_preprocessor(self) -> Preprocessor: def set_preprocessor(self, preprocessor: Preprocessor) -> None: """Set the preprocessor to use prior to executing predictions.""" self._override_preprocessor = preprocessor + self._override_preprocessor_set = True def predict( self, diff --git a/python/ray/train/examples/pytorch/torch_quick_start.py b/python/ray/train/examples/pytorch/torch_quick_start.py index 9f10e6333acbb..dd966cefc8f6a 100644 --- a/python/ray/train/examples/pytorch/torch_quick_start.py +++ b/python/ray/train/examples/pytorch/torch_quick_start.py @@ -5,79 +5,93 @@ # __torch_setup_begin__ import torch import torch.nn as nn - -num_samples = 20 -input_size = 10 -layer_size = 15 -output_size = 5 +from torch.utils.data import DataLoader +from torchvision import datasets +from torchvision.transforms import ToTensor + +def get_dataset(): + return datasets.FashionMNIST( + root="/tmp/data", + train=True, + download=True, + transform=ToTensor(), + ) class NeuralNetwork(nn.Module): def __init__(self): - super(NeuralNetwork, self).__init__() - self.layer1 = nn.Linear(input_size, layer_size) - self.relu = nn.ReLU() - self.layer2 = nn.Linear(layer_size, output_size) - - def forward(self, input): - return self.layer2(self.relu(self.layer1(input))) - -# In this example we use a randomly generated dataset. -input = torch.randn(num_samples, input_size) -labels = torch.randn(num_samples, output_size) - + super().__init__() + self.flatten = nn.Flatten() + self.linear_relu_stack = nn.Sequential( + nn.Linear(28 * 28, 512), + nn.ReLU(), + nn.Linear(512, 512), + nn.ReLU(), + nn.Linear(512, 10), + ) + + def forward(self, inputs): + inputs = self.flatten(inputs) + logits = self.linear_relu_stack(inputs) + return logits # __torch_setup_end__ # __torch_single_begin__ - -import torch.optim as optim - def train_func(): num_epochs = 3 + batch_size = 64 + + dataset = get_dataset() + dataloader = DataLoader(dataset, batch_size=batch_size) + model = NeuralNetwork() - loss_fn = nn.MSELoss() - optimizer = optim.SGD(model.parameters(), lr=0.1) + + criterion = nn.CrossEntropyLoss() + optimizer = torch.optim.SGD(model.parameters(), lr=0.01) for epoch in range(num_epochs): - output = model(input) - loss = loss_fn(output, labels) - optimizer.zero_grad() - loss.backward() - optimizer.step() + for inputs, labels in dataloader: + optimizer.zero_grad() + pred = model(inputs) + loss = criterion(pred, labels) + loss.backward() + optimizer.step() print(f"epoch: {epoch}, loss: {loss.item()}") - # __torch_single_end__ # __torch_distributed_begin__ - from ray import train def train_func_distributed(): num_epochs = 3 + batch_size = 64 + + dataset = get_dataset() + dataloader = DataLoader(dataset, batch_size=batch_size) + dataloader = train.torch.prepare_data_loader(dataloader) + model = NeuralNetwork() model = train.torch.prepare_model(model) - loss_fn = nn.MSELoss() - optimizer = optim.SGD(model.parameters(), lr=0.1) + + criterion = nn.CrossEntropyLoss() + optimizer = torch.optim.SGD(model.parameters(), lr=0.01) for epoch in range(num_epochs): - output = model(input) - loss = loss_fn(output, labels) - optimizer.zero_grad() - loss.backward() - optimizer.step() + for inputs, labels in dataloader: + optimizer.zero_grad() + pred = model(inputs) + loss = criterion(pred, labels) + loss.backward() + optimizer.step() print(f"epoch: {epoch}, loss: {loss.item()}") - # __torch_distributed_end__ if __name__ == "__main__": # __torch_single_run_begin__ - train_func() - # __torch_single_run_end__ # __torch_trainer_begin__ - from ray.train.torch import TorchTrainer from ray.air.config import ScalingConfig @@ -86,10 +100,8 @@ def train_func_distributed(): trainer = TorchTrainer( train_func_distributed, - scaling_config=ScalingConfig( - num_workers=4, use_gpu=use_gpu) + scaling_config=ScalingConfig(num_workers=4, use_gpu=use_gpu) ) results = trainer.fit() - # __torch_trainer_end__ diff --git a/python/ray/train/examples/tf/tensorflow_quick_start.py b/python/ray/train/examples/tf/tensorflow_quick_start.py index 15a8086c1a276..fea8983bb5a8a 100644 --- a/python/ray/train/examples/tf/tensorflow_quick_start.py +++ b/python/ray/train/examples/tf/tensorflow_quick_start.py @@ -3,7 +3,6 @@ # isort: skip_file # __tf_setup_begin__ - import numpy as np import tensorflow as tf @@ -32,21 +31,17 @@ def build_and_compile_cnn_model(): optimizer=tf.keras.optimizers.SGD(learning_rate=0.001), metrics=['accuracy']) return model - # __tf_setup_end__ # __tf_single_begin__ - def train_func(): batch_size = 64 single_worker_dataset = mnist_dataset(batch_size) single_worker_model = build_and_compile_cnn_model() single_worker_model.fit(single_worker_dataset, epochs=3, steps_per_epoch=70) - # __tf_single_end__ # __tf_distributed_begin__ - import json import os @@ -66,18 +61,14 @@ def train_func_distributed(): multi_worker_model = build_and_compile_cnn_model() multi_worker_model.fit(multi_worker_dataset, epochs=3, steps_per_epoch=70) - # __tf_distributed_end__ if __name__ == "__main__": # __tf_single_run_begin__ - train_func() - # __tf_single_run_end__ # __tf_trainer_begin__ - from ray.train.tensorflow import TensorflowTrainer from ray.air.config import ScalingConfig @@ -87,5 +78,4 @@ def train_func_distributed(): trainer = TensorflowTrainer(train_func_distributed, scaling_config=ScalingConfig(num_workers=4, use_gpu=use_gpu)) trainer.fit() - # __tf_trainer_end__ diff --git a/python/ray/train/huggingface/_huggingface_utils.py b/python/ray/train/huggingface/_huggingface_utils.py index ba886a6a9ff02..4254b78538369 100644 --- a/python/ray/train/huggingface/_huggingface_utils.py +++ b/python/ray/train/huggingface/_huggingface_utils.py @@ -7,7 +7,7 @@ from transformers.trainer_utils import IntervalStrategy from ray.air import session -from ray.data.dataset import Dataset +from ray.data import DatasetIterator from ray.train.huggingface.huggingface_checkpoint import HuggingFaceCheckpoint if TYPE_CHECKING: @@ -63,7 +63,7 @@ def get_train_dataloader(self): class RayDatasetHFIterable(datasets.iterable_dataset.ExamplesIterable): """HF ExamplesIterable backed by a Ray Dataset.""" - def __init__(self, dataset: Dataset) -> None: + def __init__(self, dataset: DatasetIterator) -> None: self.dataset = dataset self.generate_examples_fn = self.dataset.iter_rows @@ -75,7 +75,7 @@ def __iter__(self): yield (0, {k: v for k, v in row.as_pydict().items()}) -def process_dataset_for_hf(dataset: Dataset) -> "IterableDataset": +def process_dataset_for_hf(dataset: DatasetIterator) -> "IterableDataset": """Converts a Ray Dataset into a HF IterableDataset.""" hf_iterable = RayDatasetHFIterable(dataset) @@ -84,8 +84,8 @@ def process_dataset_for_hf(dataset: Dataset) -> "IterableDataset": ).with_format("torch") try: - dataset_length = dataset.count() - except ValueError: + dataset_length = dataset._base_dataset.count() + except (ValueError, AttributeError): # pipeline case dataset_length = None @@ -94,8 +94,8 @@ def process_dataset_for_hf(dataset: Dataset) -> "IterableDataset": def process_datasets( - train_dataset: Dataset, - eval_dataset: Dataset, + train_dataset: DatasetIterator, + eval_dataset: DatasetIterator, ) -> Tuple["IterableDataset", "IterableDataset"]: """Convert Ray train and validation to HF IterableDatasets.""" train_torch_dataset = process_dataset_for_hf(train_dataset) diff --git a/python/ray/train/huggingface/huggingface_predictor.py b/python/ray/train/huggingface/huggingface_predictor.py index b469dcaf819d7..aef519970df53 100644 --- a/python/ray/train/huggingface/huggingface_predictor.py +++ b/python/ray/train/huggingface/huggingface_predictor.py @@ -74,7 +74,8 @@ def __init__( "prediction will only use CPU. Please consider explicitly " "setting `HuggingFacePredictor(use_gpu=True)` or " "`batch_predictor.predict(ds, num_gpus_per_worker=1)` to " - "enable GPU prediction." + "enable GPU prediction. Ignore if you have set `device` or " + "`device_map` arguments in the `pipeline` manually." ) super().__init__(preprocessor) @@ -98,6 +99,11 @@ def from_checkpoint( The checkpoint is expected to be a result of ``HuggingFaceTrainer``. + Note that the Transformers ``pipeline`` used internally expects to + recieve raw text. If you have any Preprocessors in Checkpoint + that tokenize the data, remove them by calling + ``Checkpoint.set_preprocessor(None)`` beforehand. + Args: checkpoint: The checkpoint to load the model, tokenizer and preprocessor from. It is expected to be from the result of a @@ -111,13 +117,14 @@ def from_checkpoint( initialization. If ``pipeline`` is None, this must contain the 'task' argument. Cannot contain 'model'. Can be used to override the tokenizer with 'tokenizer'. If ``use_gpu`` is - True, 'device' will be set to 0 by default. + True, 'device' will be set to 0 by default, unless 'device_map' is + passed. """ if not pipeline_cls and "task" not in pipeline_kwargs: raise ValueError( "If `pipeline_cls` is not specified, 'task' must be passed as a kwarg." ) - if use_gpu: + if use_gpu and "device_map" not in pipeline_kwargs: # default to using the GPU with the first index pipeline_kwargs.setdefault("device", 0) pipeline_cls = pipeline_cls or pipeline_factory diff --git a/python/ray/train/huggingface/huggingface_trainer.py b/python/ray/train/huggingface/huggingface_trainer.py index 7c548d808b106..f58e2cc993d6f 100644 --- a/python/ray/train/huggingface/huggingface_trainer.py +++ b/python/ray/train/huggingface/huggingface_trainer.py @@ -63,16 +63,6 @@ sys.modules[spec.name] = datasets_modules spec.loader.exec_module(datasets_modules) -# This trainer uses a special checkpoint syncing logic. -# Because HF checkpoints are very large dirs (at least several GBs), -# we use directory checkpoints that are synced between nodes when -# required instead of serializing the checkpoints and sending -# bytes over nodes. This is a much more performant solution for -# large directory checkpoints. The current implementation -# is special for HuggingFaceTrainer, but can and should be -# made generic. -# TODO(ml-team): Make dir syncing checkpoint logic generic. - TRAINER_INIT_FN_KEY = "_trainer_init_per_worker" diff --git a/python/ray/train/tests/test_batch_predictor.py b/python/ray/train/tests/test_batch_predictor.py index c361b8c415c90..402211d35c303 100644 --- a/python/ray/train/tests/test_batch_predictor.py +++ b/python/ray/train/tests/test_batch_predictor.py @@ -504,6 +504,18 @@ def test_get_and_set_preprocessor(): 12.0, ] + # Remove preprocessor + batch_predictor.set_preprocessor(None) + assert batch_predictor.get_preprocessor() is None + + output_ds = batch_predictor.predict(test_dataset) + assert output_ds.to_pandas().to_numpy().squeeze().tolist() == [ + 0.0, + 2.0, + 4.0, + 6.0, + ] + def test_batch_prediction_large_predictor_kwarg(): class StubPredictor(Predictor): diff --git a/python/ray/train/tests/test_huggingface_trainer.py b/python/ray/train/tests/test_huggingface_trainer.py index 5292cca60bee6..2698d7524a8ff 100644 --- a/python/ray/train/tests/test_huggingface_trainer.py +++ b/python/ray/train/tests/test_huggingface_trainer.py @@ -56,6 +56,9 @@ def ray_start_8_cpus(): def train_function(train_dataset, eval_dataset=None, **config): + # Check that train_dataset has len + assert len(train_dataset) + model_config = AutoConfig.from_pretrained(model_checkpoint) model = AutoModelForCausalLM.from_config(model_config) evaluation_strategy = ( diff --git a/python/ray/train/tests/test_huggingface_trainer_steps.py b/python/ray/train/tests/test_huggingface_trainer_steps.py index c1d98f257a242..efa504f75f6a4 100644 --- a/python/ray/train/tests/test_huggingface_trainer_steps.py +++ b/python/ray/train/tests/test_huggingface_trainer_steps.py @@ -38,6 +38,9 @@ def ray_start_4_cpus(): def train_function(train_dataset, eval_dataset=None, **config): + # Check that train_dataset has len + assert len(train_dataset) + model_config = AutoConfig.from_pretrained(model_checkpoint) model = AutoModelForCausalLM.from_config(model_config) training_args = TrainingArguments( diff --git a/python/ray/train/tests/test_mosaic_trainer.py b/python/ray/train/tests/test_mosaic_trainer.py index 59d1c88a0284b..7a670d664e58c 100644 --- a/python/ray/train/tests/test_mosaic_trainer.py +++ b/python/ray/train/tests/test_mosaic_trainer.py @@ -258,7 +258,7 @@ def test_monitor_callbacks(ray_start_4_cpus): from ray.train.mosaic import MosaicTrainer # Test Callbacks involving logging (SpeedMonitor, LRMonitor) - from composer.callbacks import SpeedMonitor, LRMonitor, GradMonitor + from composer.callbacks import SpeedMonitor, LRMonitor trainer_init_config = { "max_duration": "1ep", @@ -270,7 +270,6 @@ def test_monitor_callbacks(ray_start_4_cpus): trainer_init_config["callbacks"] = [ SpeedMonitor(window_size=3), LRMonitor(), - GradMonitor(), ] trainer = MosaicTrainer( @@ -289,7 +288,6 @@ def test_monitor_callbacks(ray_start_4_cpus): "wall_clock/val", "wall_clock/total", "lr-DecoupledSGDW/group0", - "grad_l2_norm/step", ] for column in columns_to_check: assert column in metrics_columns, column + " is not found" diff --git a/python/ray/train/tests/test_worker_group.py b/python/ray/train/tests/test_worker_group.py index 249d2b3b4c396..e40be2ff16a00 100644 --- a/python/ray/train/tests/test_worker_group.py +++ b/python/ray/train/tests/test_worker_group.py @@ -3,7 +3,9 @@ import pytest import ray -from ray.train._internal.worker_group import WorkerGroup +from ray.train._internal.worker_group import WorkerGroup, Worker, WorkerMetadata +from copy import deepcopy +from random import seed, shuffle @pytest.fixture @@ -81,6 +83,29 @@ def test_execute_args(ray_start_2_cpus): assert all(o == 1 for o in outputs) +def test_move_workers_with_ip_to_front(ray_start_2_cpus): + wg = WorkerGroup(num_workers=2) + wg.workers = [ + Worker( + actor=None, + metadata=WorkerMetadata( + node_id="dummy", node_ip=f"10.1.10.{i}", hostname="dummy", gpu_ids=None + ), + ) + for i in range(1, 17) + ] + wg.workers += deepcopy(wg.workers) + workers_pre_move = deepcopy(wg.workers) + seed(1) + shuffle(wg.workers) + wg._move_workers_with_ip_to_front("10.1.10.1") + assert wg.workers[0].metadata.node_ip == "10.1.10.1" + assert wg.workers[1].metadata.node_ip == "10.1.10.1" + assert sorted([w.metadata.node_ip for w in workers_pre_move]) == sorted( + [w.metadata.node_ip for w in wg.workers] + ) + + def test_execute_single(ray_start_2_cpus): wg = WorkerGroup(num_workers=2) diff --git a/python/ray/tune/BUILD b/python/ray/tune/BUILD index b9030d0d3880b..e3d69b87c5058 100644 --- a/python/ray/tune/BUILD +++ b/python/ray/tune/BUILD @@ -296,6 +296,14 @@ py_test( tags = ["team:ml", "exclusive"], ) +py_test( + name = "test_util_object_cache", + size = "small", + srcs = ["tests/test_util_object_cache.py"], + deps = [":tune_lib"], + tags = ["team:ml", "exclusive"], +) + py_test( name = "test_syncer", size = "medium", diff --git a/python/ray/tune/execution/ray_trial_executor.py b/python/ray/tune/execution/ray_trial_executor.py index 31a566f3f3541..18c32a38d0b9b 100644 --- a/python/ray/tune/execution/ray_trial_executor.py +++ b/python/ray/tune/execution/ray_trial_executor.py @@ -6,15 +6,15 @@ import random import time import traceback -from collections import deque, defaultdict, Counter +from collections import deque from contextlib import contextmanager from enum import Enum from functools import partial -from typing import Callable, Dict, Iterable, List, Optional, Set, Union, Tuple +from typing import Callable, Dict, Iterable, Optional, Set, Union import ray from ray.actor import ActorHandle -from ray.air import Checkpoint, AcquiredResources, ResourceRequest +from ray.air import Checkpoint, AcquiredResources from ray.air._internal.checkpoint_manager import CheckpointStorage, _TrackedCheckpoint from ray.air.constants import ( COPY_DIRECTORY_CHECKPOINTS_INSTEAD_OF_MOVING_ENV, @@ -35,6 +35,7 @@ from ray.tune.result import STDERR_FILE, STDOUT_FILE, TRIAL_INFO from ray.tune.experiment.trial import Trial, _Location, _TrialInfo from ray.tune.utils import warn_if_slow +from ray.tune.utils.object_cache import _ObjectCache from ray.tune.utils.resource_updater import _ResourceUpdater from ray.tune.trainable.util import TrainableUtil from ray.util import log_once @@ -234,13 +235,10 @@ def __init__( # Actor re-use. # For details, see docstring of `_maybe_cache_trial_actor()` self._reuse_actors = reuse_actors - self._resource_request_to_cached_actors: Dict[ - ResourceRequest, List[Tuple[ray.actor.ActorHandle, AcquiredResources]] - ] = defaultdict(list) + self._actor_cache = _ObjectCache(may_keep_one=True) # Trials for which we requested resources self._staged_trials = set() # Staged trials - self._staged_resources = Counter() # Resources of staged trials self._trial_to_acquired_resources: Dict[Trial, AcquiredResources] = {} # Result buffer @@ -261,7 +259,7 @@ def __init__( def setup( self, max_pending_trials: int, trainable_kwargs: Optional[Dict] = None ) -> None: - if len(self._resource_request_to_cached_actors) > 0: + if self._actor_cache.num_cached_objects: logger.warning( "Cannot update maximum number of queued actors for reuse " "during a run." @@ -327,7 +325,7 @@ def _stage_and_update_status(self, trials: Iterable[Trial]): resource_request = trial.placement_group_factory self._staged_trials.add(trial) - self._staged_resources[trial.placement_group_factory] += 1 + self._actor_cache.increase_max(resource_request) self._resource_manager.request_resources(resource_request=resource_request) self._resource_manager.update_state() @@ -344,7 +342,7 @@ def get_ready_trial(self) -> Optional[Trial]: for trial in self._staged_trials: resource_request = trial.placement_group_factory # If we have a cached actor for these resources, return - if self._resource_request_to_cached_actors[resource_request]: + if self._actor_cache.has_cached_object(resource_request): return trial # If the resources are available from the resource manager, return @@ -360,12 +358,13 @@ def _maybe_use_cached_actor(self, trial, logger_creator) -> Optional[ActorHandle return None resource_request = trial.placement_group_factory - if not self._resource_request_to_cached_actors[resource_request]: + + if not self._actor_cache.has_cached_object(resource_request): return None - actor, acquired_resources = self._resource_request_to_cached_actors[ + actor, acquired_resources = self._actor_cache.pop_cached_object( resource_request - ].pop(0) + ) logger.debug(f"Trial {trial}: Reusing cached actor " f"{actor}") @@ -541,7 +540,7 @@ def _unstage_trial_with_resources(self, trial: Trial): # Case 1: The trial we started was staged. Just remove it if trial in self._staged_trials: self._staged_trials.remove(trial) - self._staged_resources[trial.placement_group_factory] -= 1 + self._actor_cache.decrease_max(trial.placement_group_factory) return # Case 2: We staged a trial "A" with the same resources, but our trial "B" @@ -560,7 +559,7 @@ def _unstage_trial_with_resources(self, trial: Trial): if candidate_trial: self._staged_trials.remove(candidate_trial) - self._staged_resources[candidate_trial.placement_group_factory] -= 1 + self._actor_cache.decrease_max(candidate_trial.placement_group_factory) return raise RuntimeError( @@ -593,16 +592,8 @@ def _maybe_cache_trial_actor(self, trial: Trial) -> bool: acquired_resources = self._trial_to_acquired_resources[trial] cached_resource_request = acquired_resources.resource_request - staged_resource_count = self._count_staged_resources() - if ( - # If we have at least one cached actor already - any(v for v in self._resource_request_to_cached_actors.values()) - # and we haven't requested resources for an actor with the - # same resources as the actor we want to cache - and len(self._resource_request_to_cached_actors[cached_resource_request]) - >= staged_resource_count[cached_resource_request] - # then we don't have an immediate need for the actor and don't - # want to cache it. + if not self._actor_cache.cache_object( + cached_resource_request, (trial.runner, acquired_resources) ): logger.debug( f"Could not cache actor of trial {trial} for " @@ -613,9 +604,6 @@ def _maybe_cache_trial_actor(self, trial: Trial) -> bool: logger.debug(f"Caching actor of trial {trial} for re-use") - self._resource_request_to_cached_actors[cached_resource_request].append( - (trial.runner, acquired_resources) - ) self._trial_to_acquired_resources.pop(trial) trial.set_runner(None) @@ -833,7 +821,7 @@ def has_resources_for_trial(self, trial: Trial) -> bool: return ( trial in self._staged_trials - or self._resource_request_to_cached_actors[resource_request] + or self._actor_cache.has_cached_object(resource_request) or len(self._staged_trials) < self._max_staged_actors or self._resource_manager.has_resources_ready(resource_request) ) @@ -861,9 +849,6 @@ def on_step_end(self, search_ended: bool = False) -> None: self._cleanup_cached_actors(search_ended=search_ended) self._do_force_trial_cleanup() - def _count_staged_resources(self): - return self._staged_resources - def _cleanup_cached_actors( self, search_ended: bool = False, force_all: bool = False ): @@ -902,21 +887,16 @@ def _cleanup_cached_actors( # (if the search ended). return - staged_resources = self._count_staged_resources() - - for resource_request, actors in self._resource_request_to_cached_actors.items(): - while len(actors) > staged_resources.get(resource_request, 0) or ( - force_all and len(actors) - ): - actor, acquired_resources = actors[-1] - actors.pop() - future = actor.stop.remote() - self._futures[future] = ( - _ExecutorEventType.STOP_RESULT, - acquired_resources, - ) - if self._trial_cleanup: # force trial cleanup within a deadline - self._trial_cleanup.add(future) + for (actor, acquired_resources) in self._actor_cache.flush_cached_objects( + force_all=force_all + ): + future = actor.stop.remote() + self._futures[future] = ( + _ExecutorEventType.STOP_RESULT, + acquired_resources, + ) + if self._trial_cleanup: # force trial cleanup within a deadline + self._trial_cleanup.add(future) def _resolve_stop_event( self, @@ -1196,18 +1176,12 @@ def get_next_executor_event( # when next_trial_exists and there are cached resources ################################################################### # There could be existing PGs from either - # `self._resource_request_to_cached_actors` + # `self._actor_cache` # or from ready trials. If so and if there is indeed # a next trial to run, we return `PG_READY` future for trial # runner. The next trial can then be scheduled on this PG. if next_trial_exists: - if ( - sum( - len(cached) - for cached in self._resource_request_to_cached_actors.values() - ) - > 0 - ): + if self._actor_cache.num_cached_objects > 0: return _ExecutorEvent(_ExecutorEventType.PG_READY) # TODO(xwjiang): Expose proper API when we decide to do # ActorPool abstraction. diff --git a/python/ray/tune/registry.py b/python/ray/tune/registry.py index 069689ae5d495..be868ce3cad72 100644 --- a/python/ray/tune/registry.py +++ b/python/ray/tune/registry.py @@ -1,5 +1,5 @@ +import atexit import logging -import uuid from functools import partial from types import FunctionType from typing import Callable, Optional, Type, Union @@ -10,6 +10,7 @@ _internal_kv_get, _internal_kv_initialized, _internal_kv_put, + _internal_kv_del, ) from ray.tune.error import TuneError from ray.util.annotations import DeveloperAPI @@ -111,6 +112,10 @@ def register_trainable(name: str, trainable: Union[Callable, Type], warn: bool = _global_registry.register(TRAINABLE_CLASS, name, trainable) +def _unregister_trainables(): + _global_registry.unregister_all(TRAINABLE_CLASS) + + @DeveloperAPI def register_env(name: str, env_creator: Callable): """Register a custom environment for use with RLlib. @@ -128,6 +133,10 @@ def register_env(name: str, env_creator: Callable): _global_registry.register(ENV_CREATOR, name, env_creator) +def _unregister_envs(): + _global_registry.unregister_all(ENV_CREATOR) + + @DeveloperAPI def register_input(name: str, input_creator: Callable): """Register a custom input api for RLlib. @@ -142,6 +151,10 @@ def register_input(name: str, input_creator: Callable): _global_registry.register(RLLIB_INPUT, name, input_creator) +def _unregister_inputs(): + _global_registry.unregister_all(RLLIB_INPUT) + + @DeveloperAPI def registry_contains_input(name: str) -> bool: return _global_registry.contains(RLLIB_INPUT, name) @@ -152,6 +165,12 @@ def registry_get_input(name: str) -> Callable: return _global_registry.get(RLLIB_INPUT, name) +def _unregister_all(): + _unregister_inputs() + _unregister_envs() + _unregister_trainables() + + def _check_serializability(key, value): _global_registry.register(TEST, key, value) @@ -179,8 +198,29 @@ def _make_key(prefix: str, category: str, key: str): class _Registry: def __init__(self, prefix: Optional[str] = None): + """If no prefix is given, use runtime context job ID.""" self._to_flush = {} - self._prefix = prefix or uuid.uuid4().hex[:8] + self._prefix = prefix + self._registered = set() + self._atexit_handler_registered = False + + @property + def prefix(self): + if not self._prefix: + self._prefix = ray.get_runtime_context().get_job_id() + return self._prefix + + def _register_atexit(self): + if self._atexit_handler_registered: + # Already registered + return + + if ray._private.worker.global_worker.mode != ray.SCRIPT_MODE: + # Only cleanup on the driver + return + + atexit.register(_unregister_all) + self._atexit_handler_registered = True def register(self, category, key, value): """Registers the value with the global registry. @@ -198,16 +238,31 @@ def register(self, category, key, value): if _internal_kv_initialized(): self.flush_values() + def unregister(self, category, key): + if _internal_kv_initialized(): + _internal_kv_del(_make_key(self.prefix, category, key)) + else: + self._to_flush.pop((category, key), None) + + def unregister_all(self, category: Optional[str] = None): + remaining = set() + for (cat, key) in self._registered: + if category and category == cat: + self.unregister(cat, key) + else: + remaining.add((cat, key)) + self._registered = remaining + def contains(self, category, key): if _internal_kv_initialized(): - value = _internal_kv_get(_make_key(self._prefix, category, key)) + value = _internal_kv_get(_make_key(self.prefix, category, key)) return value is not None else: return (category, key) in self._to_flush def get(self, category, key): if _internal_kv_initialized(): - value = _internal_kv_get(_make_key(self._prefix, category, key)) + value = _internal_kv_get(_make_key(self.prefix, category, key)) if value is None: raise ValueError( "Registry value for {}/{} doesn't exist.".format(category, key) @@ -217,14 +272,16 @@ def get(self, category, key): return pickle.loads(self._to_flush[(category, key)]) def flush_values(self): + self._register_atexit() for (category, key), value in self._to_flush.items(): _internal_kv_put( - _make_key(self._prefix, category, key), value, overwrite=True + _make_key(self.prefix, category, key), value, overwrite=True ) + self._registered.add((category, key)) self._to_flush.clear() -_global_registry = _Registry(prefix="global") +_global_registry = _Registry() ray._private.worker._post_init_hooks.append(_global_registry.flush_values) diff --git a/python/ray/tune/tests/_test_multi_tenancy_run.py b/python/ray/tune/tests/_test_multi_tenancy_run.py index 2b42a96c0f127..74da15b283289 100644 --- a/python/ray/tune/tests/_test_multi_tenancy_run.py +++ b/python/ray/tune/tests/_test_multi_tenancy_run.py @@ -33,9 +33,6 @@ # are tracked by the driver, not the trainable. VALS = [int(os.environ["VAL_1"]), int(os.environ["VAL_2"])] -# If 1, use workaround, if 0, just run (and fail in job 1). -USE_WORKAROUND = bool(int(os.environ["WORKAROUND"])) - # Wait for HANG_RUN_MARKER while HANG_RUN_MARKER and Path(HANG_RUN_MARKER).exists(): time.sleep(0.1) @@ -56,13 +53,6 @@ def train_func(config): session.report({"param": config["param"], "fixed": config["fixed"]}) -# Workaround: Just use a unique name per trainer/trainable -if USE_WORKAROUND: - import uuid - - DataParallelTrainer.__name__ = "DataParallelTrainer_" + uuid.uuid4().hex[:8] - - trainer = DataParallelTrainer( train_loop_per_worker=train_func, train_loop_config={ @@ -97,9 +87,3 @@ def train_func(config): # Put assertions last, so we don't finish early because of failures assert sorted([result.metrics["param"] for result in results]) == VALS assert [result.metrics["fixed"] for result in results] == [FIXED_VAL, FIXED_VAL] - -if USE_WORKAROUND: - from ray.experimental.internal_kv import _internal_kv_del - from ray.tune.registry import _make_key, TRAINABLE_CLASS - - _internal_kv_del(_make_key("global", TRAINABLE_CLASS, DataParallelTrainer.__name__)) diff --git a/python/ray/tune/tests/test_experiment_analysis.py b/python/ray/tune/tests/test_experiment_analysis.py index c89202b50d2b4..b99d3fbbce57b 100644 --- a/python/ray/tune/tests/test_experiment_analysis.py +++ b/python/ray/tune/tests/test_experiment_analysis.py @@ -390,9 +390,7 @@ def testPickling(self): self.assertTrue(analysis.get_best_trial(metric=self.metric, mode="max")) ray.shutdown() - ray.tune.registry._global_registry = ray.tune.registry._Registry( - prefix="global" - ) + ray.tune.registry._global_registry = ray.tune.registry._Registry() with open(pickle_path, "rb") as f: analysis = pickle.load(f) @@ -406,9 +404,7 @@ def testFromPath(self): self.assertTrue(analysis.get_best_trial(metric=self.metric, mode="max")) ray.shutdown() - ray.tune.registry._global_registry = ray.tune.registry._Registry( - prefix="global" - ) + ray.tune.registry._global_registry = ray.tune.registry._Registry() analysis = ExperimentAnalysis(self.test_path) diff --git a/python/ray/tune/tests/test_multi_tenancy.py b/python/ray/tune/tests/test_multi_tenancy.py index 71062c8595cc4..b363d7dff7b67 100644 --- a/python/ray/tune/tests/test_multi_tenancy.py +++ b/python/ray/tune/tests/test_multi_tenancy.py @@ -15,14 +15,13 @@ def ray_start_4_cpus(): ray.shutdown() -@pytest.mark.parametrize("use_workaround", [False, True]) @pytest.mark.parametrize("exit_same", [False, True]) -def test_registry_conflict(ray_start_4_cpus, tmpdir, use_workaround, exit_same): +def test_registry_conflict(ray_start_4_cpus, tmpdir, exit_same): """Two concurrent Tune runs can conflict with each other when they use a trainable with the same name. - This test starts two runs in parallel and asserts that a workaround used - in the docs can alleviate the problem. + This test starts two runs in parallel and asserts that our fix in + https://github.com/ray-project/ray/pull/33095 resolves the issue. This is how we schedule the runs: @@ -42,10 +41,6 @@ def test_registry_conflict(ray_start_4_cpus, tmpdir, use_workaround, exit_same): - Run 1 finally finishes, and we compare the expected results with the actual results. - When you don't use the workaround, expect an assertion error (if ``exit_same=True``, - see below), otherwise a KeyError (because a trial failed). - When the workaround is used, we expect everything to run without error. - NOTE: Two errors can occur with registry conflicts. First, the trainable can be overwritten and captured, for example, when a fixed value is included in the trainable. The second trial of run 1 then has a wrong @@ -57,10 +52,6 @@ def test_registry_conflict(ray_start_4_cpus, tmpdir, use_workaround, exit_same): removed already. Note that these objects are registered with ``tune.with_parameters()`` (not the global registry store). We test both scenarios using the ``exit_same`` parameter. - - NOTE: If we resolve the registry issue (for example, with unique keys) - you can remove the test that expects the assertion error. We can remove - the parametrization and the workaround and assert that no conflict occurs. """ # Create file markers run_1_running = tmpdir / "run_1_running" @@ -75,7 +66,6 @@ def test_registry_conflict(ray_start_4_cpus, tmpdir, use_workaround, exit_same): run_1_env = { "RAY_ADDRESS": ray_address, - "WORKAROUND": str(int(use_workaround)), "FIXED_VAL": str(1), "VAL_1": str(2), "VAL_2": str(3), @@ -93,7 +83,6 @@ def test_registry_conflict(ray_start_4_cpus, tmpdir, use_workaround, exit_same): run_2_env = { "RAY_ADDRESS": ray_address, - "WORKAROUND": str(int(use_workaround)), "FIXED_VAL": str(4), "VAL_1": str(5), "VAL_2": str(6), @@ -123,18 +112,7 @@ def test_registry_conflict(ray_start_4_cpus, tmpdir, use_workaround, exit_same): print("Started run 2:", run_2.pid) assert run_2.wait() == 0 - - if use_workaround: - assert run_1.wait() == 0 - else: - assert run_1.wait() != 0 - - stderr = run_1.stderr.read().decode() - - if not exit_same: - assert "OwnerDiedError" in stderr, stderr - else: - assert "AssertionError" in stderr, stderr + assert run_1.wait() == 0 if __name__ == "__main__": diff --git a/python/ray/tune/tests/test_ray_trial_executor.py b/python/ray/tune/tests/test_ray_trial_executor.py index 66e9152f84068..ad8a99d09b47e 100644 --- a/python/ray/tune/tests/test_ray_trial_executor.py +++ b/python/ray/tune/tests/test_ray_trial_executor.py @@ -648,14 +648,7 @@ def train(config): executor._stage_and_update_status([trial1, trial2, trial3]) executor.pause_trial(trial1) # Caches the PG - assert ( - len( - executor._resource_request_to_cached_actors[ - trial1.placement_group_factory - ] - ) - == 1 - ) + assert executor._actor_cache.num_cached_objects == 1 # Second trial remains staged, it will only be removed from staging when it # is started diff --git a/python/ray/tune/tests/test_trial_runner_pg.py b/python/ray/tune/tests/test_trial_runner_pg.py index 0349188b1f0d6..6af195c90932f 100644 --- a/python/ray/tune/tests/test_trial_runner_pg.py +++ b/python/ray/tune/tests/test_trial_runner_pg.py @@ -112,10 +112,7 @@ def on_step_end(self, iteration, trials, **info): len(s) for s in resource_manager._request_to_ready_pgs.values() ) num_in_use = len(resource_manager._acquired_pgs) - num_cached = sum( - len(a) - for a in trial_executor._resource_request_to_cached_actors.values() - ) + num_cached = trial_executor._actor_cache.num_cached_objects total_num_tracked = num_staging + num_ready + num_in_use + num_cached diff --git a/python/ray/tune/tests/test_util_object_cache.py b/python/ray/tune/tests/test_util_object_cache.py new file mode 100644 index 0000000000000..dd61c03b8de46 --- /dev/null +++ b/python/ray/tune/tests/test_util_object_cache.py @@ -0,0 +1,124 @@ +import pytest + +from ray.tune.utils.object_cache import _ObjectCache + + +@pytest.mark.parametrize("eager", [False, True]) +def test_no_may_keep_one(eager): + """Test object caching. + + - After init, no objects are cached (as max cached is 0), except when eager caching + - After increasing max to 2, up to 2 objects are cached + - Decreasing max objects will evict them on flush + """ + cache = _ObjectCache(may_keep_one=eager) + + # max(A) = 0, so we we only cache when eager caching + assert cache.cache_object("A", 1) == eager + assert cache.num_cached_objects == int(eager) + + # Set max(A) = 2 + cache.increase_max("A", 2) + + # max(A) = 2, so we cache up to two objects + if not eager: + assert cache.cache_object("A", 1) + + assert cache.cache_object("A", 2) + assert not cache.cache_object("A", 3) + + assert cache.num_cached_objects == 2 + + # Nothing has to be evicted + assert not list(cache.flush_cached_objects()) + + # Set max(A) = 1, so we have one object too much + cache.decrease_max("A", 1) + + # First cached object is evicted + assert list(cache.flush_cached_objects()) == [1] + assert cache.num_cached_objects == 1 + + # Set max(A) = 0 + cache.decrease_max("A", 1) + + # Second cached object is evicted if not eager caching + assert list(cache.flush_cached_objects()) == ([2] if not eager else []) + assert cache.num_cached_objects == (0 if not eager else 1) + + +@pytest.mark.parametrize("eager", [False, True]) +def test_multi(eager): + """Test caching with multiple objects""" + cache = _ObjectCache(may_keep_one=eager) + + # max(A) = 0, so we we only cache when eager caching + assert cache.cache_object("A", 1) == eager + assert cache.num_cached_objects == int(eager) + + # max(B) = 0, so no caching + assert not cache.cache_object("B", 5) + assert cache.num_cached_objects == int(eager) + + # Increase maximums levels + cache.increase_max("A", 1) + cache.increase_max("B", 1) + + # Cache objects (A is already cached if eager) + assert cache.cache_object("A", 1) != eager + assert cache.cache_object("B", 5) + + # No further objects can be cached + assert not cache.cache_object("A", 2) + assert not cache.cache_object("B", 6) + + assert cache.num_cached_objects == 2 + + # Decrease + cache.decrease_max("A", 1) + + # Evict A object + assert list(cache.flush_cached_objects()) == [1] + + cache.decrease_max("B", 1) + + # If eager, keep B object, otherwise, evict B + assert list(cache.flush_cached_objects()) == ([5] if not eager else []) + assert cache.num_cached_objects == (0 if not eager else 1) + + +def test_multi_eager_other(): + """On eager caching, only cache an object if no other object is expected. + + - Expect up to one cached A object + - Try to cache object B --> doesn't get cached + - Remove expectation for A object + - Try to cache object B --> get's cached + """ + cache = _ObjectCache(may_keep_one=True) + + cache.increase_max("A", 1) + assert not cache.cache_object("B", 2) + + cache.decrease_max("A", 1) + assert cache.cache_object("B", 3) + + +@pytest.mark.parametrize("eager", [False, True]) +def test_force_all(eager): + """Assert that force_all=True will always evict all object.""" + cache = _ObjectCache(may_keep_one=eager) + + cache.increase_max("A", 2) + + assert cache.cache_object("A", 1) + assert cache.cache_object("A", 2) + + assert list(cache.flush_cached_objects(force_all=True)) == [1, 2] + assert cache.num_cached_objects == 0 + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main(["-v", __file__])) diff --git a/python/ray/tune/trainable/util.py b/python/ray/tune/trainable/util.py index 2c62278571ad2..44714da403768 100644 --- a/python/ray/tune/trainable/util.py +++ b/python/ray/tune/trainable/util.py @@ -148,9 +148,10 @@ def get_checkpoints_paths(logdir): iter_chkpt_pairs = [] for marker_path in marker_paths: chkpt_dir = os.path.dirname(marker_path) + basename = os.path.basename(chkpt_dir) # Skip temporary checkpoints - if os.path.basename(chkpt_dir).startswith("checkpoint_tmp"): + if basename.startswith("checkpoint_tmp"): continue metadata_file = glob.glob( @@ -162,9 +163,21 @@ def get_checkpoints_paths(logdir): os.path.join(glob.escape(chkpt_dir), _TUNE_METADATA_FILENAME) ) metadata_file = list(set(metadata_file)) # avoid duplication - if len(metadata_file) != 1: + if len(metadata_file) == 0: + logger.warning( + f"The checkpoint {basename} does not have a metadata file. " + f"This usually means that the training process was interrupted " + f"while the checkpoint was being written. The checkpoint will be " + f"excluded from analysis. Consider deleting the directory. " + f"Full path: {chkpt_dir}" + ) + continue + elif len(metadata_file) > 1: raise ValueError( - "{} has zero or more than one tune_metadata.".format(chkpt_dir) + f"The checkpoint {basename} contains more than one metadata file. " + f"If this happened without manual intervention, please file an " + f"issue at https://github.com/ray-project/ray/issues. " + f"Full path: {chkpt_dir}" ) metadata_file = metadata_file[0] diff --git a/python/ray/tune/utils/object_cache.py b/python/ray/tune/utils/object_cache.py new file mode 100644 index 0000000000000..a93af2f37b274 --- /dev/null +++ b/python/ray/tune/utils/object_cache.py @@ -0,0 +1,169 @@ +from collections import defaultdict, Counter + +from typing import Dict, Generator, List, Optional, TypeVar + +# Grouping key - must be hashable +T = TypeVar("T") +# Objects to cache +U = TypeVar("U") + + +class _ObjectCache: + """Cache up to some maximum count given a grouping key. + + This object cache can e.g. be used to cache Ray Tune trainable actors + given their resource requirements (reuse_actors=True). + + If the max number of cached objects for a grouping key is reached, + no more objects for this group will be cached. + + However, if `may_keep_one=True`, one object (globally across all grouping + keys) may be cached, even if the max number of objects is 0. This is to + allow to cache an object if the max number of objects of this key + will increase shortly after (as is the case e.g. in the Ray Tune control + loop). + + Args: + may_keep_one: If True, one object (globally) may be cached if no desired + maximum objects are defined. + + """ + + def __init__(self, may_keep_one: bool = True): + self._num_cached_objects: int = 0 + self._cached_objects: Dict[T, List[U]] = defaultdict(list) + self._max_num_objects: Counter[T] = Counter() + + self._may_keep_one = may_keep_one + + @property + def num_cached_objects(self): + return self._num_cached_objects + + def increase_max(self, key: T, by: int = 1) -> None: + """Increase number of max objects for this key. + + Args: + key: Group key. + by: Decrease by this amount. + """ + self._max_num_objects[key] += by + + def decrease_max(self, key: T, by: int = 1) -> None: + """Decrease number of max objects for this key. + + Args: + key: Group key. + by: Decrease by this amount. + """ + self._max_num_objects[key] -= by + + def has_cached_object(self, key: T) -> bool: + """Return True if at least one cached object exists for this key. + + Args: + key: Group key. + + Returns: + True if at least one cached object exists for this key. + """ + return bool(self._cached_objects[key]) + + def cache_object(self, key: T, obj: U) -> bool: + """Cache object for a given key. + + This will put the object into a cache, assuming the number + of cached objects for this key is less than the number of + max objects for this key. + + An exception is made if `max_keep_one=True` and no other + objects are cached globally. In that case, the object can + still be cached. + + Args: + key: Group key. + obj: Object to cache. + + Returns: + True if the object has been cached. False otherwise. + + """ + # If we have more objects cached already than we desire + if len(self._cached_objects[key]) >= self._max_num_objects[key]: + # If may_keep_one is False, never cache + if not self._may_keep_one: + return False + + # If we have more than one other cached object, don't cache + if self._num_cached_objects > 0: + return False + + # If any other objects are expected to be cached, don't cache + if any(v for v in self._max_num_objects.values()): + return False + + # Otherwise, cache (for now). + + self._cached_objects[key].append(obj) + self._num_cached_objects += 1 + return True + + def pop_cached_object(self, key: T) -> Optional[U]: + """Get one cached object for a key. + + This will remove the object from the cache. + + Args: + key: Group key. + + Returns: + Cached object. + """ + if not self.has_cached_object(key): + return None + + self._num_cached_objects -= 1 + return self._cached_objects[key].pop(0) + + def flush_cached_objects(self, force_all: bool = False) -> Generator[U, None, None]: + """Return a generator over cached objects evicted from the cache. + + This method yields all cached objects that should be evicted from the + cache for cleanup by the caller. + + If the number of max objects is lower than the number of + cached objects for a given key, objects are evicted until + the numbers are equal. + + If `max_keep_one=True` (and ``force_all=False``), one cached object + may be retained. + + Objects are evicted FIFO. + + If ``force_all=True``, all objects are evicted. + + Args: + force_all: If True, all objects are flushed. This takes precedence + over ``keep_one``. + + Yields: + Evicted objects to be cleaned up by caller. + + """ + # If force_all=True, don't keep one. + keep_one = self._may_keep_one and not force_all + + for key, objs in self._cached_objects.items(): + max = self._max_num_objects[key] if not force_all else 0 + + if ( + self._num_cached_objects == 1 + and keep_one + # Only keep this object if we don't expect a different one + and not any(v for v in self._max_num_objects.values()) + ): + break + + while len(objs) > max: + self._num_cached_objects -= 1 + yield objs.pop(0) diff --git a/python/ray/tune/utils/util.py b/python/ray/tune/utils/util.py index 4f2953555d033..f6979cdca6654 100644 --- a/python/ray/tune/utils/util.py +++ b/python/ray/tune/utils/util.py @@ -250,17 +250,31 @@ def __init__(self, stream1, stream2): self.stream1 = stream1 self.stream2 = stream2 + def _warn(self, op, s, args, kwargs): + msg = f"ValueError when calling '{op}' on stream ({s}). " + msg += f"args: {args} kwargs: {kwargs}" + logger.warning(msg) + def seek(self, *args, **kwargs): - self.stream1.seek(*args, **kwargs) - self.stream2.seek(*args, **kwargs) + for s in [self.stream1, self.stream2]: + try: + s.seek(*args, **kwargs) + except ValueError: + self._warn("seek", s, args, kwargs) def write(self, *args, **kwargs): - self.stream1.write(*args, **kwargs) - self.stream2.write(*args, **kwargs) + for s in [self.stream1, self.stream2]: + try: + s.write(*args, **kwargs) + except ValueError: + self._warn("write", s, args, kwargs) def flush(self, *args, **kwargs): - self.stream1.flush(*args, **kwargs) - self.stream2.flush(*args, **kwargs) + for s in [self.stream1, self.stream2]: + try: + s.flush(*args, **kwargs) + except ValueError: + self._warn("flush", s, args, kwargs) @property def encoding(self): diff --git a/python/ray/util/multiprocessing/pool.py b/python/ray/util/multiprocessing/pool.py index fa659a5b4ae55..299b271ca1869 100644 --- a/python/ray/util/multiprocessing/pool.py +++ b/python/ray/util/multiprocessing/pool.py @@ -7,14 +7,12 @@ import queue import sys import threading -import warnings import time from multiprocessing import TimeoutError from typing import Any, Callable, Dict, Hashable, Iterable, List, Optional, Tuple import ray from ray.util import log_once -from ray.util.annotations import RayDeprecationWarning try: from joblib._parallel_backends import SafeFunction @@ -390,21 +388,7 @@ def __init__(self, pool, func, iterable, chunksize=None): # submitted chunks. Ordering mirrors that in the in the ResultThread. self._submitted_chunks = [] self._ready_objects = collections.deque() - try: - self._iterator = iter(iterable) - except TypeError: - warnings.warn( - "Passing a non-iterable argument to the " - "ray.util.multiprocessing.Pool imap and imap_unordered " - "methods is deprecated as of Ray 2.3 and " - " will be removed in a future release. See " - "https://github.com/ray-project/ray/issues/24237 for more " - "information.", - category=RayDeprecationWarning, - stacklevel=3, - ) - iterable = [iterable] - self._iterator = iter(iterable) + self._iterator = iter(iterable) if isinstance(iterable, collections.abc.Iterator): # Got iterator (which has no len() function). # Make default chunksize 1 instead of using _calculate_chunksize(). diff --git a/python/ray/util/spark/cluster_init.py b/python/ray/util/spark/cluster_init.py index 100ec69e09c80..ef344f9a92ceb 100644 --- a/python/ray/util/spark/cluster_init.py +++ b/python/ray/util/spark/cluster_init.py @@ -213,7 +213,12 @@ def _convert_ray_node_option_key(key): def _convert_ray_node_options(options): - return [f"{_convert_ray_node_option_key(k)}={str(v)}" for k, v in options.items()] + return [ + f"{_convert_ray_node_option_key(k)}" + if v is None + else f"{_convert_ray_node_option_key(k)}={str(v)}" + for k, v in options.items() + ] _RAY_HEAD_STARTUP_TIMEOUT = 5 @@ -835,10 +840,16 @@ def setup_ray_cluster( head_node_options: A dict representing Ray head node extra options, these options will be passed to `ray start` script. Note you need to convert `ray start` options key from `--foo-bar` format to `foo_bar` format. + For flag options (e.g. '--disable-usage-stats'), you should set the value + to None in the option dict, like `{"disable_usage_stats": None}`. + Note: Short name options (e.g. '-v') are not supported. worker_node_options: A dict representing Ray worker node extra options, these options will be passed to `ray start` script. Note you need to convert `ray start` options key from `--foo-bar` format to `foo_bar` format. + For flag options (e.g. '--disable-usage-stats'), you should set the value + to None in the option dict, like `{"disable_usage_stats": None}`. + Note: Short name options (e.g. '-v') are not supported. ray_temp_root_dir: A local disk path to store the ray temporary data. The created cluster will create a subdirectory "ray-{head_port}-{random_suffix}" beneath this path. diff --git a/python/requirements/ml/requirements_dl.txt b/python/requirements/ml/requirements_dl.txt index 7ab1796596d1a..5549414a9f69d 100644 --- a/python/requirements/ml/requirements_dl.txt +++ b/python/requirements/ml/requirements_dl.txt @@ -1,7 +1,8 @@ # These requirements are used for the CI and CPU-only Docker images so we install CPU only versions of torch. # For GPU Docker images, you should install requirements_ml_docker.txt afterwards. -tensorflow==2.11.0 +tensorflow==2.11.0; sys_platform != 'darwin' or platform_machine != 'arm64' +tensorflow-macos==2.11.0; sys_platform == 'darwin' and platform_machine == 'arm64' tensorflow-probability==0.19.0 # If you make changes below this line, please also make the corresponding changes to `requirements_ml_docker.txt`! @@ -11,7 +12,7 @@ tensorflow-probability==0.19.0 torch==1.13.0 torchvision==0.14.0 torch-scatter==2.1.0 -torch-sparse==0.6.15 +torch-sparse==0.6.16 torch-cluster==1.6.0 torch-spline-conv==1.2.1 torch-geometric==2.1.0 diff --git a/python/requirements/ml/requirements_rllib.txt b/python/requirements/ml/requirements_rllib.txt index 8432397dbe9d5..1d5fea8705430 100644 --- a/python/requirements/ml/requirements_rllib.txt +++ b/python/requirements/ml/requirements_rllib.txt @@ -24,7 +24,8 @@ supersuit==3.7.0; python_version >= '3.7' # For tests on minigrid. minigrid==2.1.1 # For tests on RecSim and Kaggle envs. -recsim==0.2.4 +# Explicitly depends on `tensorflow` and doesn't accept `tensorflow-macos` +recsim==0.2.4; sys_platform != 'darwin' or platform_machine != 'arm64' tensorflow_estimator==2.11.0 # DeepMind's OpenSpiel open-spiel==1.2 @@ -37,8 +38,10 @@ higher==0.2.1 pyglet==1.5.15 imageio-ffmpeg==0.4.5 # ONNX -onnx==1.12.0 -onnxruntime==1.14.0 -tf2onnx==1.13.0 +# ONNX 1.13.0 depends on protobuf > 3.20, conflicting with tensorflow. +# ONNX 1.12.0 is not published for mac arm64, so we exclude it for now. +onnx==1.12.0; sys_platform != 'darwin' or platform_machine != 'arm64' +onnxruntime==1.14.1; sys_platform != 'darwin' or platform_machine != 'arm64' +tf2onnx==1.13.0; sys_platform != 'darwin' or platform_machine != 'arm64' typer==0.6.1 rich==12.0.1 diff --git a/python/requirements/ml/requirements_train.txt b/python/requirements/ml/requirements_train.txt index 0c10b988a1f5b..f539e7c2b1a94 100644 --- a/python/requirements/ml/requirements_train.txt +++ b/python/requirements/ml/requirements_train.txt @@ -2,7 +2,7 @@ -r requirements_dl.txt -mosaicml==0.10.1 +mosaicml==0.12.1 mlflow==1.30.0 tensorboardX==2.4.1 diff --git a/python/requirements/ml/requirements_tune.txt b/python/requirements/ml/requirements_tune.txt index 839c940b75087..8891aad90bbde 100644 --- a/python/requirements/ml/requirements_tune.txt +++ b/python/requirements/ml/requirements_tune.txt @@ -9,18 +9,21 @@ ConfigSpace==0.4.18 dragonfly-opt==0.1.6 flaml==1.1.1 freezegun==1.1.0 -gluoncv==0.10.1.post0 +# Requires decord which is unavailable for arm64 +gluoncv==0.10.1.post0; platform_machine != "arm64" gpy==1.10.0 -autorom[accept-rom-license] +# Requires libtorrent which is unavailable for arm64 +autorom[accept-rom-license]; platform_machine != "arm64" h5py==3.7.0 hpbandster==0.7.4 HEBO==0.3.2 hyperopt==0.2.5 jupyterlab==3.6.1 -lightgbm==3.2.1 +lightgbm==3.3.5 matplotlib!=3.4.3 mlflow==1.30.0 -mxnet==1.8.0.post0 +# Unavailable for arm64 in more recent versions +mxnet==1.8.0.post0; platform_machine != "arm64" nevergrad==0.4.3.post7 optuna==2.10.0 # For HEBO compatibility @@ -36,5 +39,6 @@ timm==0.4.5 transformers==4.18.0; python_version <= '3.6' transformers==4.19.1; python_version > '3.6' wandb==0.13.4 -xgboost==1.3.3 +xgboost==1.6.2; python_version <= '3.7' +xgboost==1.7.4; python_version > '3.7' zoopt==0.4.1 diff --git a/python/setup.py b/python/setup.py index a4e5c4c721f05..8330995949769 100644 --- a/python/setup.py +++ b/python/setup.py @@ -18,6 +18,15 @@ from enum import Enum from itertools import chain +# Workaround for setuptools_scm (used on macos) adding junk files +# https://stackoverflow.com/a/61274968/8162137 +try: + import setuptools_scm.integration + + setuptools_scm.integration.find_files = lambda _: [] +except ImportError: + pass + logger = logging.getLogger(__name__) SUPPORTED_PYTHONS = [(3, 6), (3, 7), (3, 8), (3, 9), (3, 10), (3, 11)] @@ -101,7 +110,7 @@ def __init__( def get_packages(self): if self.type == SetupType.RAY: - return setuptools.find_packages() + return setuptools.find_packages(exclude=("tests", "*.tests", "*.tests.*")) else: return [] @@ -789,6 +798,11 @@ def has_ext_modules(self): "ray": ["includes/*.pxd", "*.pxd"], }, include_package_data=True, + exclude_package_data={ + # Empty string means "any package". + # Therefore, exclude BUILD from every package: + "": ["BUILD"], + }, zip_safe=False, license="Apache 2.0", ) if __name__ == "__main__" else None diff --git a/release/air_examples/dreambooth/dreambooth b/release/air_examples/dreambooth/dreambooth new file mode 120000 index 0000000000000..362a68d08d3f5 --- /dev/null +++ b/release/air_examples/dreambooth/dreambooth @@ -0,0 +1 @@ +../../../python/ray/air/examples/dreambooth \ No newline at end of file diff --git a/release/air_examples/dreambooth/dreambooth_compute.yaml b/release/air_examples/dreambooth/dreambooth_compute.yaml new file mode 100644 index 0000000000000..ce81b50f9fc64 --- /dev/null +++ b/release/air_examples/dreambooth/dreambooth_compute.yaml @@ -0,0 +1,10 @@ +cloud_id: {{env["ANYSCALE_CLOUD_ID"]}} +region: us-west-2 + +max_workers: 0 + +head_node_type: + name: head_node + instance_type: g5.12xlarge + +worker_node_types: [] diff --git a/release/air_examples/dreambooth/dreambooth_env.yaml b/release/air_examples/dreambooth/dreambooth_env.yaml new file mode 100644 index 0000000000000..e11714bc99f75 --- /dev/null +++ b/release/air_examples/dreambooth/dreambooth_env.yaml @@ -0,0 +1,12 @@ +base_image: {{ env["RAY_IMAGE_ML_NIGHTLY_GPU"] | default("anyscale/ray-ml:nightly-py37-gpu") }} +env_vars: {} +debian_packages: + - curl + +python: + pip_packages: [] + conda_packages: [] + +post_build_cmds: + - pip uninstall -y ray || true && pip3 install -U {{ env["RAY_WHEELS"] | default("ray") }} + - {{ env["RAY_WHEELS_SANITY_CHECK"] | default("echo No Ray wheels sanity check") }} diff --git a/release/air_examples/dreambooth/dreambooth_run.sh b/release/air_examples/dreambooth/dreambooth_run.sh new file mode 100644 index 0000000000000..561f01e9ff4e0 --- /dev/null +++ b/release/air_examples/dreambooth/dreambooth_run.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +# ATTN: This should be kept in sync with python/ray/air/examples/dreambooth/README.md + +set -xe + +# Step 0 +pushd dreambooth || true +pip install -Ur requirements.txt + +# Step 0 cont +export DATA_PREFIX="/tmp" +export ORIG_MODEL_NAME="CompVis/stable-diffusion-v1-4" +export ORIG_MODEL_HASH="249dd2d739844dea6a0bc7fc27b3c1d014720b28" +export ORIG_MODEL_DIR="$DATA_PREFIX/model-orig" +export ORIG_MODEL_PATH="$ORIG_MODEL_DIR/models--${ORIG_MODEL_NAME/\//--}/snapshots/$ORIG_MODEL_HASH" +export TUNED_MODEL_DIR="$DATA_PREFIX/model-tuned" +export IMAGES_REG_DIR="$DATA_PREFIX/images-reg" +export IMAGES_OWN_DIR="$DATA_PREFIX/images-own" +export IMAGES_NEW_DIR="$DATA_PREFIX/images-new" + +export CLASS_NAME="lego car" + +mkdir -p $ORIG_MODEL_DIR $TUNED_MODEL_DIR $IMAGES_REG_DIR $IMAGES_OWN_DIR $IMAGES_NEW_DIR + +# Copy own images into IMAGES_OWN_DIR +cp -rf ./images/unqtkn/*.jpg "$IMAGES_OWN_DIR/" + +# Step 1 +python cache_model.py --model_dir=$ORIG_MODEL_DIR --model_name=$ORIG_MODEL_NAME --revision=$ORIG_MODEL_HASH + +# Clear reg dir +rm -rf "$IMAGES_REG_DIR"/*.jpg + +# Step 2 +# ATTN: Reduced the number of samples per prompt for faster testing +python run_model.py \ + --model_dir=$ORIG_MODEL_PATH \ + --output_dir=$IMAGES_REG_DIR \ + --prompts="photo of a $CLASS_NAME" \ + --num_samples_per_prompt=20 + +# Step 3 +python train.py \ + --model_dir=$ORIG_MODEL_PATH \ + --output_dir=$TUNED_MODEL_DIR \ + --instance_images_dir=$IMAGES_OWN_DIR \ + --instance_prompt="a photo of unqtkn $CLASS_NAME" \ + --class_images_dir=$IMAGES_REG_DIR \ + --class_prompt="a photo of a $CLASS_NAME" + +# Clear new dir +rm -rf "$IMAGES_NEW_DIR"/*.jpg + +# Step 4 +# ATTN: Reduced the number of samples per prompt for faster testing +python run_model.py \ + --model_dir=$TUNED_MODEL_DIR \ + --output_dir=$IMAGES_NEW_DIR \ + --prompts="photo of a unqtkn $CLASS_NAME" \ + --num_samples_per_prompt=5 + +# Save artifact +mkdir -p /tmp/artifacts +cp -f "$IMAGES_NEW_DIR"/0-*.jpg /tmp/artifacts/example_out.jpg + +# Exit +popd || true diff --git a/release/air_examples/gptj_deepspeed_finetuning/gptj_deepspeed_compute.yaml b/release/air_examples/gptj_deepspeed_finetuning/gptj_deepspeed_compute.yaml new file mode 100644 index 0000000000000..f5c461c1618b1 --- /dev/null +++ b/release/air_examples/gptj_deepspeed_finetuning/gptj_deepspeed_compute.yaml @@ -0,0 +1,20 @@ +cloud_id: {{env["ANYSCALE_CLOUD_ID"]}} +region: us-west-2 + +head_node_type: + name: head_node + instance_type: g4dn.4xlarge + +worker_node_types: + - name: worker_node + instance_type: g4dn.4xlarge + min_workers: 15 + max_workers: 15 + use_spot: false + +aws: + TagSpecifications: + - ResourceType: "instance" + Tags: + - Key: ttl-hours + Value: '24' diff --git a/release/air_examples/gptj_deepspeed_finetuning/gptj_deepspeed_env.yaml b/release/air_examples/gptj_deepspeed_finetuning/gptj_deepspeed_env.yaml new file mode 100644 index 0000000000000..cbecd450c8496 --- /dev/null +++ b/release/air_examples/gptj_deepspeed_finetuning/gptj_deepspeed_env.yaml @@ -0,0 +1,21 @@ +base_image: {{ env["RAY_IMAGE_ML_NIGHTLY_GPU"] | default("anyscale/ray-ml:nightly-py37-gpu") }} +env_vars: {} +debian_packages: + - curl + +python: + pip_packages: + - "datasets" + - "evaluate" + - "accelerate>=0.16.0" + - "transformers>=4.26.0" + - "torch>=1.12.0" + - "deepspeed" + - myst-parser==0.15.2 + - myst-nb==0.13.1 + - jupytext==1.13.6 + conda_packages: [] + +post_build_cmds: + - pip uninstall -y ray || true && pip3 install -U {{ env["RAY_WHEELS"] | default("ray") }} + - {{ env["RAY_WHEELS_SANITY_CHECK"] | default("echo No Ray wheels sanity check") }} \ No newline at end of file diff --git a/release/air_examples/gptj_deepspeed_finetuning/gptj_deepspeed_fine_tuning.ipynb b/release/air_examples/gptj_deepspeed_finetuning/gptj_deepspeed_fine_tuning.ipynb new file mode 120000 index 0000000000000..a65044dfacf95 --- /dev/null +++ b/release/air_examples/gptj_deepspeed_finetuning/gptj_deepspeed_fine_tuning.ipynb @@ -0,0 +1 @@ +../../../doc/source/ray-air/examples/gptj_deepspeed_fine_tuning.ipynb \ No newline at end of file diff --git a/release/air_examples/gptj_deepspeed_finetuning/test_myst_doc.py b/release/air_examples/gptj_deepspeed_finetuning/test_myst_doc.py new file mode 120000 index 0000000000000..c265ccc7b062b --- /dev/null +++ b/release/air_examples/gptj_deepspeed_finetuning/test_myst_doc.py @@ -0,0 +1 @@ +../../../doc/test_myst_doc.py \ No newline at end of file diff --git a/release/benchmarks/distributed/test_many_actors.py b/release/benchmarks/distributed/test_many_actors.py index f56dd0e910b71..a40d8a583b095 100644 --- a/release/benchmarks/distributed/test_many_actors.py +++ b/release/benchmarks/distributed/test_many_actors.py @@ -1,4 +1,3 @@ -import json import os import ray import ray._private.test_utils as test_utils @@ -63,23 +62,21 @@ def no_resource_leaks(): f"{end_time - start_time}s. ({rate} actors/s)" ) -if "TEST_OUTPUT_JSON" in os.environ: - out_file = open(os.environ["TEST_OUTPUT_JSON"], "w") - results = { - "actors_per_second": rate, - "num_actors": MAX_ACTORS_IN_CLUSTER, - "time": end_time - start_time, - "success": "1", - "_peak_memory": round(used_gb, 2), - "_peak_process_memory": usage, - } - if not is_smoke_test: - results["perf_metrics"] = [ - { - "perf_metric_name": "actors_per_second", - "perf_metric_value": rate, - "perf_metric_type": "THROUGHPUT", - } - ] - dashboard_test.update_release_test_result(results) - json.dump(results, out_file) +results = { + "actors_per_second": rate, + "num_actors": MAX_ACTORS_IN_CLUSTER, + "time": end_time - start_time, + "success": "1", + "_peak_memory": round(used_gb, 2), + "_peak_process_memory": usage, +} +if not is_smoke_test: + results["perf_metrics"] = [ + { + "perf_metric_name": "actors_per_second", + "perf_metric_value": rate, + "perf_metric_type": "THROUGHPUT", + } + ] +dashboard_test.update_release_test_result(results) +test_utils.safe_write_to_results_json(results) diff --git a/release/benchmarks/distributed/test_many_pgs.py b/release/benchmarks/distributed/test_many_pgs.py index e6993a002e2cb..40029bcf601d0 100644 --- a/release/benchmarks/distributed/test_many_pgs.py +++ b/release/benchmarks/distributed/test_many_pgs.py @@ -1,4 +1,3 @@ -import json import os import ray import ray._private.test_utils as test_utils @@ -100,23 +99,21 @@ def no_resource_leaks(): f"{end_time - start_time}s. ({rate} pgs/s)" ) -if "TEST_OUTPUT_JSON" in os.environ: - out_file = open(os.environ["TEST_OUTPUT_JSON"], "w") - results = { - "pgs_per_second": rate, - "num_pgs": MAX_PLACEMENT_GROUPS, - "time": end_time - start_time, - "success": "1", - "_peak_memory": round(used_gb, 2), - "_peak_process_memory": usage, - } - if not is_smoke_test: - results["perf_metrics"] = [ - { - "perf_metric_name": "pgs_per_second", - "perf_metric_value": rate, - "perf_metric_type": "THROUGHPUT", - } - ] - dashboard_test.update_release_test_result(results) - json.dump(results, out_file) +results = { + "pgs_per_second": rate, + "num_pgs": MAX_PLACEMENT_GROUPS, + "time": end_time - start_time, + "success": "1", + "_peak_memory": round(used_gb, 2), + "_peak_process_memory": usage, +} +if not is_smoke_test: + results["perf_metrics"] = [ + { + "perf_metric_name": "pgs_per_second", + "perf_metric_value": rate, + "perf_metric_type": "THROUGHPUT", + } + ] +dashboard_test.update_release_test_result(results) +test_utils.safe_write_to_results_json(results) diff --git a/release/benchmarks/distributed/test_many_tasks.py b/release/benchmarks/distributed/test_many_tasks.py index c11ad908d4652..365d27e355ab3 100644 --- a/release/benchmarks/distributed/test_many_tasks.py +++ b/release/benchmarks/distributed/test_many_tasks.py @@ -1,6 +1,4 @@ import click -import json -import os import ray import ray._private.test_utils as test_utils import time @@ -67,14 +65,7 @@ def no_resource_leaks(): @click.command() @click.option("--num-tasks", required=True, type=int, help="Number of tasks to launch.") -@click.option( - "--smoke-test", - is_flag=True, - type=bool, - default=False, - help="If set, it's a smoke test", -) -def test(num_tasks, smoke_test): +def test(num_tasks): addr = ray.init(address="auto") test_utils.wait_for_condition(no_resource_leaks) @@ -109,32 +100,30 @@ def not_none(res): f"({rate} tasks/s)" ) - if "TEST_OUTPUT_JSON" in os.environ: - out_file = open(os.environ["TEST_OUTPUT_JSON"], "w") - results = { - "tasks_per_second": rate, - "num_tasks": num_tasks, - "time": end_time - start_time, - "used_cpus": used_cpus, - "success": "1", - "_peak_memory": round(used_gb, 2), - "_peak_process_memory": usage, - } - if not smoke_test: - results["perf_metrics"] = [ - { - "perf_metric_name": "tasks_per_second", - "perf_metric_value": rate, - "perf_metric_type": "THROUGHPUT", - }, - { - "perf_metric_name": "used_cpus_by_deadline", - "perf_metric_value": used_cpus, - "perf_metric_type": "THROUGHPUT", - }, - ] - dashboard_test.update_release_test_result(results) - json.dump(results, out_file) + results = { + "tasks_per_second": rate, + "num_tasks": num_tasks, + "time": end_time - start_time, + "used_cpus": used_cpus, + "success": "1", + "_peak_memory": round(used_gb, 2), + "_peak_process_memory": usage, + "perf_metrics": [ + { + "perf_metric_name": "tasks_per_second", + "perf_metric_value": rate, + "perf_metric_type": "THROUGHPUT", + }, + { + "perf_metric_name": "used_cpus_by_deadline", + "perf_metric_value": used_cpus, + "perf_metric_type": "THROUGHPUT", + }, + ], + } + + dashboard_test.update_release_test_result(results) + test_utils.safe_write_to_results_json(results) if __name__ == "__main__": diff --git a/release/benchmarks/distributed/test_scheduling.py b/release/benchmarks/distributed/test_scheduling.py index 0ea75f2a9ed80..2e4dd138d7a7f 100644 --- a/release/benchmarks/distributed/test_scheduling.py +++ b/release/benchmarks/distributed/test_scheduling.py @@ -2,8 +2,7 @@ import argparse from time import time, sleep from math import floor -import os -import json +from ray._private.test_utils import safe_write_to_results_json @ray.remote @@ -105,8 +104,6 @@ def start_actor(num_actors, num_actors_per_nodes, job): args.total_num_actors, args.num_actors_per_nodes, job ) - output = os.environ.get("TEST_OUTPUT_JSON") - result = { "total_num_task": args.total_num_task, "num_cpu_per_task": args.num_cpu_per_task, @@ -121,10 +118,6 @@ def start_actor(num_actors, num_actors_per_nodes, job): "_runtime": submission_cost + ready_cost + actor_job_cost, } - if output is not None: - from pathlib import Path - - p = Path(output) - p.write_text(json.dumps(result)) + safe_write_to_results_json(result) print(result) diff --git a/release/long_running_tests/app_config.yaml b/release/long_running_tests/app_config.yaml index 6a6bf3aba6af2..5d13555ff713a 100755 --- a/release/long_running_tests/app_config.yaml +++ b/release/long_running_tests/app_config.yaml @@ -1,5 +1,5 @@ base_image: {{ env["RAY_IMAGE_NIGHTLY_CPU"] | default("anyscale/ray:nightly-py37") }} -env_vars: {} +env_vars: {"RLLIB_TEST_NO_JAX_IMPORT": "1"} debian_packages: - curl @@ -7,15 +7,13 @@ debian_packages: python: pip_packages: - # These dependencies should be handled by requirements_rllib.txt and - # requirements_ml_docker.txt and removed here + # These dependencies should be handled by requirements_rllib.txt and requirements_ml_docker.txt and removed here - gym>=0.21.0,<0.24.1 - ale-py==0.7.5 - pytest - tensorflow - # AutoROM downloads ROMs via torrent when they are built. The torrent is unreliable, - # so we built it for py3 and use that instead. This wheel was tested for python 3.7, 3.8, - # and 3.9. + # AutoROM downloads ROMs via torrent when they are built. The torrent is unreliable, so we built it for py3 and + # use that instead. This wheel was tested for python 3.7, 3.8, and 3.9. - https://ray-ci-deps-wheels.s3.us-west-2.amazonaws.com/AutoROM.accept_rom_license-0.5.4-py3-none-any.whl conda_packages: [] diff --git a/release/ml_user_tests/horovod/horovod_user_test.py b/release/ml_user_tests/horovod/horovod_user_test.py index 3355d3e21826e..69dcf71b8fffd 100644 --- a/release/ml_user_tests/horovod/horovod_user_test.py +++ b/release/ml_user_tests/horovod/horovod_user_test.py @@ -22,6 +22,7 @@ num_workers=6, use_gpu=True, placement_group_timeout_s=2000, + timeout_s=120, kwargs={"num_epochs": 20}, ) diff --git a/release/nightly_tests/dataset/pipelined_training.py b/release/nightly_tests/dataset/pipelined_training.py index 49fb50014f8d9..69adfc7dfa7fb 100644 --- a/release/nightly_tests/dataset/pipelined_training.py +++ b/release/nightly_tests/dataset/pipelined_training.py @@ -327,7 +327,7 @@ def consume(split, rank=None, batch_size=None): ray.get(tasks) else: print("Create Ray executor") - settings = RayExecutor.create_settings(timeout_s=30) + settings = RayExecutor.create_settings(timeout_s=120) executor = RayExecutor(settings, num_workers=args.num_workers, use_gpu=True) executor.start() executor.run(train_main, args=[args, splits]) diff --git a/release/release_logs/2.3.0/benchmarks/many_actors.json b/release/release_logs/2.3.0/benchmarks/many_actors.json new file mode 100644 index 0000000000000..4f63c20bdc67d --- /dev/null +++ b/release/release_logs/2.3.0/benchmarks/many_actors.json @@ -0,0 +1,32 @@ +{ + "_dashboard_memory_usage_mb": 518.770688, + "_dashboard_test_success": true, + "_peak_memory": 3.58, + "_peak_process_memory": "PID\tMEM\tCOMMAND\n291\t1.75GiB\t/home/ray/anaconda3/lib/python3.7/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n2427\t0.85GiB\tpython distributed/test_many_actors.py\n464\t0.31GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/lib/python3.7/site-packages/ray/dashboard/dashboa\n47\t0.09GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/bin/anyscale session web_terminal_server --deploy\n764\t0.08GiB\t/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.7/site-packages/ray/dashboard/agen\n2648\t0.07GiB\tray::DashboardTester.run\n44\t0.06GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/bin/jupyter-notebook --NotebookApp.token=aph0_CkY\n2564\t0.06GiB\tray::MemoryMonitorActor.run\n391\t0.05GiB\t/home/ray/anaconda3/bin/python -m ray.util.client.server --address=172.31.206.170:9031 --host=0.0.0.\n663\t0.05GiB\t/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.7/site-packages/ray/_private/log_m", + "actors_per_second": 777.2062125146554, + "num_actors": 10000, + "perf_metrics": [ + { + "perf_metric_name": "actors_per_second", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 777.2062125146554 + }, + { + "perf_metric_name": "dashboard_p50_latency_ms", + "perf_metric_type": "LATENCY", + "perf_metric_value": 30.356 + }, + { + "perf_metric_name": "dashboard_p95_latency_ms", + "perf_metric_type": "LATENCY", + "perf_metric_value": 1313.017 + }, + { + "perf_metric_name": "dashboard_p99_latency_ms", + "perf_metric_type": "LATENCY", + "perf_metric_value": 4925.524 + } + ], + "success": "1", + "time": 12.866598129272461 +} diff --git a/release/release_logs/2.3.0/benchmarks/many_nodes.json b/release/release_logs/2.3.0/benchmarks/many_nodes.json new file mode 100644 index 0000000000000..3717336d2fbd1 --- /dev/null +++ b/release/release_logs/2.3.0/benchmarks/many_nodes.json @@ -0,0 +1,38 @@ +{ + "_dashboard_memory_usage_mb": 253.00992, + "_dashboard_test_success": true, + "_peak_memory": 1.92, + "_peak_process_memory": "PID\tMEM\tCOMMAND\n291\t0.74GiB\t/home/ray/anaconda3/lib/python3.7/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n464\t0.22GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/lib/python3.7/site-packages/ray/dashboard/dashboa\n3086\t0.2GiB\tpython distributed/test_many_tasks.py --num-tasks=1000\n47\t0.09GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/bin/anyscale session web_terminal_server --deploy\n764\t0.08GiB\t/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.7/site-packages/ray/dashboard/agen\n3386\t0.07GiB\tray::StateAPIGeneratorActor.start\n3308\t0.07GiB\tray::DashboardTester.run\n3221\t0.07GiB\tray::MemoryMonitorActor.run\n44\t0.06GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/bin/jupyter-notebook --NotebookApp.token=agh0_Ckg\n391\t0.05GiB\t/home/ray/anaconda3/bin/python -m ray.util.client.server --address=172.31.167.22:9031 --host=0.0.0.0", + "num_tasks": 1000, + "perf_metrics": [ + { + "perf_metric_name": "tasks_per_second", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 219.0605760959142 + }, + { + "perf_metric_name": "used_cpus_by_deadline", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 250.0 + }, + { + "perf_metric_name": "dashboard_p50_latency_ms", + "perf_metric_type": "LATENCY", + "perf_metric_value": 3.356 + }, + { + "perf_metric_name": "dashboard_p95_latency_ms", + "perf_metric_type": "LATENCY", + "perf_metric_value": 44.061 + }, + { + "perf_metric_name": "dashboard_p99_latency_ms", + "perf_metric_type": "LATENCY", + "perf_metric_value": 152.793 + } + ], + "success": "1", + "tasks_per_second": 219.0605760959142, + "time": 304.5649473667145, + "used_cpus": 250.0 +} diff --git a/release/release_logs/2.3.0/benchmarks/many_pgs.json b/release/release_logs/2.3.0/benchmarks/many_pgs.json new file mode 100644 index 0000000000000..1c78ccfa44d5b --- /dev/null +++ b/release/release_logs/2.3.0/benchmarks/many_pgs.json @@ -0,0 +1,32 @@ +{ + "_dashboard_memory_usage_mb": 241.78688, + "_dashboard_test_success": true, + "_peak_memory": 2.37, + "_peak_process_memory": "PID\tMEM\tCOMMAND\n291\t1.07GiB\t/home/ray/anaconda3/lib/python3.7/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n2413\t0.4GiB\tpython distributed/test_many_pgs.py\n464\t0.21GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/lib/python3.7/site-packages/ray/dashboard/dashboa\n47\t0.09GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/bin/anyscale session web_terminal_server --deploy\n590\t0.08GiB\t/home/ray/anaconda3/lib/python3.7/site-packages/ray/core/src/ray/raylet/raylet --raylet_socket_name=\n764\t0.08GiB\t/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.7/site-packages/ray/dashboard/agen\n2551\t0.06GiB\tray::MemoryMonitorActor.run\n2634\t0.06GiB\tray::DashboardTester.run\n44\t0.06GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/bin/jupyter-notebook --NotebookApp.token=aph0_Ckc\n391\t0.05GiB\t/home/ray/anaconda3/bin/python -m ray.util.client.server --address=172.31.255.206:9031 --host=0.0.0.", + "num_pgs": 1000, + "perf_metrics": [ + { + "perf_metric_name": "pgs_per_second", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 17.89601570164703 + }, + { + "perf_metric_name": "dashboard_p50_latency_ms", + "perf_metric_type": "LATENCY", + "perf_metric_value": 3.192 + }, + { + "perf_metric_name": "dashboard_p95_latency_ms", + "perf_metric_type": "LATENCY", + "perf_metric_value": 12.196 + }, + { + "perf_metric_name": "dashboard_p99_latency_ms", + "perf_metric_type": "LATENCY", + "perf_metric_value": 149.699 + } + ], + "pgs_per_second": 17.89601570164703, + "success": "1", + "time": 55.87835955619812 +} diff --git a/release/release_logs/2.3.0/benchmarks/many_tasks.json b/release/release_logs/2.3.0/benchmarks/many_tasks.json new file mode 100644 index 0000000000000..133326ff16c85 --- /dev/null +++ b/release/release_logs/2.3.0/benchmarks/many_tasks.json @@ -0,0 +1,38 @@ +{ + "_dashboard_memory_usage_mb": 498.393088, + "_dashboard_test_success": true, + "_peak_memory": 4.39, + "_peak_process_memory": "PID\tMEM\tCOMMAND\n291\t2.09GiB\t/home/ray/anaconda3/lib/python3.7/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n2422\t0.86GiB\tpython distributed/test_many_tasks.py --num-tasks=10000\n464\t0.59GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/lib/python3.7/site-packages/ray/dashboard/dashboa\n2644\t0.1GiB\tray::DashboardTester.run\n47\t0.09GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/bin/anyscale session web_terminal_server --deploy\n765\t0.08GiB\t/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.7/site-packages/ray/dashboard/agen\n2710\t0.07GiB\tray::StateAPIGeneratorActor.start\n2560\t0.06GiB\tray::MemoryMonitorActor.run\n44\t0.06GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/bin/jupyter-notebook --NotebookApp.token=aph0_CkY\n391\t0.05GiB\t/home/ray/anaconda3/bin/python -m ray.util.client.server --address=172.31.223.72:9031 --host=0.0.0.0", + "num_tasks": 10000, + "perf_metrics": [ + { + "perf_metric_name": "tasks_per_second", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 210.67184928063165 + }, + { + "perf_metric_name": "used_cpus_by_deadline", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 2500.0 + }, + { + "perf_metric_name": "dashboard_p50_latency_ms", + "perf_metric_type": "LATENCY", + "perf_metric_value": 3.985 + }, + { + "perf_metric_name": "dashboard_p95_latency_ms", + "perf_metric_type": "LATENCY", + "perf_metric_value": 665.858 + }, + { + "perf_metric_name": "dashboard_p99_latency_ms", + "perf_metric_type": "LATENCY", + "perf_metric_value": 1404.251 + } + ], + "success": "1", + "tasks_per_second": 210.67184928063165, + "time": 347.46718668937683, + "used_cpus": 2500.0 +} diff --git a/release/release_logs/2.3.0/microbenchmark.json b/release/release_logs/2.3.0/microbenchmark.json new file mode 100644 index 0000000000000..6da062760215f --- /dev/null +++ b/release/release_logs/2.3.0/microbenchmark.json @@ -0,0 +1,283 @@ +{ + "1_1_actor_calls_async": [ + 8266.227716717382, + 154.83431912796019 + ], + "1_1_actor_calls_concurrent": [ + 4822.515544345499, + 69.03417417619077 + ], + "1_1_actor_calls_sync": [ + 2326.919630897046, + 18.048639582599087 + ], + "1_1_async_actor_calls_async": [ + 3472.864053257764, + 18.492804155673895 + ], + "1_1_async_actor_calls_sync": [ + 1600.088775357745, + 41.89765647230385 + ], + "1_1_async_actor_calls_with_args_async": [ + 2350.1130733425607, + 169.642459588056 + ], + "1_n_actor_calls_async": [ + 10819.31324573512, + 556.3564152930876 + ], + "1_n_async_actor_calls_async": [ + 10741.070943872623, + 74.14284598217124 + ], + "client__1_1_actor_calls_async": [ + 916.2458814581453, + 20.970867830230226 + ], + "client__1_1_actor_calls_concurrent": [ + 925.777467637641, + 25.193859971158158 + ], + "client__1_1_actor_calls_sync": [ + 521.8726144223089, + 7.962424357055347 + ], + "client__get_calls": [ + 1149.982480593273, + 18.207879410048996 + ], + "client__put_calls": [ + 861.7294897745508, + 20.886660586339037 + ], + "client__put_gigabytes": [ + 0.04579083755671009, + 0.0011758702466256328 + ], + "client__tasks_and_get_batch": [ + 0.85332224821338, + 0.009728477126712553 + ], + "client__tasks_and_put_batch": [ + 10926.28867356008, + 112.00041581027524 + ], + "multi_client_put_calls_Plasma_Store": [ + 11907.518691049167, + 140.67822025073386 + ], + "multi_client_put_gigabytes": [ + 38.34929639975205, + 3.8021163438852286 + ], + "multi_client_tasks_async": [ + 29213.877508157526, + 1402.303220638465 + ], + "n_n_actor_calls_async": [ + 32206.022554465333, + 1389.5479846707906 + ], + "n_n_actor_calls_with_arg_async": [ + 2569.4536725239686, + 27.598753015224556 + ], + "n_n_async_actor_calls_async": [ + 27823.30349707046, + 169.7830788090935 + ], + "perf_metrics": [ + { + "perf_metric_name": "single_client_get_calls_Plasma_Store", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 6507.325737010699 + }, + { + "perf_metric_name": "single_client_put_calls_Plasma_Store", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 5657.281266012957 + }, + { + "perf_metric_name": "multi_client_put_calls_Plasma_Store", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 11907.518691049167 + }, + { + "perf_metric_name": "single_client_put_gigabytes", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 20.68898688869432 + }, + { + "perf_metric_name": "single_client_tasks_and_get_batch", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 11.039401889292254 + }, + { + "perf_metric_name": "multi_client_put_gigabytes", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 38.34929639975205 + }, + { + "perf_metric_name": "single_client_get_object_containing_10k_refs", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 13.146682237214058 + }, + { + "perf_metric_name": "single_client_wait_1k_refs", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 5.498008946220131 + }, + { + "perf_metric_name": "single_client_tasks_sync", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 1282.1050579108664 + }, + { + "perf_metric_name": "single_client_tasks_async", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 10781.5375768007 + }, + { + "perf_metric_name": "multi_client_tasks_async", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 29213.877508157526 + }, + { + "perf_metric_name": "1_1_actor_calls_sync", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 2326.919630897046 + }, + { + "perf_metric_name": "1_1_actor_calls_async", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 8266.227716717382 + }, + { + "perf_metric_name": "1_1_actor_calls_concurrent", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 4822.515544345499 + }, + { + "perf_metric_name": "1_n_actor_calls_async", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 10819.31324573512 + }, + { + "perf_metric_name": "n_n_actor_calls_async", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 32206.022554465333 + }, + { + "perf_metric_name": "n_n_actor_calls_with_arg_async", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 2569.4536725239686 + }, + { + "perf_metric_name": "1_1_async_actor_calls_sync", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 1600.088775357745 + }, + { + "perf_metric_name": "1_1_async_actor_calls_async", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 3472.864053257764 + }, + { + "perf_metric_name": "1_1_async_actor_calls_with_args_async", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 2350.1130733425607 + }, + { + "perf_metric_name": "1_n_async_actor_calls_async", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 10741.070943872623 + }, + { + "perf_metric_name": "n_n_async_actor_calls_async", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 27823.30349707046 + }, + { + "perf_metric_name": "placement_group_create/removal", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 989.8353575694138 + }, + { + "perf_metric_name": "client__get_calls", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 1149.982480593273 + }, + { + "perf_metric_name": "client__put_calls", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 861.7294897745508 + }, + { + "perf_metric_name": "client__put_gigabytes", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 0.04579083755671009 + }, + { + "perf_metric_name": "client__tasks_and_put_batch", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 10926.28867356008 + }, + { + "perf_metric_name": "client__1_1_actor_calls_sync", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 521.8726144223089 + }, + { + "perf_metric_name": "client__1_1_actor_calls_async", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 916.2458814581453 + }, + { + "perf_metric_name": "client__1_1_actor_calls_concurrent", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 925.777467637641 + }, + { + "perf_metric_name": "client__tasks_and_get_batch", + "perf_metric_type": "THROUGHPUT", + "perf_metric_value": 0.85332224821338 + } + ], + "placement_group_create/removal": [ + 989.8353575694138, + 8.999610265563307 + ], + "single_client_get_calls_Plasma_Store": [ + 6507.325737010699, + 169.10049512002783 + ], + "single_client_get_object_containing_10k_refs": [ + 13.146682237214058, + 0.40574499297791244 + ], + "single_client_put_calls_Plasma_Store": [ + 5657.281266012957, + 65.81470332190243 + ], + "single_client_put_gigabytes": [ + 20.68898688869432, + 6.118551705446652 + ], + "single_client_tasks_and_get_batch": [ + 11.039401889292254, + 0.07053691123976474 + ], + "single_client_tasks_async": [ + 10781.5375768007, + 80.41585546336292 + ], + "single_client_tasks_sync": [ + 1282.1050579108664, + 25.53646612884218 + ], + "single_client_wait_1k_refs": [ + 5.498008946220131, + 0.019177007797266794 + ] +} diff --git a/release/release_logs/2.3.0/scalability/object_store.json b/release/release_logs/2.3.0/scalability/object_store.json new file mode 100644 index 0000000000000..5d16a4b8ee474 --- /dev/null +++ b/release/release_logs/2.3.0/scalability/object_store.json @@ -0,0 +1,13 @@ +{ + "broadcast_time": 92.42755234200001, + "num_nodes": 50, + "object_size": 1073741824, + "perf_metrics": [ + { + "perf_metric_name": "time_to_broadcast_1073741824_bytes_to_50_nodes", + "perf_metric_type": "LATENCY", + "perf_metric_value": 92.42755234200001 + } + ], + "success": "1" +} diff --git a/release/release_logs/2.3.0/scalability/single_node.json b/release/release_logs/2.3.0/scalability/single_node.json new file mode 100644 index 0000000000000..bababa5f97d9d --- /dev/null +++ b/release/release_logs/2.3.0/scalability/single_node.json @@ -0,0 +1,40 @@ +{ + "args_time": 16.49447203899996, + "get_time": 24.665624744000013, + "large_object_size": 107374182400, + "large_object_time": 325.56598805499993, + "num_args": 10000, + "num_get_args": 10000, + "num_queued": 1000000, + "num_returns": 3000, + "perf_metrics": [ + { + "perf_metric_name": "10000_args_time", + "perf_metric_type": "LATENCY", + "perf_metric_value": 16.49447203899996 + }, + { + "perf_metric_name": "3000_returns_time", + "perf_metric_type": "LATENCY", + "perf_metric_value": 6.200411088999999 + }, + { + "perf_metric_name": "10000_get_time", + "perf_metric_type": "LATENCY", + "perf_metric_value": 24.665624744000013 + }, + { + "perf_metric_name": "1000000_queued_time", + "perf_metric_type": "LATENCY", + "perf_metric_value": 190.882864026 + }, + { + "perf_metric_name": "107374182400_large_object_time", + "perf_metric_type": "LATENCY", + "perf_metric_value": 325.56598805499993 + } + ], + "queued_time": 190.882864026, + "returns_time": 6.200411088999999, + "success": "1" +} diff --git a/release/release_logs/2.3.0/stress_tests/stress_test_dead_actors.json b/release/release_logs/2.3.0/stress_tests/stress_test_dead_actors.json new file mode 100644 index 0000000000000..ad110b1a960c7 --- /dev/null +++ b/release/release_logs/2.3.0/stress_tests/stress_test_dead_actors.json @@ -0,0 +1,14 @@ +{ + "avg_iteration_time": 1.9298896980285645, + "max_iteration_time": 7.609254598617554, + "min_iteration_time": 0.8152294158935547, + "perf_metrics": [ + { + "perf_metric_name": "avg_iteration_time", + "perf_metric_type": "LATENCY", + "perf_metric_value": 1.9298896980285645 + } + ], + "success": 1, + "total_time": 192.9892761707306 +} diff --git a/release/release_logs/2.3.0/stress_tests/stress_test_many_tasks.json b/release/release_logs/2.3.0/stress_tests/stress_test_many_tasks.json new file mode 100644 index 0000000000000..6e03dbd3e78c9 --- /dev/null +++ b/release/release_logs/2.3.0/stress_tests/stress_test_many_tasks.json @@ -0,0 +1,47 @@ +{ + "perf_metrics": [ + { + "perf_metric_name": "stage_0_time", + "perf_metric_type": "LATENCY", + "perf_metric_value": 10.636847257614136 + }, + { + "perf_metric_name": "stage_1_avg_iteration_time", + "perf_metric_type": "LATENCY", + "perf_metric_value": 22.78462884426117 + }, + { + "perf_metric_name": "stage_2_avg_iteration_time", + "perf_metric_type": "LATENCY", + "perf_metric_value": 63.28760423660278 + }, + { + "perf_metric_name": "stage_3_creation_time", + "perf_metric_type": "LATENCY", + "perf_metric_value": 0.05330848693847656 + }, + { + "perf_metric_name": "stage_3_time", + "perf_metric_type": "LATENCY", + "perf_metric_value": 2655.176714658737 + }, + { + "perf_metric_name": "stage_4_spread", + "perf_metric_type": "LATENCY", + "perf_metric_value": 0.7394418476633471 + } + ], + "stage_0_time": 10.636847257614136, + "stage_1_avg_iteration_time": 22.78462884426117, + "stage_1_max_iteration_time": 23.30313539505005, + "stage_1_min_iteration_time": 21.94757604598999, + "stage_1_time": 227.84637260437012, + "stage_2_avg_iteration_time": 63.28760423660278, + "stage_2_max_iteration_time": 64.29542255401611, + "stage_2_min_iteration_time": 61.036946058273315, + "stage_2_time": 316.43891644477844, + "stage_3_creation_time": 0.05330848693847656, + "stage_3_time": 2655.176714658737, + "stage_4_spread": 0.7394418476633471, + "success": 1 +} diff --git a/release/release_logs/2.3.0/stress_tests/stress_test_placement_group.json b/release/release_logs/2.3.0/stress_tests/stress_test_placement_group.json new file mode 100644 index 0000000000000..8c12e0c86876b --- /dev/null +++ b/release/release_logs/2.3.0/stress_tests/stress_test_placement_group.json @@ -0,0 +1,17 @@ +{ + "avg_pg_create_time_ms": 0.8553733138129204, + "avg_pg_remove_time_ms": 0.8286827042024993, + "perf_metrics": [ + { + "perf_metric_name": "avg_pg_create_time_ms", + "perf_metric_type": "LATENCY", + "perf_metric_value": 0.8553733138129204 + }, + { + "perf_metric_name": "avg_pg_remove_time_ms", + "perf_metric_type": "LATENCY", + "perf_metric_value": 0.8286827042024993 + } + ], + "success": 1 +} diff --git a/release/release_tests.yaml b/release/release_tests.yaml index 5e0c9149e95ec..eb19c3cf0166b 100644 --- a/release/release_tests.yaml +++ b/release/release_tests.yaml @@ -570,6 +570,46 @@ timeout: 1800 script: bash heterogeneity_benchmark.sh 2 + +####################### +# AIR examples +####################### + + +# Test additional CPU nodes for preprocessing. +- name: air_example_dreambooth_finetuning + group: AIR examples + working_dir: air_examples/dreambooth + + stable: false + + frequency: weekly + team: ml + cluster: + cluster_env: dreambooth_env.yaml + cluster_compute: dreambooth_compute.yaml + + run: + timeout: 1800 + script: bash dreambooth_run.sh + + +- name: air_example_gptj_deepspeed_fine_tuning + group: AIR examples + working_dir: air_examples/gptj_deepspeed_finetuning + + python: "3.9" + + frequency: weekly + team: ml + cluster: + cluster_env: gptj_deepspeed_env.yaml + cluster_compute: gptj_deepspeed_compute.yaml + + run: + timeout: 3600 + script: python test_myst_doc.py --path gptj_deepspeed_fine_tuning.ipynb + ####################### # XGBoost release tests ####################### @@ -3421,18 +3461,6 @@ num_nodes: 65 - smoke_test: - frequency: nightly - cluster: - cluster_env: app_config.yaml - cluster_compute: distributed_smoke_test.yaml - run: - timeout: 3600 - script: python distributed/test_many_tasks.py --num-tasks=100 - wait_for_nodes: - num_nodes: 2 - - - name: many_pgs group: core-scalability-test working_dir: benchmarks diff --git a/release/rllib_tests/app_config.yaml b/release/rllib_tests/app_config.yaml index 2dae5b40912ca..c6a77e18d845a 100755 --- a/release/rllib_tests/app_config.yaml +++ b/release/rllib_tests/app_config.yaml @@ -1,5 +1,5 @@ base_image: {{ env["RAY_IMAGE_ML_NIGHTLY_GPU"] | default("anyscale/ray-ml:nightly-py37-gpu") }} -env_vars: {"LD_LIBRARY_PATH": "$LD_LIBRARY_PATH:/home/ray/.mujoco/mujoco210/bin"} +env_vars: {"LD_LIBRARY_PATH": "$LD_LIBRARY_PATH:/home/ray/.mujoco/mujoco210/bin", "RLLIB_TEST_NO_JAX_IMPORT": "1"} debian_packages: - unzip - zip diff --git a/rllib/BUILD b/rllib/BUILD index 69d187b6a8fe0..3806403a56f90 100644 --- a/rllib/BUILD +++ b/rllib/BUILD @@ -1848,7 +1848,7 @@ py_test( py_test( name = "test_catalog", tags = ["team:rllib", "core"], - size = "small", + size = "medium", srcs = ["core/models/tests/test_catalog.py"] ) diff --git a/rllib/algorithms/algorithm.py b/rllib/algorithms/algorithm.py index bbfcdfa5623ae..44c69355c8712 100644 --- a/rllib/algorithms/algorithm.py +++ b/rllib/algorithms/algorithm.py @@ -2690,7 +2690,7 @@ def _create_local_replay_buffer_if_necessary( None, if local replay buffer is not needed. """ if not config.get("replay_buffer_config") or config["replay_buffer_config"].get( - "no_local_replay_buffer" or config.get("no_local_replay_buffer") + "no_local_replay_buffer" ): return diff --git a/rllib/algorithms/algorithm_config.py b/rllib/algorithms/algorithm_config.py index 3b0eb2146363d..0888cb428950a 100644 --- a/rllib/algorithms/algorithm_config.py +++ b/rllib/algorithms/algorithm_config.py @@ -116,6 +116,15 @@ def _resolve_class_path(module) -> Type: return getattr(module, class_name) +def _check_rl_module_spec(module_spec: ModuleSpec) -> None: + if not isinstance(module_spec, (SingleAgentRLModuleSpec, MultiAgentRLModuleSpec)): + raise ValueError( + "rl_module_spec must be an instance of " + "SingleAgentRLModuleSpec or MultiAgentRLModuleSpec." + f"Got {type(module_spec)} instead." + ) + + class AlgorithmConfig(_Config): """A RLlib AlgorithmConfig builds an RLlib Algorithm from a given configuration. @@ -901,17 +910,26 @@ def validate(self) -> None: # compatibility for now. User only needs to set num_rollout_workers. self.input_config["parallelism"] = self.num_rollout_workers or 1 - # resolve rl_module_spec class - if self._enable_rl_module_api and self.rl_module_spec is None: - self.rl_module_spec = self.get_default_rl_module_spec() - if not isinstance( - self.rl_module_spec, (SingleAgentRLModuleSpec, MultiAgentRLModuleSpec) - ): - raise ValueError( - "rl_module_spec must be an instance of " - "SingleAgentRLModuleSpec or MultiAgentRLModuleSpec." - f"Got {type(self.rl_module_spec)} instead." - ) + if self._enable_rl_module_api: + default_rl_module_spec = self.get_default_rl_module_spec() + _check_rl_module_spec(default_rl_module_spec) + + if self.rl_module_spec is not None: + # Merge provided RL Module spec class with defaults + _check_rl_module_spec(self.rl_module_spec) + # We can only merge if we have SingleAgentRLModuleSpecs. + # TODO(Artur): Support merging for MultiAgentRLModuleSpecs. + if isinstance(self.rl_module_spec, SingleAgentRLModuleSpec): + if isinstance(default_rl_module_spec, SingleAgentRLModuleSpec): + default_rl_module_spec.update(self.rl_module_spec) + self.rl_module_spec = default_rl_module_spec + elif isinstance(default_rl_module_spec, MultiAgentRLModuleSpec): + raise ValueError( + "Cannot merge MultiAgentRLModuleSpec with " + "SingleAgentRLModuleSpec!" + ) + else: + self.rl_module_spec = default_rl_module_spec # make sure the resource requirements for learner_group is valid if self.num_learner_workers == 0 and self.num_gpus_per_worker > 1: @@ -2283,9 +2301,9 @@ def rl_module( Args: rl_module_spec: The RLModule spec to use for this config. It can be either a SingleAgentRLModuleSpec or a MultiAgentRLModuleSpec. If the - observation_space, action_space, or the model_config is not specified - it will be inferred from the env and other parts of the algorithm - config object. + observation_space, action_space, catalog_class, or the model_config is + not specified it will be inferred from the env and other parts of the + algorithm config object. _enable_rl_module_api: Whether to enable the RLModule API for this config. By default if you call `config.rl_module(...)`, the RLModule API will NOT be enabled. If you want to enable it, you can call @@ -2833,8 +2851,8 @@ def get_marl_module_spec( module_spec.observation_space = policy_spec.observation_space if module_spec.action_space is None: module_spec.action_space = policy_spec.action_space - if module_spec.model_config is None: - module_spec.model_config = policy_spec.config.get("model", {}) + if module_spec.model_config_dict is None: + module_spec.model_config_dict = policy_spec.config.get("model", {}) return marl_module_spec @@ -3060,7 +3078,7 @@ def _resolve_tf_settings(self, _tf1, _tfv): # Recommend setting tracing to True for speedups. logger.info( f"Executing eagerly (framework='{self.framework_str}')," - f" with eager_tracing={self.framework_str}. For " + f" with eager_tracing={self.eager_tracing}. For " "production workloads, make sure to set eager_tracing=True" " in order to match the speed of tf-static-graph " "(framework='tf'). For debugging purposes, " diff --git a/rllib/algorithms/impala/impala.py b/rllib/algorithms/impala/impala.py index 2f1db2ed74f2f..c8d95bb931411 100644 --- a/rllib/algorithms/impala/impala.py +++ b/rllib/algorithms/impala/impala.py @@ -14,6 +14,7 @@ ImpalaHPs, _reduce_impala_results, ) +from ray.rllib.algorithms.ppo.ppo_catalog import PPOCatalog from ray.rllib.core.learner.learner_group_config import ( LearnerGroupConfig, ModuleSpec, @@ -429,7 +430,9 @@ def get_default_rl_module_spec(self) -> SingleAgentRLModuleSpec: if self.framework_str == "tf2": from ray.rllib.algorithms.ppo.tf.ppo_tf_rl_module import PPOTfRLModule - return SingleAgentRLModuleSpec(module_class=PPOTfRLModule) + return SingleAgentRLModuleSpec( + module_class=PPOTfRLModule, catalog_class=PPOCatalog + ) else: raise ValueError(f"The framework {self.framework_str} is not supported.") @@ -831,9 +834,9 @@ def get_samples_from_workers( timeout_seconds=self._timeout_s_sampler_manager, return_obj_refs=return_object_refs, ) - elif self.workers.local_worker() and ( - self.config.create_env_on_local_worker - or self.config.num_rollout_workers == 0 + elif ( + self.workers.local_worker() + and self.workers.local_worker().async_env is not None ): # Sampling from the local worker sample_batch = self.workers.local_worker().sample() diff --git a/rllib/algorithms/impala/tests/tf/test_impala_learner.py b/rllib/algorithms/impala/tests/tf/test_impala_learner.py index bef4067dd124f..61f9c4e6bb923 100644 --- a/rllib/algorithms/impala/tests/tf/test_impala_learner.py +++ b/rllib/algorithms/impala/tests/tf/test_impala_learner.py @@ -89,7 +89,8 @@ def test_impala_loss(self): module_class=algo_config.rl_module_spec.module_class, observation_space=policy.observation_space, action_space=policy.action_space, - model_config=policy.config["model"], + model_config_dict=policy.config["model"], + catalog_class=algo_config.rl_module_spec.catalog_class, ) ) learner_group_config.num_learner_workers = 0 diff --git a/rllib/algorithms/ppo/ppo.py b/rllib/algorithms/ppo/ppo.py index 8ff7e4d106024..dd53cc52b022c 100644 --- a/rllib/algorithms/ppo/ppo.py +++ b/rllib/algorithms/ppo/ppo.py @@ -17,6 +17,7 @@ from ray.rllib.algorithms.algorithm_config import AlgorithmConfig, NotProvided from ray.rllib.algorithms.pg import PGConfig from ray.rllib.algorithms.ppo.ppo_learner_config import PPOLearnerHPs +from ray.rllib.algorithms.ppo.ppo_catalog import PPOCatalog from ray.rllib.core.rl_module.rl_module import SingleAgentRLModuleSpec from ray.rllib.execution.rollout_ops import ( standardize_fields, @@ -129,11 +130,15 @@ def get_default_rl_module_spec(self) -> SingleAgentRLModuleSpec: PPOTorchRLModule, ) - return SingleAgentRLModuleSpec(module_class=PPOTorchRLModule) + return SingleAgentRLModuleSpec( + module_class=PPOTorchRLModule, catalog_class=PPOCatalog + ) elif self.framework_str == "tf2": from ray.rllib.algorithms.ppo.tf.ppo_tf_rl_module import PPOTfRLModule - return SingleAgentRLModuleSpec(module_class=PPOTfRLModule) + return SingleAgentRLModuleSpec( + module_class=PPOTfRLModule, catalog_class=PPOCatalog + ) else: raise ValueError(f"The framework {self.framework_str} is not supported.") diff --git a/rllib/algorithms/ppo/ppo_base_rl_module.py b/rllib/algorithms/ppo/ppo_base_rl_module.py index 6bed5d37ecfe5..235384e9ecc44 100644 --- a/rllib/algorithms/ppo/ppo_base_rl_module.py +++ b/rllib/algorithms/ppo/ppo_base_rl_module.py @@ -3,14 +3,11 @@ """ import abc -from typing import Mapping, Any import gymnasium as gym -from ray.rllib.algorithms.ppo.ppo_catalog import PPOCatalog -from ray.rllib.algorithms.ppo.ppo_rl_module_config import PPOModuleConfig from ray.rllib.core.rl_module.rl_module import RLModule, RLModuleConfig -from ray.rllib.utils.annotations import override, ExperimentalAPI +from ray.rllib.utils.annotations import ExperimentalAPI from ray.rllib.utils.gym import convert_old_gym_space_to_gymnasium_space from ray.rllib.core.models.base import ActorCriticEncoder @@ -20,42 +17,18 @@ class PPORLModuleBase(RLModule, abc.ABC): framework = None def __init__(self, config: RLModuleConfig): - super().__init__() - self.config = config - catalog = config.catalog - - assert isinstance(catalog, PPOCatalog), "A PPOCatalog is required for PPO." + super().__init__(config) + catalog = self.config.get_catalog() # Build models from catalog self.encoder = catalog.build_actor_critic_encoder(framework=self.framework) self.pi = catalog.build_pi_head(framework=self.framework) self.vf = catalog.build_vf_head(framework=self.framework) + self.action_dist_cls = catalog.get_action_dist_cls(framework=self.framework) + self._is_discrete = isinstance( convert_old_gym_space_to_gymnasium_space(self.config.action_space), gym.spaces.Discrete, ) assert isinstance(self.encoder, ActorCriticEncoder) - - @classmethod - @override(RLModule) - def from_model_config( - cls, - observation_space: gym.Space, - action_space: gym.Space, - *, - model_config_dict: Mapping[str, Any], - ) -> "PPORLModuleBase": - catalog = PPOCatalog( - observation_space=observation_space, - action_space=action_space, - model_config_dict=model_config_dict, - ) - - config = PPOModuleConfig( - observation_space=observation_space, - action_space=action_space, - catalog=catalog, - ) - - return config.build(framework=cls.framework) diff --git a/rllib/algorithms/ppo/ppo_catalog.py b/rllib/algorithms/ppo/ppo_catalog.py index d1c929cd7070a..57cc9934e6407 100644 --- a/rllib/algorithms/ppo/ppo_catalog.py +++ b/rllib/algorithms/ppo/ppo_catalog.py @@ -2,6 +2,7 @@ from ray.rllib.core.models.catalog import Catalog from ray.rllib.core.models.configs import ActorCriticEncoderConfig, MLPHeadConfig +from ray.rllib.core.models.base import Encoder, ActorCriticEncoder, Model from ray.rllib.utils import override @@ -54,7 +55,8 @@ def __init__( assert len(observation_space.shape) in ( 1, - ), "This simple PPO Module only supports 1D observation spaces." + 3, + ), "This simple PPO Module only supports 1D and 3D observation spaces." assert isinstance(action_space, (gym.spaces.Discrete, gym.spaces.Box)), ( "This simple PPO Module only supports Discrete and Box action spaces.", @@ -66,40 +68,27 @@ def __init__( shared=self.model_config_dict["vf_share_layers"], ) - if isinstance(action_space, gym.spaces.Discrete): - pi_output_dim = action_space.n - else: - pi_output_dim = action_space.shape[0] * 2 - post_fcnet_hiddens = self.model_config_dict["post_fcnet_hiddens"] post_fcnet_activation = self.model_config_dict["post_fcnet_activation"] self.pi_head_config = MLPHeadConfig( - input_dim=self.encoder_config.output_dim, + input_dims=self.latent_dims, hidden_layer_dims=post_fcnet_hiddens, hidden_layer_activation=post_fcnet_activation, output_activation="linear", - output_dim=pi_output_dim, + output_dims=None, # We don't know the output dimension yet, because it + # depends on the action distribution input dimension ) self.vf_head_config = MLPHeadConfig( - input_dim=self.encoder_config.output_dim, + input_dims=self.latent_dims, hidden_layer_dims=post_fcnet_hiddens, hidden_layer_activation=post_fcnet_activation, output_activation="linear", - output_dim=1, + output_dims=[1], ) - # Set input- and output dimensions to fit PPO's needs. - self.encoder_config.input_dim = observation_space.shape[0] - self.pi_head_config.input_dim = self.encoder_config.output_dim - if isinstance(action_space, gym.spaces.Discrete): - self.pi_head_config.output_dim = int(action_space.n) - else: - self.pi_head_config.output_dim = int(action_space.shape[0] * 2) - self.vf_head_config.output_dim = 1 - - def build_actor_critic_encoder(self, framework: str): + def build_actor_critic_encoder(self, framework: str) -> ActorCriticEncoder: """Builds the ActorCriticEncoder. The default behavior is to build the encoder from the encoder_config. @@ -115,7 +104,7 @@ def build_actor_critic_encoder(self, framework: str): return self.actor_critic_encoder_config.build(framework=framework) @override(Catalog) - def build_encoder(self, framework: str): + def build_encoder(self, framework: str) -> Encoder: """Builds the encoder. Since PPO uses an ActorCriticEncoder, this method should not be implemented. @@ -124,7 +113,7 @@ def build_encoder(self, framework: str): "Use PPOCatalog.build_actor_critic_encoder() instead." ) - def build_pi_head(self, framework: str): + def build_pi_head(self, framework: str) -> Model: """Builds the policy head. The default behavior is to build the head from the pi_head_config. @@ -137,9 +126,16 @@ def build_pi_head(self, framework: str): Returns: The policy head. """ + # Get action_distribution_cls to find out about the output dimension for pi_head + action_distribution_cls = self.get_action_dist_cls(framework=framework) + self.pi_head_config.output_dims = ( + action_distribution_cls.required_model_output_shape( + space=self.action_space, model_config=self.model_config_dict + ) + ) return self.pi_head_config.build(framework=framework) - def build_vf_head(self, framework: str): + def build_vf_head(self, framework: str) -> Model: """Builds the value function head. The default behavior is to build the head from the vf_head_config. diff --git a/rllib/algorithms/ppo/ppo_rl_module_config.py b/rllib/algorithms/ppo/ppo_rl_module_config.py deleted file mode 100644 index f07a87d1d99e4..0000000000000 --- a/rllib/algorithms/ppo/ppo_rl_module_config.py +++ /dev/null @@ -1,43 +0,0 @@ -from dataclasses import dataclass - -import gymnasium as gym - -from ray.rllib.core.models.catalog import Catalog -from ray.rllib.core.rl_module.rl_module import RLModuleConfig -from ray.rllib.utils.annotations import ExperimentalAPI - - -@ExperimentalAPI -@dataclass -class PPOModuleConfig(RLModuleConfig): - """Configuration for the PPORLModule. - - Attributes: - observation_space: The observation space of the environment. - action_space: The action space of the environment. - catalog: The PPOCatalog object to use for building the models. - """ - - observation_space: gym.Space = None - action_space: gym.Space = None - catalog: Catalog = None - - def build(self, framework: str): - """Builds a PPORLModule. - - Args: - framework: The framework to use for the module. - - Returns: - PPORLModule: The module. - """ - if framework == "torch": - from ray.rllib.algorithms.ppo.torch.ppo_torch_rl_module import ( - PPOTorchRLModule, - ) - - return PPOTorchRLModule(self) - else: - from ray.rllib.algorithms.ppo.tf.ppo_tf_rl_module import PPOTfRLModule - - return PPOTfRLModule(self) diff --git a/rllib/algorithms/ppo/tests/test_ppo_learner.py b/rllib/algorithms/ppo/tests/test_ppo_learner.py index a5a3482e84334..12e910ed85996 100644 --- a/rllib/algorithms/ppo/tests/test_ppo_learner.py +++ b/rllib/algorithms/ppo/tests/test_ppo_learner.py @@ -6,6 +6,7 @@ import tree # pip install dm-tree import ray.rllib.algorithms.ppo as ppo +from ray.rllib.algorithms.ppo.ppo_catalog import PPOCatalog from ray.rllib.core.rl_module.rl_module import SingleAgentRLModuleSpec from ray.rllib.policy.sample_batch import SampleBatch @@ -102,7 +103,8 @@ def test_loss(self): module_class=algo_config.rl_module_spec.module_class, observation_space=policy.observation_space, action_space=policy.action_space, - model_config=policy.config["model"], + model_config_dict=policy.config["model"], + catalog_class=PPOCatalog, ) ) learner_group = learner_group_config.build() diff --git a/rllib/algorithms/ppo/tests/test_ppo_rl_module.py b/rllib/algorithms/ppo/tests/test_ppo_rl_module.py index 0eb56db3b32fc..d648d99c5b5d9 100644 --- a/rllib/algorithms/ppo/tests/test_ppo_rl_module.py +++ b/rllib/algorithms/ppo/tests/test_ppo_rl_module.py @@ -6,17 +6,18 @@ import tensorflow as tf import torch import tree -from ray.rllib.algorithms.ppo.ppo_catalog import PPOCatalog -from ray.rllib.algorithms.ppo.ppo_rl_module_config import PPOModuleConfig import ray from ray.rllib import SampleBatch +from ray.rllib.algorithms.ppo.ppo_catalog import PPOCatalog from ray.rllib.algorithms.ppo.tf.ppo_tf_rl_module import ( PPOTfRLModule, ) from ray.rllib.algorithms.ppo.torch.ppo_torch_rl_module import ( PPOTorchRLModule, ) +from ray.rllib.core.rl_module.rl_module import RLModuleConfig +from ray.rllib.models.preprocessors import get_preprocessor from ray.rllib.utils.numpy import convert_to_numpy from ray.rllib.utils.torch_utils import convert_to_torch_tensor @@ -24,8 +25,8 @@ def get_expected_module_config( env: gym.Env, model_config_dict: dict, - observation_space: gym.Space, -) -> PPOModuleConfig: + observation_space: gym.spaces.Space, +) -> RLModuleConfig: """Get a PPOModuleConfig that we would expect from the catalog otherwise. Args: @@ -36,17 +37,14 @@ def get_expected_module_config( Returns: A PPOModuleConfig containing the relevant configs to build PPORLModule """ - catalog = PPOCatalog( + config = RLModuleConfig( observation_space=observation_space, action_space=env.action_space, model_config_dict=model_config_dict, + catalog_class=PPOCatalog, ) - return PPOModuleConfig( - observation_space=env.observation_space, - action_space=env.action_space, - catalog=catalog, - ) + return config def dummy_torch_ppo_loss(batch, fwd_out): @@ -124,9 +122,9 @@ def tearDownClass(cls): ray.shutdown() def test_rollouts(self): - # TODO: Add ALE/Breakout-v5 to cover a 3D obs space + # TODO: Add FrozenLake-v1 to cover LSTM case. frameworks = ["torch", "tf2"] - env_names = ["CartPole-v1", "Pendulum-v1"] + env_names = ["CartPole-v1", "Pendulum-v1", "ALE/Breakout-v5"] fwd_fns = ["forward_exploration", "forward_inference"] # TODO(Artur): Re-enable LSTM lstm = [False] @@ -136,15 +134,26 @@ def test_rollouts(self): if lstm and fw == "tf2": # LSTM not implemented in TF2 yet continue + if env_name == "ALE/Breakout-v5" and fw == "tf2": + # TODO(Artur): Implement CNN in TF2. + continue print(f"[FW={fw} | [ENV={env_name}] | [FWD={fwd_fn}] | LSTM" f"={lstm}") - env = gym.make(env_name) + if env_name.startswith("ALE/"): + env = gym.make("GymV26Environment-v0", env_id=env_name) + else: + env = gym.make(env_name) + + preprocessor_cls = get_preprocessor(env.observation_space) + preprocessor = preprocessor_cls(env.observation_space) + module = _get_ppo_module( framework=fw, env=env, lstm=lstm, - observation_space=env.observation_space, + observation_space=preprocessor.observation_space, ) obs, _ = env.reset() + obs = preprocessor.transform(obs) batch = _get_input_batch_from_obs(fw, obs) @@ -162,9 +171,9 @@ def test_rollouts(self): module.forward_inference(batch) def test_forward_train(self): - # TODO: Add ALE/Breakout-v5 to cover a 3D obs space + # TODO: Add FrozenLake-v1 to cover LSTM case. frameworks = ["torch", "tf2"] - env_names = ["CartPole-v1", "Pendulum-v1"] + env_names = ["CartPole-v1", "Pendulum-v1", "ALE/Breakout-v5"] # TODO(Artur): Re-enable LSTM lstm = [False] config_combinations = [frameworks, env_names, lstm] @@ -174,21 +183,29 @@ def test_forward_train(self): # LSTM not implemented in TF2 yet continue if env_name == "ALE/Breakout-v5" and fw == "tf2": - # CNN not implement in TF2 yet + # TODO(Artur): Implement CNN in TF2. continue print(f"[FW={fw} | [ENV={env_name}] | LSTM={lstm}") - env = gym.make(env_name) + # TODO(Artur): Figure out why this is needed and fix it. + if env_name.startswith("ALE/"): + env = gym.make("GymV26Environment-v0", env_id=env_name) + else: + env = gym.make(env_name) + + preprocessor_cls = get_preprocessor(env.observation_space) + preprocessor = preprocessor_cls(env.observation_space) module = _get_ppo_module( framework=fw, env=env, lstm=lstm, - observation_space=env.observation_space, + observation_space=preprocessor.observation_space, ) # collect a batch of data batches = [] obs, _ = env.reset() + obs = preprocessor.transform(obs) tstep = 0 # TODO (Artur): Un-uncomment once Policy supports RNN # state_in = module.get_initial_state() @@ -206,6 +223,7 @@ def test_forward_train(self): fwd_out = module.forward_exploration(input_batch) action = convert_to_numpy(fwd_out["action_dist"].sample()[0]) new_obs, reward, terminated, truncated, _ = env.step(action) + new_obs = preprocessor.transform(new_obs) output_batch = { SampleBatch.OBS: obs, SampleBatch.NEXT_OBS: new_obs, diff --git a/rllib/algorithms/ppo/tests/test_ppo_with_rl_module.py b/rllib/algorithms/ppo/tests/test_ppo_with_rl_module.py index ab32f7460f6e0..5d39e836ae679 100644 --- a/rllib/algorithms/ppo/tests/test_ppo_with_rl_module.py +++ b/rllib/algorithms/ppo/tests/test_ppo_with_rl_module.py @@ -115,8 +115,11 @@ def test_ppo_compilation_and_schedule_mixins(self): for fw in framework_iterator( config, frameworks=("torch", "tf2"), with_eager_tracing=True ): - # TODO (Kourosh) Bring back "FrozenLake-v1" and "MsPacmanNoFrameskip-v4" - for env in ["CartPole-v1", "Pendulum-v1"]: + # TODO (Kourosh) Bring back "FrozenLake-v1" + for env in ["CartPole-v1", "Pendulum-v1", "ALE/Breakout-v5"]: + if env == "ALE/Breakout-v5" and fw == "tf2": + # TODO(Artur): Implement CNN in TF2. + continue print("Env={}".format(env)) # TODO (Kourosh, Avnishn): for now just do lstm=False for lstm in [False]: diff --git a/rllib/core/learner/learner.py b/rllib/core/learner/learner.py index 468fef74eb0c6..a3ea1cb91c7aa 100644 --- a/rllib/core/learner/learner.py +++ b/rllib/core/learner/learner.py @@ -135,7 +135,7 @@ class Learner: module_class=MyModule, observation_space=env.observation_space, action_space=env.action_space, - model_config = {"hidden": [128, 128]} + model_config_dict = {"hidden": [128, 128]} ) # create a learner instance that will train the module @@ -154,7 +154,7 @@ class Learner: module_class=NewPlayerModule, observation_space=env.observation_space, action_space=env.action_space, - model_config = {"hidden": [128, 128]} + model_config_dict = {"hidden": [128, 128]} ) ) diff --git a/rllib/core/learner/tests/test_learner.py b/rllib/core/learner/tests/test_learner.py index 04b7e8fbe76cb..4b6932c2b5347 100644 --- a/rllib/core/learner/tests/test_learner.py +++ b/rllib/core/learner/tests/test_learner.py @@ -23,7 +23,7 @@ def get_learner() -> Learner: module_class=DiscreteBCTFModule, observation_space=env.observation_space, action_space=env.action_space, - model_config={"fcnet_hiddens": [32]}, + model_config_dict={"fcnet_hiddens": [32]}, ), optimizer_config={"lr": 1e-3}, learner_scaling_config=LearnerGroupScalingConfig(), @@ -128,7 +128,7 @@ def set_optimizer_fn(module): module_class=DiscreteBCTFModule, observation_space=env.observation_space, action_space=env.action_space, - model_config={"fcnet_hiddens": [16]}, + model_config_dict={"fcnet_hiddens": [16]}, ), set_optimizer_fn=set_optimizer_fn, ) diff --git a/rllib/core/learner/tests/test_learner_group_config.py b/rllib/core/learner/tests/test_learner_group_config.py index 476ba86f2cec9..7bf543647b069 100644 --- a/rllib/core/learner/tests/test_learner_group_config.py +++ b/rllib/core/learner/tests/test_learner_group_config.py @@ -46,7 +46,7 @@ def test_learner_group_build_from_algorithm_config(self): module_class=DiscreteBCTFModule, observation_space=env.observation_space, action_space=env.action_space, - model_config={"fcnet_hiddens": [32]}, + model_config_dict={"fcnet_hiddens": [32]}, ) ) learner_group_config.build() diff --git a/rllib/core/learner/torch/tests/test_torch_learner.py b/rllib/core/learner/torch/tests/test_torch_learner.py index ef12b17437137..8eb1ed1b62b52 100644 --- a/rllib/core/learner/torch/tests/test_torch_learner.py +++ b/rllib/core/learner/torch/tests/test_torch_learner.py @@ -114,7 +114,7 @@ def set_optimizer_fn(module): module_class=DiscreteBCTorchModule, observation_space=env.observation_space, action_space=env.action_space, - model_config={"fcnet_hiddens": [16]}, + model_config_dict={"fcnet_hiddens": [16]}, ), set_optimizer_fn=set_optimizer_fn, ) diff --git a/rllib/core/models/base.py b/rllib/core/models/base.py index 88a3f79e86290..6d45a1ea8613e 100644 --- a/rllib/core/models/base.py +++ b/rllib/core/models/base.py @@ -1,6 +1,6 @@ import abc from dataclasses import dataclass -from typing import List, Union +from typing import List, Union, Tuple from ray.rllib import SampleBatch from ray.rllib.models.specs.specs_base import Spec @@ -30,8 +30,15 @@ class ModelConfig(abc.ABC): It is therefore a means of configuration for RLModules. However, ModelConfigs are not restricted to be used only with Catalog or RLModules. A usage Example together with a Model can be found in the Model. + + Args: + input_dims: The input dimensions of the network + output_dims: The output dimensions of the network. """ + input_dims: Union[List[int], Tuple[int]] = None + output_dims: Union[List[int], Tuple[int]] = None + @abc.abstractmethod def build(self, framework: str): """Builds the model. @@ -195,7 +202,8 @@ class Encoder(Model, abc.ABC): Similarly, their output_spec contains the latent space dimensions. Encoders can be recurrent, in which case the state should be part of input- and output_specs. The latents that are produced by an encoder are fed into subsequent - heads. + heads. Any implementation of Encoder should also be callable. This should be done + by also inheriting from a framework-specific model base-class, s.a. TorchModel. Abstract illustration of typical flow of tensors: diff --git a/rllib/core/models/catalog.py b/rllib/core/models/catalog.py index 991244a6c5307..4230d28f5b113 100644 --- a/rllib/core/models/catalog.py +++ b/rllib/core/models/catalog.py @@ -1,13 +1,21 @@ +from typing import Optional, Mapping, Any +import functools + +import numpy as np import gymnasium as gym +from gymnasium.spaces import Box, Dict, Discrete, MultiDiscrete, Tuple + +from ray.rllib.core.models.base import ModelConfig +from ray.rllib.core.models.base import Encoder from ray.rllib.core.models.configs import ( MLPEncoderConfig, LSTMEncoderConfig, CNNEncoderConfig, ) -from ray.rllib.core.models.base import ModelConfig from ray.rllib.models import MODEL_DEFAULTS -from gymnasium.spaces import Box from ray.rllib.models.utils import get_filter_config +from ray.rllib.utils.error import UnsupportedSpaceException +from ray.rllib.utils.spaces.simplex import Simplex class Catalog: @@ -40,8 +48,8 @@ def __init__( super().__init__(observation_space, action_space, model_config_dict) self.my_model_config_dict = MLPHeadConfig( hidden_layer_dims=[64, 32], - input_dim=self.observation_space.shape[0], - output_dim=1, + input_dims=[self.observation_space.shape[0]], + output_dims=[1], ) def build_my_head(self, framework: str): @@ -81,18 +89,56 @@ def __init__( self.model_config_dict = {**MODEL_DEFAULTS, **model_config_dict} self.view_requirements = view_requirements - # Produce a basic encoder config. + self._latent_dims = None + + # Overwrite this post-init hook in subclasses + self.__post_init__() + + @property + def latent_dims(self): + """Returns the latent dimensions of the encoder. + + This establishes an agreement between encoder and heads about the latent + dimensions. Encoders can be built to output a latent tensor with + `latent_dims` dimensions, and heads can be built with tensors of + `latent_dims` dimensions as inputs. This can be safely ignored if this + agreement is not needed in case of modifications to the Catalog. + + Returns: + The latent dimensions of the encoder. + """ + return self._latent_dims + + @latent_dims.setter + def latent_dims(self, value): + self._latent_dims = value + + def __post_init__(self): + """Post-init hook for subclasses to override. + + This makes it so that subclasses are not forced to create an encoder config + if the rest of their catalog is not dependent on it or if it breaks. + At the end of Catalog initialization, an attribute `Catalog.latent_dims` + should be set so that heads can be built using that information. + """ self.encoder_config = self.get_encoder_config( - observation_space=observation_space, - action_space=action_space, - model_config_dict=model_config_dict, - view_requirements=view_requirements, + observation_space=self.observation_space, + action_space=self.action_space, + model_config_dict=self.model_config_dict, + view_requirements=self.view_requirements, ) + + # Create a function that can be called when framework is known to retrieve the + # class type for action distributions + self.action_dist_class_fn = functools.partial( + self.get_dist_cls_from_action_space, action_space=self.action_space + ) + # The dimensions of the latent vector that is output by the encoder and fed # to the heads. - self.latent_dim = self.encoder_config.output_dim + self.latent_dims = self.encoder_config.output_dims - def build_encoder(self, framework: str): + def build_encoder(self, framework: str) -> Encoder: """Builds the encoder. By default this method builds an encoder instance from Catalog.encoder_config. @@ -103,8 +149,35 @@ def build_encoder(self, framework: str): Returns: The encoder. """ + assert hasattr(self, "encoder_config"), ( + "You must define a `Catalog.encoder_config` attribute in your Catalog " + "subclass or override the `Catalog.build_encoder` method. By default, " + "an encoder_config is created in the __post_init__ method." + ) return self.encoder_config.build(framework=framework) + def get_action_dist_cls(self, framework: str): + """Get the action distribution class. + + The default behavior is to get the action distribution from the + `Catalog.action_dist_class_fn`. This can be overridden to build a custom action + distribution as a means of configuring the behavior of a PPORLModuleBase + implementation. + + Args: + framework: The framework to use. Either "torch" or "tf". + + Returns: + The action distribution. + """ + assert hasattr(self, "action_dist_class_fn"), ( + "You must define a `Catalog.action_dist_class_fn` attribute in your " + "Catalog subclass or override the `Catalog.action_dist_class_fn` method. " + "By default, an action_dist_class_fn is created in the __post_init__ " + "method." + ) + return self.action_dist_class_fn(framework=framework) + @classmethod def get_encoder_config( cls, @@ -149,20 +222,22 @@ def get_encoder_config( encoder_latent_dim = ( model_config_dict["encoder_latent_dim"] or fcnet_hiddens[-1] ) + use_lstm = model_config_dict["use_lstm"] + use_attention = model_config_dict["use_attention"] - if model_config_dict["use_lstm"]: + if use_lstm: encoder_config = LSTMEncoderConfig( hidden_dim=model_config_dict["lstm_cell_size"], batch_first=not model_config_dict["_time_major"], num_layers=1, - output_dim=model_config_dict["lstm_cell_size"], + output_dims=[model_config_dict["lstm_cell_size"]], output_activation=output_activation, observation_space=observation_space, action_space=action_space, view_requirements_dict=view_requirements, get_tokenizer_config=cls.get_tokenizer_config, ) - elif model_config_dict["use_attention"]: + elif use_attention: raise NotImplementedError else: # TODO (Artur): Maybe check for original spaces here @@ -176,10 +251,10 @@ def get_encoder_config( else: hidden_layer_dims = model_config_dict["fcnet_hiddens"][:-1] encoder_config = MLPEncoderConfig( - input_dim=observation_space.shape[0], + input_dims=[observation_space.shape[0]], hidden_layer_dims=hidden_layer_dims, hidden_layer_activation=activation, - output_dim=encoder_latent_dim, + output_dims=[encoder_latent_dim], output_activation=output_activation, ) @@ -197,12 +272,18 @@ def get_encoder_config( filter_specifiers=model_config_dict["conv_filters"], filter_layer_activation=activation, output_activation=output_activation, - output_dim=encoder_latent_dim, + output_dims=[encoder_latent_dim], ) # input_space is a possibly nested structure of spaces. else: # NestedModelConfig - raise NotImplementedError("No default config for complex spaces yet!") + raise ValueError( + f"No default encoder config for " + f"obs space={observation_space}," + f" lstm={use_lstm} and " + f"attention={use_attention} " + f"found." + ) return encoder_config @@ -224,3 +305,107 @@ def get_tokenizer_config( **{"use_lstm": False, "use_attention": False}, }, ) + + @classmethod + def get_dist_cls_from_action_space( + cls, + action_space: gym.Space, + *, + framework: Optional[str] = None, + deterministic: Optional[bool] = False, + ) -> Mapping[str, Any]: + """Returns a distribution class for the given action space. + + You can get the required input dimension for the distribution by calling + `action_dict_cls.required_model_output_shape(action_space, model_config_dict)` + on the retrieved class. This is useful, because the Catalog needs to find out + about the required input dimension for the distribution before the model that + outputs these inputs is configured. + + Args: + action_space: Action space of the target gym env. + framework: The framework to use. + deterministic: Whether to return a Deterministic distribution on input + logits instead of a stochastic distributions. For example for Discrete + spaces, the stochastic is a Categorical distribution with output logits, + while the deterministic distribution will be to output the argmax of + logits directly. + + + Returns: + The distribution class for the given action space. + """ + + if framework == "torch": + from ray.rllib.models.torch.torch_distributions import ( + TorchCategorical, + TorchDeterministic, + TorchDiagGaussian, + ) + + distribution_dicts = { + "deterministic": TorchDeterministic, + "gaussian": TorchDiagGaussian, + "categorical": TorchCategorical, + } + elif framework == "tf": + from ray.rllib.models.tf.tf_distributions import ( + TfCategorical, + TfDeterministic, + TfDiagGaussian, + ) + + distribution_dicts = { + "deterministic": TfDeterministic, + "gaussian": TfDiagGaussian, + "categorical": TfCategorical, + } + else: + raise ValueError( + f"Unknown framework: {framework}. Only 'torch' and 'tf2' are " + "supported for RLModule Catalogs." + ) + + # Box space -> DiagGaussian OR Deterministic. + if isinstance(action_space, Box): + if action_space.dtype.char in np.typecodes["AllInteger"]: + raise ValueError( + "Box(..., `int`) action spaces are not supported. " + "Use MultiDiscrete or Box(..., `float`)." + ) + else: + if len(action_space.shape) > 1: + raise UnsupportedSpaceException( + "Action space has multiple dimensions " + "{}. ".format(action_space.shape) + + "Consider reshaping this into a single dimension, " + "using a custom action distribution, " + "using a Tuple action space, or the multi-agent API." + ) + if deterministic: + return distribution_dicts["deterministic"] + else: + return distribution_dicts["gaussian"] + + # Discrete Space -> Categorical. + elif isinstance(action_space, Discrete): + return distribution_dicts["categorical"] + + # Tuple/Dict Spaces -> MultiAction. + elif isinstance(action_space, (Tuple, Dict)): + # TODO(Artur): Supported Tuple/Dict. + raise NotImplementedError("Tuple/Dict spaces not yet supported.") + + # Simplex -> Dirichlet. + elif isinstance(action_space, Simplex): + # TODO(Artur): Supported Simplex (in torch). + raise NotImplementedError("Simplex action space not yet supported.") + + # MultiDiscrete -> MultiCategorical. + elif isinstance(action_space, MultiDiscrete): + # TODO(Artur): Support multi-discrete. + raise NotImplementedError("MultiDiscrete spaces not yet supported.") + + # Unknown type -> Error. + else: + raise NotImplementedError(f"Unsupported action space: `{action_space}`") diff --git a/rllib/core/models/configs.py b/rllib/core/models/configs.py index eccb998e61568..5135e7e950e87 100644 --- a/rllib/core/models/configs.py +++ b/rllib/core/models/configs.py @@ -71,19 +71,19 @@ class MLPHeadConfig(ModelConfig): The configured MLP encodes 1D-observations into a latent space. The stack of layers is composed of a sequence of linear layers. The first layer - has `input_dim` inputs and the last layer has `output_dim` outputs. The number of + has `input_dims` inputs and the last layer has `output_dims` outputs. The number of units inbetween is determined by `hidden_layer_dims`. If `hidden_layer_dims` is - None, there is only one linear layer with `input_dim` inputs and `output_dim` + None, there is only one linear layer with `input_dims` inputs and `output_dims` outputs. Each layer is followed by an activation function as per this config. See ModelConfig for usage details. Example: Configuration: - input_dim = 4 + input_dims = [4] hidden_layer_dims = [8, 8] hidden_layer_activation = "relu" - output_dim = 2 + output_dims = [2] output_activation = "linear" Resulting stack in pseudocode: @@ -94,19 +94,15 @@ class MLPHeadConfig(ModelConfig): Linear(8, 2) Attributes: - input_dim: The input dimension of the network. It cannot be None. hidden_layer_dims: The sizes of the hidden layers. hidden_layer_activation: The activation function to use after each layer ( except for the output). output_activation: The activation function to use for the output layer. - output_dim: The output dimension of the network. """ - input_dim: int = None hidden_layer_dims: List[int] = field(default_factory=lambda: [256, 256]) hidden_layer_activation: str = "relu" output_activation: str = "linear" - output_dim: int = None @_framework_implemented() def build(self, framework: str = "torch") -> Model: @@ -137,7 +133,7 @@ class CNNEncoderConfig(ModelConfig): The stack of layers is composed of a sequence of convolutional layers. `input_dims` describes the shape of the input tensor. Beyond that, each layer specified by `filter_specifiers` is followed by an activation function according - to `filter_activation`. The `output_dim` is reached by flattening a final + to `filter_activation`. `output_dims` is reached by flattening a final convolutional layer and applying a linear layer with `output_activation`. See ModelConfig for usage details. @@ -150,7 +146,7 @@ class CNNEncoderConfig(ModelConfig): [32, [4, 4], 2], ] filter_activation = "relu" - output_dim = 256 + output_dims = [256] output_activation = "linear" Resulting stack in pseudocode: @@ -172,9 +168,6 @@ class CNNEncoderConfig(ModelConfig): filter_layer_activation: The activation function to use after each layer ( except for the output). output_activation: The activation function to use for the output layer. - output_dim: The output dimension. We append a final convolutional layer - depth-only filters that is flattened and a final linear layer to achieve - this dimension regardless of the previous filters. """ input_dims: Union[List[int], Tuple[int]] = None @@ -183,7 +176,6 @@ class CNNEncoderConfig(ModelConfig): ) filter_layer_activation: str = "relu" output_activation: str = "linear" - output_dim: int = None @_framework_implemented(tf2=False) def build(self, framework: str = "torch") -> Model: @@ -263,7 +255,6 @@ class LSTMEncoderConfig(ModelConfig): action_space: gym.Space = None view_requirements_dict: ViewRequirementsDict = None get_tokenizer_config: Callable[[gym.Space, Dict], ModelConfig] = None - output_dim: int = None @_framework_implemented(tf2=False) def build(self, framework: str = "torch") -> Encoder: diff --git a/rllib/core/models/tests/test_catalog.py b/rllib/core/models/tests/test_catalog.py index b9926083c1ec2..73668020e5812 100644 --- a/rllib/core/models/tests/test_catalog.py +++ b/rllib/core/models/tests/test_catalog.py @@ -1,18 +1,38 @@ import itertools import unittest +import functools +from collections import namedtuple -import gym +import gymnasium as gym import numpy as np import tree -from gymnasium.spaces import Box +from gymnasium.spaces import Box, Discrete +from ray.rllib.algorithms.ppo.ppo import PPOConfig +from ray.rllib.core.models.torch.base import TorchModel +from ray.rllib.core.models.base import ModelConfig, Encoder +from ray.rllib.algorithms.ppo.torch.ppo_torch_rl_module import PPOTorchRLModule +from ray.rllib.algorithms.ppo.ppo_catalog import PPOCatalog from ray.rllib.core.models.base import STATE_IN, ENCODER_OUT, STATE_OUT -from ray.rllib.core.models.catalog import Catalog from ray.rllib.core.models.configs import MLPEncoderConfig, CNNEncoderConfig +from ray.rllib.core.models.catalog import Catalog +from ray.rllib.core.rl_module.rl_module import SingleAgentRLModuleSpec +from ray.rllib.models import MODEL_DEFAULTS +from ray.rllib.models.tf.tf_distributions import ( + TfCategorical, + TfDeterministic, + TfDiagGaussian, +) +from ray.rllib.models.torch.torch_distributions import ( + TorchCategorical, + TorchDeterministic, + TorchDiagGaussian, +) from ray.rllib.policy.sample_batch import SampleBatch from ray.rllib.utils.framework import try_import_tf from ray.rllib.utils.framework import try_import_torch from ray.rllib.utils.spaces.space_utils import get_dummy_batch_for_space +from ray.rllib.utils.test_utils import framework_iterator from ray.rllib.utils.torch_utils import convert_to_torch_tensor _, tf, _ = try_import_tf() @@ -51,9 +71,9 @@ def _check_model_outputs(self, model, framework, model_config_dict, input_space) } outputs = model(inputs) - assert outputs[ENCODER_OUT].shape == (32, latent_dim) + self.assertEqual(outputs[ENCODER_OUT].shape, (32, latent_dim)) tree.map_structure_with_path( - lambda p, v: self.assertTrue(v.shape == states[p].shape), + lambda p, v: self.assertEqual(v.shape, states[p].shape), outputs[STATE_OUT], ) @@ -164,7 +184,7 @@ def test_get_encoder_config(self): model_config = catalog.get_encoder_config( observation_space=input_space, model_config_dict=model_config_dict ) - assert type(model_config) == model_config_type + self.assertEqual(type(model_config), model_config_type) model = model_config.build(framework=framework) # Do a forward pass and check if the output has the correct shape @@ -173,6 +193,164 @@ def test_get_encoder_config(self): # TODO(Artur): Add support for composite spaces and test here # Today, Catalog does not handle composite spaces, so we can't test them + def test_get_dist_cls_from_action_space(self): + """Tests if we can create a bunch of action distributions. + + Action distributions are created from the base catalog class. Things this + test checks: + - Whether we output the correct action distributions classes. + - Whether we can instantiate the action distributions, query their + required input dimensions and sample from them. + + """ + TestConfig = namedtuple( + "TestConfig", ("action_space", "deterministic", "expected_dist_cls_dict") + ) + test_configs = [ + TestConfig( + Box(-np.inf, np.inf, (7,), dtype=np.float32), + False, + {"torch": TorchDiagGaussian, "tf": TfDiagGaussian}, + ), + TestConfig( + Box(-np.inf, np.inf, (7,), dtype=np.float32), + True, + {"torch": TorchDeterministic, "tf": TfDeterministic}, + ), + TestConfig( + Discrete(5), None, {"torch": TorchCategorical, "tf": TfCategorical} + ), + ] + + for ( + action_space, + deterministic, + expected_cls_dict, + ) in test_configs: + print( + f"Testing action space: {action_space} and deterministic:" + f" {deterministic}" + ) + catalog = Catalog( + observation_space=Box(-1.0, 1.0, (84, 84, 1), dtype=np.float32), + action_space=action_space, + model_config_dict=MODEL_DEFAULTS.copy(), + ) + + for framework in framework_iterator(frameworks=["tf2", "torch"]): + + if framework == "tf2": + framework = "tf" + + dist_cls = catalog.get_dist_cls_from_action_space( + action_space=action_space, + deterministic=deterministic, + framework=framework, + ) + + # Check if we can query the required input dimensions + input_shape = expected_cls_dict[framework].required_model_output_shape( + action_space, model_config=MODEL_DEFAULTS.copy() + ) + logits = np.ones((32, *input_shape), dtype=np.float32) + if framework == "torch": + logits = torch.from_numpy(logits) + else: + logits = tf.convert_to_tensor(logits) + # We don't need a model if we input tensors + dist = dist_cls.from_logits(logits=logits) + self.assertTrue(isinstance(dist, expected_cls_dict[framework])) + actions = dist.sample() + self.assertTrue(action_space.contains(actions.numpy()[0])) + + def test_customize_catalog_from_algorithm_config(self): + """Test if we can pass catalog to algorithm config and it ends up inside + RLModule and is used to build models there.""" + + class MyCatalog(PPOCatalog): + def build_vf_head(self, framework): + return torch.nn.Linear(self.latent_dims[0], 1) + + config = ( + PPOConfig() + .rl_module(rl_module_spec=SingleAgentRLModuleSpec(catalog_class=MyCatalog)) + .framework("torch") + ) + + algo = config.build(env="CartPole-v0") + self.assertEqual( + algo.get_policy("default_policy").model.config.catalog_class, MyCatalog + ) + + # Test if we can pass custom catalog to algorithm config and train with it. + + config = ( + PPOConfig() + .rl_module( + rl_module_spec=SingleAgentRLModuleSpec( + module_class=PPOTorchRLModule, catalog_class=MyCatalog + ) + ) + .framework("torch") + ) + + algo = config.build(env="CartPole-v0") + algo.train() + + def test_post_init_overwrite(self): + """Test if we can overwrite post_init method of a catalog class. + + This tests: + - Defines a custom encoder and its config. + - Defines a custom catalog class that uses the custom encoder by + overwriting the __post_init__ method and defining a custom + Catalog.encoder_config. + - Defines a custom RLModule that uses the custom catalog. + - Runs a forward pass through the custom RLModule to check if + everything is working together as expected. + + """ + env = gym.make("CartPole-v0") + + class MyCostumTorchEncoderConfig(ModelConfig): + def build(self, framework): + return MyCostumTorchEncoder() + + class MyCostumTorchEncoder(TorchModel, Encoder): + def __init__(self): + super().__init__({}) + self.net = torch.nn.Linear(env.observation_space.shape[0], 10) + + def _forward(self, input_dict, **kwargs): + return { + ENCODER_OUT: (self.net(input_dict["obs"])), + STATE_OUT: None, + } + + class MyCustomCatalog(PPOCatalog): + def __post_init__(self): + self.action_dist_class_fn = functools.partial( + self.get_dist_cls_from_action_space, action_space=self.action_space + ) + self.latent_dims = (10,) + self.encoder_config = MyCostumTorchEncoderConfig( + input_dims=self.observation_space.shape, + output_dims=self.latent_dims, + ) + + spec = SingleAgentRLModuleSpec( + module_class=PPOTorchRLModule, + observation_space=env.observation_space, + action_space=env.action_space, + model_config_dict=MODEL_DEFAULTS.copy(), + catalog_class=MyCustomCatalog, + ) + module = spec.build() + + module.forward_inference( + input_data={"obs": torch.ones((32, *env.observation_space.shape))} + ) + if __name__ == "__main__": import pytest diff --git a/rllib/core/models/tf/encoder.py b/rllib/core/models/tf/encoder.py index 0ca9c4e285b1a..15332244bb267 100644 --- a/rllib/core/models/tf/encoder.py +++ b/rllib/core/models/tf/encoder.py @@ -1,7 +1,5 @@ from typing import Union -import tree - from ray.rllib.core.models.base import ( Encoder, ActorCriticEncoder, @@ -15,7 +13,6 @@ from ray.rllib.models.specs.specs_base import Spec from ray.rllib.models.specs.specs_dict import SpecDict from ray.rllib.models.specs.specs_tf import TFTensorSpecs -from ray.rllib.policy.rnn_sequencing import add_time_dimension from ray.rllib.policy.sample_batch import SampleBatch from ray.rllib.utils.annotations import override from ray.rllib.utils.framework import try_import_torch @@ -31,9 +28,9 @@ def __init__(self, config: ModelConfig) -> None: # Create the neural networks self.net = TfMLP( - input_dim=config.input_dim, + input_dim=config.input_dims[0], hidden_layer_dims=config.hidden_layer_dims, - output_dim=config.output_dim, + output_dim=config.output_dims[0], hidden_layer_activation=config.hidden_layer_activation, ) @@ -41,7 +38,7 @@ def __init__(self, config: ModelConfig) -> None: def get_input_spec(self) -> Union[Spec, None]: return SpecDict( { - SampleBatch.OBS: TFTensorSpecs("b, h", h=self.config.input_dim), + SampleBatch.OBS: TFTensorSpecs("b, h", h=self.config.input_dims[0]), STATE_IN: None, SampleBatch.SEQ_LENS: None, } @@ -51,7 +48,7 @@ def get_input_spec(self) -> Union[Spec, None]: def get_output_spec(self) -> Union[Spec, None]: return SpecDict( { - ENCODER_OUT: TFTensorSpecs("b, h", h=self.config.output_dim), + ENCODER_OUT: TFTensorSpecs("b, h", h=self.config.output_dims[0]), STATE_OUT: None, } ) @@ -66,91 +63,6 @@ def _forward(self, inputs: NestedDict) -> NestedDict: ) -class LSTMEncoder(Encoder, TfModel): - """An encoder that uses an LSTM cell and a linear layer.""" - - def __init__(self, config: ModelConfig) -> None: - TfModel.__init__(self, config) - Encoder.__init__(self, config) - - # Create the neural networks - self.lstm = nn.LSTM( - config.input_dim, - config.hidden_dim, - config.num_layers, - batch_first=config.batch_first, - ) - self.linear = nn.Linear(config.hidden_dim, config.output_dim) - - @override(Model) - def get_input_spec(self) -> Union[Spec, None]: - return SpecDict( - { - # bxt is just a name for better readability to indicated padded batch - SampleBatch.OBS: TFTensorSpecs("bxt, h", h=self.config.input_dim), - STATE_IN: { - "h": TFTensorSpecs( - "b, l, h", h=self.config.hidden_dim, l=self.config.num_layers - ), - "c": TFTensorSpecs( - "b, l, h", h=self.config.hidden_dim, l=self.config.num_layers - ), - }, - SampleBatch.SEQ_LENS: None, - } - ) - - @override(Model) - def get_output_spec(self) -> Union[Spec, None]: - return SpecDict( - { - ENCODER_OUT: TFTensorSpecs("bxt, h", h=self.config.output_dim), - STATE_OUT: { - "h": TFTensorSpecs( - "b, l, h", h=self.config.hidden_dim, l=self.config.num_layers - ), - "c": TFTensorSpecs( - "b, l, h", h=self.config.hidden_dim, l=self.config.num_layers - ), - }, - } - ) - - @override(Model) - def get_initial_state(self): - config = self.config - return { - "h": torch.zeros(config.num_layers, config.hidden_dim), - "c": torch.zeros(config.num_layers, config.hidden_dim), - } - - @override(Model) - def _forward(self, inputs: NestedDict, **kwargs) -> NestedDict: - x = inputs[SampleBatch.OBS] - states = inputs[STATE_IN] - # states are batch-first when coming in - states = tree.map_structure(lambda x: x.transpose(0, 1), states) - - x = add_time_dimension( - x, - seq_lens=inputs[SampleBatch.SEQ_LENS], - framework="tf", - time_major=not self.config.batch_first, - ) - states_o = {} - x, (states_o["h"], states_o["c"]) = self.lstm(x, (states["h"], states["c"])) - - x = self.linear(x) - x = x.view(-1, x.shape[-1]) - - return NestedDict( - { - ENCODER_OUT: x, - STATE_OUT: tree.map_structure(lambda x: x.transpose(0, 1), states_o), - } - ) - - class TfActorCriticEncoder(TfModel, ActorCriticEncoder): """An encoder that can hold two encoders.""" diff --git a/rllib/core/models/tf/mlp.py b/rllib/core/models/tf/mlp.py index d5929c91c46aa..96056fc71a34c 100644 --- a/rllib/core/models/tf/mlp.py +++ b/rllib/core/models/tf/mlp.py @@ -17,20 +17,20 @@ def __init__(self, config: ModelConfig) -> None: TfModel.__init__(self, config) self.net = TfMLP( - input_dim=config.input_dim, + input_dim=config.input_dims[0], hidden_layer_dims=config.hidden_layer_dims, - output_dim=config.output_dim, + output_dim=config.output_dims[0], hidden_layer_activation=config.hidden_layer_activation, output_activation=config.output_activation, ) @override(Model) def get_input_spec(self) -> Union[Spec, None]: - return TFTensorSpecs("b, h", h=self.config.input_dim) + return TFTensorSpecs("b, h", h=self.config.input_dims[0]) @override(Model) def get_output_spec(self) -> Union[Spec, None]: - return TFTensorSpecs("b, h", h=self.config.output_dim) + return TFTensorSpecs("b, h", h=self.config.output_dims[0]) @override(Model) def _forward(self, inputs: NestedDict, **kwargs) -> NestedDict: diff --git a/rllib/core/models/tf/tests/test_tf_mlp_head.py b/rllib/core/models/tf/tests/test_tf_mlp_head.py index 1599c449dc012..727ae76e787bb 100644 --- a/rllib/core/models/tf/tests/test_tf_mlp_head.py +++ b/rllib/core/models/tf/tests/test_tf_mlp_head.py @@ -9,26 +9,25 @@ class TestTfMLPHead(unittest.TestCase): def test_tf_mlp_head(self): - - inputs_dims = [1, 2, 1000] + inputs_dims_configs = [[1], [2], [1000]] list_of_hidden_layer_dims = [[], [1], [64, 64], [1000, 1000, 1000, 1000]] hidden_layer_activations = [None, "linear", "relu", "tanh", "elu", "swish"] - output_dims = inputs_dims + output_dims_configs = inputs_dims_configs output_activations = hidden_layer_activations for permutation in itertools.product( - inputs_dims, + inputs_dims_configs, list_of_hidden_layer_dims, hidden_layer_activations, output_activations, - output_dims, + output_dims_configs, ): ( - inputs_dim, + inputs_dims, hidden_layer_dims, hidden_layer_activation, output_activation, @@ -37,7 +36,7 @@ def test_tf_mlp_head(self): print( f"Testing ...\n" - f"inputs_dim: {inputs_dim}\n" + f"inputs_dim: {inputs_dims}\n" f"hidden_layer_dims: {hidden_layer_dims}\n" f"hidden_layer_activation: {hidden_layer_activation}\n" f"output_activation: {output_activation}\n" @@ -45,20 +44,20 @@ def test_tf_mlp_head(self): ) config = MLPHeadConfig( - input_dim=inputs_dim, + input_dims=inputs_dims, hidden_layer_dims=hidden_layer_dims, - output_dim=output_dims, + output_dims=output_dims, hidden_layer_activation=hidden_layer_activation, output_activation=output_activation, ) model = config.build(framework="tf") - inputs = tf.random.uniform((1, inputs_dim)) + inputs = tf.random.uniform((1, inputs_dims[0])) outputs = model(inputs) - self.assertEqual(outputs.shape, (1, output_dims)) + self.assertEqual(outputs.shape, (1, output_dims[0])) if __name__ == "__main__": diff --git a/rllib/core/models/torch/encoder.py b/rllib/core/models/torch/encoder.py index 87fb2ccd23f9d..fb18670b18150 100644 --- a/rllib/core/models/torch/encoder.py +++ b/rllib/core/models/torch/encoder.py @@ -1,8 +1,7 @@ from typing import Union -import torch -import torch.nn as nn import tree + from ray.rllib.core.models.base import ( Encoder, ActorCriticEncoder, @@ -13,7 +12,6 @@ from ray.rllib.core.models.base import ModelConfig, Model from ray.rllib.core.models.torch.base import TorchModel from ray.rllib.core.models.torch.primitives import TorchMLP, TorchCNN - from ray.rllib.models.specs.specs_base import Spec from ray.rllib.models.specs.specs_dict import SpecDict from ray.rllib.models.specs.specs_torch import TorchTensorSpec @@ -21,8 +19,11 @@ from ray.rllib.policy.rnn_sequencing import add_time_dimension from ray.rllib.policy.sample_batch import SampleBatch from ray.rllib.utils.annotations import override +from ray.rllib.utils.framework import try_import_torch from ray.rllib.utils.nested_dict import NestedDict +torch, nn = try_import_torch() + class TorchMLPEncoder(TorchModel, Encoder): def __init__(self, config: ModelConfig) -> None: @@ -31,9 +32,9 @@ def __init__(self, config: ModelConfig) -> None: # Create the neural networks self.net = TorchMLP( - input_dim=config.input_dim, + input_dim=config.input_dims[0], hidden_layer_dims=config.hidden_layer_dims, - output_dim=config.output_dim, + output_dim=config.output_dims[0], hidden_layer_activation=config.hidden_layer_activation, ) @@ -41,7 +42,7 @@ def __init__(self, config: ModelConfig) -> None: def get_input_spec(self) -> Union[Spec, None]: return SpecDict( { - SampleBatch.OBS: TorchTensorSpec("b, h", h=self.config.input_dim), + SampleBatch.OBS: TorchTensorSpec("b, h", h=self.config.input_dims[0]), STATE_IN: None, SampleBatch.SEQ_LENS: None, } @@ -51,7 +52,7 @@ def get_input_spec(self) -> Union[Spec, None]: def get_output_spec(self) -> Union[Spec, None]: return SpecDict( { - ENCODER_OUT: TorchTensorSpec("b, h", h=self.config.output_dim), + ENCODER_OUT: TorchTensorSpec("b, h", h=self.config.output_dims[0]), STATE_OUT: None, } ) @@ -89,7 +90,9 @@ def __init__(self, config: ModelConfig) -> None: # Add a final linear layer to make sure that the outputs have the correct # dimensionality. layers.append( - nn.Linear(int(cnn.output_width) * int(cnn.output_height), config.output_dim) + nn.Linear( + int(cnn.output_width) * int(cnn.output_height), config.output_dims[0] + ) ) if output_activation is not None: layers.append(output_activation()) @@ -115,7 +118,7 @@ def get_input_spec(self) -> Union[Spec, None]: def get_output_spec(self) -> Union[Spec, None]: return SpecDict( { - ENCODER_OUT: TorchTensorSpec("b, h", h=self.config.output_dim), + ENCODER_OUT: TorchTensorSpec("b, h", h=self.config.output_dims[0]), STATE_OUT: None, } ) @@ -143,14 +146,14 @@ def __init__(self, config: ModelConfig) -> None: config.num_layers, batch_first=config.batch_first, ) - self.linear = nn.Linear(config.hidden_dim, config.output_dim) + self.linear = nn.Linear(config.hidden_dim, config.output_dims[0]) @override(Model) def get_input_spec(self) -> Union[Spec, None]: return SpecDict( { # bxt is just a name for better readability to indicated padded batch - SampleBatch.OBS: TorchTensorSpec("bxt, h", h=self.config.input_dim), + SampleBatch.OBS: TorchTensorSpec("bxt, h", h=self.config.input_dims[0]), STATE_IN: { "h": TorchTensorSpec( "b, l, h", h=self.config.hidden_dim, l=self.config.num_layers @@ -167,7 +170,7 @@ def get_input_spec(self) -> Union[Spec, None]: def get_output_spec(self) -> Union[Spec, None]: return SpecDict( { - ENCODER_OUT: TorchTensorSpec("bxt, h", h=self.config.output_dim), + ENCODER_OUT: TorchTensorSpec("bxt, h", h=self.config.output_dims[0]), STATE_OUT: { "h": TorchTensorSpec( "b, l, h", h=self.config.hidden_dim, l=self.config.num_layers @@ -189,7 +192,7 @@ def get_initial_state(self): @override(Model) def _forward(self, inputs: NestedDict, **kwargs) -> NestedDict: - x = inputs[SampleBatch.OBS] + x = inputs[SampleBatch.OBS].float() states = inputs[STATE_IN] # states are batch-first when coming in states = tree.map_structure(lambda x: x.transpose(0, 1), states) diff --git a/rllib/core/models/torch/mlp.py b/rllib/core/models/torch/mlp.py index f2570f70fe3ab..80423014ad432 100644 --- a/rllib/core/models/torch/mlp.py +++ b/rllib/core/models/torch/mlp.py @@ -18,20 +18,20 @@ def __init__(self, config: ModelConfig) -> None: TorchModel.__init__(self, config) self.net = TorchMLP( - input_dim=config.input_dim, + input_dim=config.input_dims[0], hidden_layer_dims=config.hidden_layer_dims, - output_dim=config.output_dim, + output_dim=config.output_dims[0], hidden_layer_activation=config.hidden_layer_activation, output_activation=config.output_activation, ) @override(Model) def get_input_spec(self) -> Union[Spec, None]: - return TorchTensorSpec("b, h", h=self.config.input_dim) + return TorchTensorSpec("b, h", h=self.config.input_dims[0]) @override(Model) def get_output_spec(self) -> Union[Spec, None]: - return TorchTensorSpec("b, h", h=self.config.output_dim) + return TorchTensorSpec("b, h", h=self.config.output_dims[0]) @override(Model) def _forward(self, inputs: torch.Tensor, **kwargs) -> torch.Tensor: diff --git a/rllib/core/models/torch/primitives.py b/rllib/core/models/torch/primitives.py index 8a08dbe663d96..c7bf9af5fc760 100644 --- a/rllib/core/models/torch/primitives.py +++ b/rllib/core/models/torch/primitives.py @@ -60,8 +60,10 @@ def __init__( self.output_dim = dims[-1] self.mlp = nn.Sequential(*layers) + self.expected_input_dtype = torch.float32 + def forward(self, x): - return self.mlp(x) + return self.mlp(x.type(self.expected_input_dtype)) class TorchCNN(nn.Module): @@ -69,7 +71,7 @@ class TorchCNN(nn.Module): def __init__( self, - input_dims: Union[List, Tuple] = None, + input_dims: Union[List[int], Tuple[int]] = None, filter_specifiers: List[List[Union[int, List]]] = None, filter_layer_activation: str = "relu", output_activation: str = "linear", @@ -152,7 +154,9 @@ def __init__( # Create the cnn that potentially includes a flattened layer self.cnn = nn.Sequential(*layers) + self.expected_input_dtype = torch.float32 + def forward(self, x): # Permute b/c data comes in as [B, dim, dim, channels]: inputs = x.permute(0, 3, 1, 2) - return self.cnn(inputs) + return self.cnn(inputs.type(self.expected_input_dtype)) diff --git a/rllib/core/models/torch/tests/test_torch_cnn_encoder.py b/rllib/core/models/torch/tests/test_torch_cnn_encoder.py index 4802f96daf025..358b3079aa367 100644 --- a/rllib/core/models/torch/tests/test_torch_cnn_encoder.py +++ b/rllib/core/models/torch/tests/test_torch_cnn_encoder.py @@ -30,7 +30,7 @@ def test_torch_cnn_encoder(self): filter_layer_activation = [None, "linear", "relu"] - output_dims = [1, 100] + output_dims_configs = [[1], [100]] output_activations = filter_layer_activation @@ -38,7 +38,7 @@ def test_torch_cnn_encoder(self): inputs_dimss, filter_layer_activation, output_activations, - output_dims, + output_dims_configs, ): ( inputs_dims, @@ -63,7 +63,7 @@ def test_torch_cnn_encoder(self): filter_specifiers=filter_specifiers, filter_layer_activation=filter_layer_activation, output_activation=output_activation, - output_dim=output_dims, + output_dims=output_dims, ) model = config.build(framework="torch") @@ -77,7 +77,7 @@ def test_torch_cnn_encoder(self): {SampleBatch.OBS: obs, SampleBatch.SEQ_LENS: seq_lens, STATE_IN: state} ) - self.assertEqual(outputs[ENCODER_OUT].shape, (1, output_dims)) + self.assertEqual(outputs[ENCODER_OUT].shape, (1, output_dims[0])) self.assertEqual(outputs[STATE_OUT], None) diff --git a/rllib/core/models/torch/tests/test_torch_mlp_encoder.py b/rllib/core/models/torch/tests/test_torch_mlp_encoder.py index b43b9b21b1511..b1be558892bee 100644 --- a/rllib/core/models/torch/tests/test_torch_mlp_encoder.py +++ b/rllib/core/models/torch/tests/test_torch_mlp_encoder.py @@ -12,25 +12,25 @@ class TestTorchMLPEncoder(unittest.TestCase): def test_torch_mlp_encoder(self): - inputs_dims = [1, 2, 1000] + inputs_dims_configs = [[1], [2], [1000]] list_of_hidden_layer_dims = [[], [1], [64, 64], [1000, 1000, 1000, 1000]] hidden_layer_activations = [None, "linear", "relu", "tanh", "elu", "swish"] - output_dims = inputs_dims + output_dims_configs = inputs_dims_configs output_activations = hidden_layer_activations for permutation in itertools.product( - inputs_dims, + inputs_dims_configs, list_of_hidden_layer_dims, hidden_layer_activations, output_activations, - output_dims, + output_dims_configs, ): ( - inputs_dim, + inputs_dims, hidden_layer_dims, hidden_layer_activation, output_activation, @@ -39,7 +39,7 @@ def test_torch_mlp_encoder(self): print( f"Testing ...\n" - f"inputs_dim: {inputs_dim}\n" + f"inputs_dim: {inputs_dims}\n" f"hidden_layer_dims: {hidden_layer_dims}\n" f"hidden_layer_activation: {hidden_layer_activation}\n" f"output_activation: {output_activation}\n" @@ -47,16 +47,16 @@ def test_torch_mlp_encoder(self): ) config = MLPEncoderConfig( - input_dim=inputs_dim, + input_dims=inputs_dims, hidden_layer_dims=hidden_layer_dims, - output_dim=output_dims, + output_dims=output_dims, hidden_layer_activation=hidden_layer_activation, output_activation=output_activation, ) model = config.build(framework="torch") - obs = torch.randn(1, inputs_dim) + obs = torch.randn(1, inputs_dims[0]) seq_lens = torch.tensor([1]) state = None @@ -64,7 +64,7 @@ def test_torch_mlp_encoder(self): {SampleBatch.OBS: obs, SampleBatch.SEQ_LENS: seq_lens, STATE_IN: state} ) - self.assertEqual(outputs[ENCODER_OUT].shape, (1, output_dims)) + self.assertEqual(outputs[ENCODER_OUT].shape, (1, output_dims[0])) self.assertEqual(outputs[STATE_OUT], None) diff --git a/rllib/core/models/torch/tests/test_torch_mlp_head.py b/rllib/core/models/torch/tests/test_torch_mlp_head.py index b748fe8f9966e..266c3e552a9a5 100644 --- a/rllib/core/models/torch/tests/test_torch_mlp_head.py +++ b/rllib/core/models/torch/tests/test_torch_mlp_head.py @@ -10,25 +10,25 @@ class TestTorchMLPHead(unittest.TestCase): def test_torch_mlp_head(self): - inputs_dims = [1, 2, 1000] + inputs_dims_configs = [[1], [2], [1000]] list_of_hidden_layer_dims = [[], [1], [64, 64], [1000, 1000, 1000, 1000]] hidden_layer_activations = [None, "linear", "relu", "tanh", "elu", "swish"] - output_dims = inputs_dims + output_dims_configs = inputs_dims_configs output_activations = hidden_layer_activations for permutation in itertools.product( - inputs_dims, + inputs_dims_configs, list_of_hidden_layer_dims, hidden_layer_activations, output_activations, - output_dims, + output_dims_configs, ): ( - inputs_dim, + inputs_dims, hidden_layer_dims, hidden_layer_activation, output_activation, @@ -37,7 +37,7 @@ def test_torch_mlp_head(self): print( f"Testing ...\n" - f"inputs_dim: {inputs_dim}\n" + f"inputs_dim: {inputs_dims}\n" f"hidden_layer_dims: {hidden_layer_dims}\n" f"hidden_layer_activation: {hidden_layer_activation}\n" f"output_activation: {output_activation}\n" @@ -45,20 +45,20 @@ def test_torch_mlp_head(self): ) config = MLPHeadConfig( - input_dim=inputs_dim, + input_dims=inputs_dims, hidden_layer_dims=hidden_layer_dims, - output_dim=output_dims, + output_dims=output_dims, hidden_layer_activation=hidden_layer_activation, output_activation=output_activation, ) model = config.build(framework="torch") - inputs = torch.randn(1, inputs_dim) + inputs = torch.randn(1, inputs_dims[0]) outputs = model(inputs) - self.assertEqual(outputs.shape, (1, output_dims)) + self.assertEqual(outputs.shape, (1, output_dims[0])) if __name__ == "__main__": diff --git a/rllib/core/rl_module/marl_module.py b/rllib/core/rl_module/marl_module.py index 312e09535a29f..2970b16b5fa7b 100644 --- a/rllib/core/rl_module/marl_module.py +++ b/rllib/core/rl_module/marl_module.py @@ -1,5 +1,4 @@ -import copy -from dataclasses import dataclass +from dataclasses import dataclass, field import pprint from typing import Iterator, Mapping, Any, Union, Dict, Optional, Type @@ -17,22 +16,6 @@ ModuleID = str -def _get_module_configs(config: Dict[str, Any]): - """Constructs a mapping from module_id to module config. - - It takes care of the inheritance of common configs to individual module configs. - See `from_multi_agent_config` for more details. - """ - config = copy.deepcopy(config) - module_specs = config.pop("modules", {}) - for common_spec in config: - for module_spec in module_specs.values(): - if getattr(module_spec, common_spec) is None: - setattr(module_spec, common_spec, config[common_spec]) - - return module_specs - - @PublicAPI(stability="alpha") class MultiAgentRLModule(RLModule): """Base class for multi-agent RLModules. @@ -58,79 +41,21 @@ class MultiAgentRLModule(RLModule): `MultiAgentRLModule`. """ - def __init__(self, rl_modules: Mapping[ModuleID, RLModule] = None) -> None: - super().__init__() - self._rl_modules: Mapping[ModuleID, RLModule] = rl_modules or {} + def __init__(self, config: "MultiAgentRLModuleConfig" = None) -> None: + if config is None: + config = MultiAgentRLModuleConfig() - @classmethod - def from_multi_agent_config(cls, config: Mapping[str, Any]) -> "MultiAgentRLModule": - """Creates a MultiAgentRLModule from a multi-agent config. - - The input config should contain "modules" key that is a mapping from module_id - to the module spec for each RLModule which is a SingleAgentRLModuleSpec object. - If there are multiple modules that do share the same - `observation_space`, `action_space`, or `model_config`, you can specify these - keys at the top level of the config, and the module spec will inherit the - values from the top level config. - - Examples: - - .. code-block:: python - - config = { - "modules": { - "module_1": SingleAgentRLModuleSpec( - module_class="RLModule1", - observation_space=gym.spaces.Box(...), - action_space=gym.spaces.Discrete(...), - model_config={hidden_dim: 256} - ) - "module_2": SingleAgentRLModuleSpec( - module_class="RLModule2", - observation_space=gym.spaces.Box(...), - ) - }, - "action_space": gym.spaces.Box(...), - "model_config": {hidden_dim: 32} - } + super().__init__(config) - # This is equivalent to the following config: - - config = { - "modules": { - "module_1": SingleAgentRLModuleSpec( - module_class="RLModule1", - observation_space=gym.spaces.Box(...), - action_space=gym.spaces.Discrete(...), - model_config={hidden_dim: 256} - ) - "module_2": SingleAgentRLModuleSpec( - module_class="RLModule2", - observation_space=gym.spaces.Box(...), - action_space=gym.spaces.Box(...), # Inherited - model_config={hidden_dim: 32} # Inherited - } - }, - } + # self.build() will abstract the construction of rl_modules + self._rl_modules = {} + self.build() - Args: - config: A config dict that contains the module configs. See above for the - format required. - - Returns: - The MultiAgentRLModule. - """ - - module_configs: Dict[ModuleID, Any] = _get_module_configs(config) - cls.__check_module_configs(module_configs) - - multiagent_module = cls() - - for module_id, module_spec in module_configs.items(): - module = module_spec.build() - multiagent_module.add_module(module_id, module) - - return multiagent_module + def build(self): + """Builds the underlying RLModules.""" + self.__check_module_configs(self.config.modules) + for module_id, module_spec in self.config.modules.items(): + self._rl_modules[module_id] = module_spec.build() @classmethod def __check_module_configs(cls, module_configs: Dict[ModuleID, Any]): @@ -335,47 +260,6 @@ def set_state(self, state_dict: Mapping[str, Any]) -> None: for module_id, module in self._rl_modules.items(): module.set_state(state_dict[module_id]) - def serialize(self) -> Mapping[str, Any]: - """Return the serialized state of the module. - - NOTE: This method needs to be implemented in order to support - checkpointing and fault tolerance. - - """ - return { - "class": self.__class__, - "rl_modules": { - module_id: module.serialize() - for module_id, module in self._rl_modules.items() - }, - } - - @classmethod - def deserialize(cls, state: Mapping[str, Any]) -> "MultiAgentRLModule": - """Construct a module from a serialized state. - - Args: - state: The serialized state of the module. - The state should contain the keys "class", "kwargs", and "state". - - - "class" is the class of the RLModule to be constructed. - - "rl_modules" is a dict mapping module ids of the RLModules to - their serialized states. The serialized states can be obtained - from `RLModule.serialize()`. - - NOTE: this state is typically obtained from `serialize()`. - - NOTE: This method needs to be implemented in order to support - checkpointing and fault tolerance. - - Returns: - A deserialized MultiAgentRLModule. - """ - rl_modules = {} - for module_id, module_state in state["rl_modules"].items(): - rl_modules[module_id] = RLModule.deserialize(module_state) - return cls(rl_modules) - def __repr__(self) -> str: return f"MARL({pprint.pformat(self._rl_modules)})" @@ -421,7 +305,7 @@ def _check_module_exists(self, module_id: ModuleID) -> None: ) -@ExperimentalAPI +@PublicAPI(stability="alpha") @dataclass class MultiAgentRLModuleSpec: """A utility spec class to make it constructing MARL modules easier. @@ -454,6 +338,9 @@ def __post_init__(self): "SingleAgentRLModuleSpecs for each individual module." ) + def get_marl_config(self) -> "MultiAgentRLModuleConfig": + return MultiAgentRLModuleConfig(modules=self.module_specs) + def build( self, module_id: Optional[ModuleID] = None ) -> Union[SingleAgentRLModuleSpec, "MultiAgentRLModule"]: @@ -478,27 +365,100 @@ def build( if module_id: return self.module_specs[module_id].build() - return self.marl_module_class.from_multi_agent_config( - {"modules": self.module_specs} - ) + + module_config = self.get_marl_config() + return self.marl_module_class(module_config) def add_modules( - self, module_specs: Dict[ModuleID, SingleAgentRLModuleSpec] + self, + module_specs: Dict[ModuleID, SingleAgentRLModuleSpec], + overwrite: bool = True, ) -> None: - """Add new module specs to the spec. + """Add new module specs to the spec or updates existing ones. Args: module_specs: The mapping for the module_id to the single-agent module specs to be added to this multi-agent module spec. + overwrite: Whether to overwrite the existing module specs if they already + exist. If False, they will be updated only. """ if self.module_specs is None: self.module_specs = {} - self.module_specs.update(module_specs) + for module_id, module_spec in module_specs.items(): + if overwrite or module_id not in self.module_specs: + self.module_specs[module_id] = module_spec + else: + self.module_specs[module_id].update(module_spec) + + @classmethod + def from_module(self, module: MultiAgentRLModule) -> "MultiAgentRLModuleSpec": + """Creates a MultiAgentRLModuleSpec from a MultiAgentRLModule. + + Args: + module: The MultiAgentRLModule to create the spec from. + + Returns: + The MultiAgentRLModuleSpec. + """ + module_specs = { + module_id: SingleAgentRLModuleSpec.from_module(rl_module) + for module_id, rl_module in module._rl_modules.items() + } + marl_module_class = module.__class__ + return MultiAgentRLModuleSpec( + marl_module_class=marl_module_class, module_specs=module_specs + ) def _check_before_build(self): if not isinstance(self.module_specs, dict): raise ValueError( - f"When build() is called on {self.__class__} the module_specs " + f"When build() is called on {self.__class__}, the module_specs " "should be a dictionary mapping from module IDs to " "SingleAgentRLModuleSpecs for each individual module." ) + + def update(self, other: "MultiAgentRLModuleSpec", overwrite=False) -> None: + """Updates this spec with the other spec. + + Traverses this MultiAgentRLModuleSpec's module_specs and updates them with + the module specs from the other MultiAgentRLModuleSpec. + + Args: + other: The other spec to update this spec with. + overwrite: Whether to overwrite the existing module specs if they already + exist. If False, they will be updated only. + """ + assert type(other) is MultiAgentRLModuleSpec + + if isinstance(other.module_specs, dict): + self.add_modules(other.module_specs, overwrite=overwrite) + else: + if not self.module_specs: + self.module_specs = other.module_specs + else: + self.module_specs.update(other.module_specs) + + +@ExperimentalAPI +@dataclass +class MultiAgentRLModuleConfig: + + modules: Mapping[ModuleID, SingleAgentRLModuleSpec] = field(default_factory=dict) + + def to_dict(self): + + return { + "modules": { + module_id: module_spec.to_dict() + for module_id, module_spec in self.modules.items() + } + } + + @classmethod + def from_dict(cls, d) -> "MultiAgentRLModuleConfig": + return cls( + modules={ + module_id: SingleAgentRLModuleSpec.from_dict(module_spec) + for module_id, module_spec in d["modules"].items() + } + ) diff --git a/rllib/core/rl_module/rl_module.py b/rllib/core/rl_module/rl_module.py index d73d052314ffe..53220a41e0205 100644 --- a/rllib/core/rl_module/rl_module.py +++ b/rllib/core/rl_module/rl_module.py @@ -1,16 +1,20 @@ import abc from dataclasses import dataclass +import datetime import gymnasium as gym -from typing import Mapping, Any, TYPE_CHECKING, Optional, Type, Dict +import json +import pathlib +from typing import Any, Dict, Mapping, Optional, Type, TYPE_CHECKING, Union if TYPE_CHECKING: from ray.rllib.core.rl_module.marl_module import MultiAgentRLModule + from ray.rllib.core.models.catalog import Catalog +import ray from ray.rllib.utils.annotations import ( ExperimentalAPI, OverrideToImplementCustomLogic_CallToSuperRecommended, ) -from ray.rllib.utils.serialization import check_if_args_kwargs_serializable from ray.rllib.models.specs.typing import SpecType from ray.rllib.models.specs.checker import ( @@ -22,9 +26,16 @@ from ray.rllib.policy.sample_batch import DEFAULT_POLICY_ID, SampleBatch from ray.rllib.utils.nested_dict import NestedDict from ray.rllib.utils.typing import SampleBatchType +from ray.rllib.utils.serialization import ( + gym_space_from_dict, + gym_space_to_dict, + serialize_type, + deserialize_type, +) ModuleID = str +METADATA_FILE_NAME = "rl_module_metadata.json" @ExperimentalAPI @@ -33,50 +44,147 @@ class SingleAgentRLModuleSpec: """A utility spec class to make it constructing RLModules (in single-agent case) easier. Args: - module_class: ... - observation_space: ... - action_space: ... - model_config: ... + module_class: The RLModule class to use. + observation_space: The observation space of the RLModule. + action_space: The action space of the RLModule. + model_config_dict: The model config dict to use. + catalog_class: The Catalog class to use. """ module_class: Optional[Type["RLModule"]] = None - observation_space: Optional["gym.Space"] = None - action_space: Optional["gym.Space"] = None - model_config: Optional[Dict[str, Any]] = None + observation_space: Optional[gym.Space] = None + action_space: Optional[gym.Space] = None + model_config_dict: Optional[Mapping[str, Any]] = None + catalog_class: Optional[Type["Catalog"]] = None + + def get_rl_module_config(self) -> "RLModuleConfig": + """Returns the RLModule config for this spec.""" + return RLModuleConfig( + observation_space=self.observation_space, + action_space=self.action_space, + model_config_dict=self.model_config_dict, + catalog_class=self.catalog_class, + ) def build(self) -> "RLModule": - + if self.module_class is None: + raise ValueError("RLModule class is not set.") if self.observation_space is None: - raise ValueError("Observation space must be specified.") + raise ValueError("Observation space is not set.") if self.action_space is None: - raise ValueError("Action space must be specified.") - if self.model_config is None: - raise ValueError("Model config must be specified.") + raise ValueError("Action space is not set.") + if self.model_config_dict is None: + raise ValueError("Model config is not set.") - return self.module_class.from_model_config( - observation_space=self.observation_space, - action_space=self.action_space, - model_config_dict=self.model_config, + module_config = self.get_rl_module_config() + return self.module_class(module_config) + + @classmethod + def from_module(cls, module: "RLModule") -> "SingleAgentRLModuleSpec": + from ray.rllib.core.rl_module.marl_module import MultiAgentRLModule + + if isinstance(module, MultiAgentRLModule): + raise ValueError( + "MultiAgentRLModule cannot be converted to SingleAgentRLModuleSpec." + ) + + return SingleAgentRLModuleSpec( + module_class=type(module), + observation_space=module.config.observation_space, + action_space=module.config.action_space, + model_config_dict=module.config.model_config_dict, + catalog_class=module.config.catalog_class, + ) + + def to_dict(self): + """Returns a serialized representation of the spec.""" + + return { + "module_class": serialize_type(self.module_class), + "module_config": self.get_rl_module_config().to_dict(), + } + + @classmethod + def from_dict(cls, d): + """Returns a single agent RLModule spec from a serialized representation.""" + module_class = deserialize_type(d["module_class"]) + + module_config = RLModuleConfig.from_dict(d["module_config"]) + observation_space = module_config.observation_space + action_space = module_config.action_space + model_config_dict = module_config.model_config_dict + catalog_class = module_config.catalog_class + + return SingleAgentRLModuleSpec( + module_class=module_class, + observation_space=observation_space, + action_space=action_space, + model_config_dict=model_config_dict, + catalog_class=catalog_class, ) + def update(self, other) -> None: + """Updates this spec with the given other spec. Works like dict.update().""" + if not isinstance(other, SingleAgentRLModuleSpec): + raise ValueError("Can only update with another SingleAgentRLModuleSpec.") + + # If the field is None in the other, keep the current field, otherwise update + # with the new value. + self.module_class = other.module_class or self.module_class + self.observation_space = other.observation_space or self.observation_space + self.action_space = other.action_space or self.action_space + self.model_config_dict = other.model_config_dict or self.model_config_dict + self.catalog_class = other.catalog_class or self.catalog_class + @ExperimentalAPI @dataclass class RLModuleConfig: - """Configuration for the PPO module. - # TODO (Kourosh): Whether we need this or not really depends on how the catalog - # design end up being. - Attributes: - observation_space: The observation space of the environment. - action_space: The action space of the environment. - max_seq_len: Max seq len for training an RNN model. - (TODO (Kourosh) having max_seq_len here seems a bit unnatural, can we rethink - this design?) - """ observation_space: gym.Space = None action_space: gym.Space = None - max_seq_len: int = None + model_config_dict: Mapping[str, Any] = None + catalog_class: Type["Catalog"] = None + + def get_catalog(self) -> "Catalog": + """Returns the catalog for this config.""" + return self.catalog_class( + observation_space=self.observation_space, + action_space=self.action_space, + model_config_dict=self.model_config_dict, + ) + + def to_dict(self): + """Returns a serialized representation of the config. + + NOTE: This should be JSON-able. Users can test this by calling + json.dumps(config.to_dict()). + + """ + catalog_class_path = ( + serialize_type(type(self.catalog_class)) if self.catalog_class else "" + ) + return { + "observation_space": gym_space_to_dict(self.observation_space), + "action_space": gym_space_to_dict(self.action_space), + "model_config_dict": self.model_config_dict, + "catalog_class_path": catalog_class_path, + } + + @classmethod + def from_dict(cls, d: Dict[str, Any]): + """Creates a config from a serialized representation.""" + catalog_class = ( + None + if d["catalog_class_path"] == "" + else deserialize_type(d["catalog_class_path"]) + ) + return cls( + observation_space=gym_space_from_dict(d["observation_space"]), + action_space=gym_space_from_dict(d["action_space"]), + model_config_dict=d["model_config_dict"], + catalog_class=catalog_class, + ) @ExperimentalAPI @@ -143,9 +251,8 @@ class RLModule(abc.ABC): More details here: https://github.com/pytorch/pytorch/issues/49726. """ - def __init__(self, *args, **kwargs): - check_if_args_kwargs_serializable(args, kwargs) - self._args_and_kwargs = {"args": args, "kwargs": kwargs} + def __init__(self, config: RLModuleConfig): + self.config = config def __init_subclass__(cls, **kwargs): # Automatically add a __post_init__ method to all subclasses of RLModule. @@ -187,58 +294,6 @@ def __post_init__(self): self.output_specs_inference() ) - @classmethod - def from_model_config( - cls, - observation_space: gym.Space, - action_space: gym.Space, - *, - model_config: Mapping[str, Any], - ) -> "RLModule": - """Creates a RLModule instance from a model config dict and spaces. - - The model config dict is the same as the one passed to the AlgorithmConfig - object that contains global model configurations parameters. - - This method can also be used to create a config dict for the module constructor - so it can be re-used to create multiple instances of the module. - - Example: - - .. code-block:: python - - class MyModule(RLModule): - def __init__(self, input_dim, output_dim): - self.input_dim, self.output_dim = input_dim, output_dim - - @classmethod - def from_model_config( - cls, - observation_space: gym.Space, - action_space: gym.Space, - model_config: Mapping[str, Any], - ): - return cls( - input_dim=observation_space.shape[0], - output_dim=action_space.n - ) - - module = MyModule.from_model_config( - observation_space=gym.spaces.Box(low=0, high=1, shape=(4,)), - action_space=gym.spaces.Discrete(2), - model_config={}, - ) - - - Args: - observation_space: The observation space of the env. - action_space: The action space of the env. - model_config: The model config dict. - """ - raise NotImplementedError - - # TODO: (Artur) Add a method `from_catalog` that creates RLModule from Catalog - def get_initial_state(self) -> NestedDict: """Returns the initial state of the module. @@ -361,39 +416,126 @@ def get_state(self) -> Mapping[str, Any]: def set_state(self, state_dict: Mapping[str, Any]) -> None: """Sets the state dict of the module.""" - def serialize(self) -> Mapping[str, Any]: - """Return the serialized state of the module.""" - return { - "class": self.__class__, - "args": self._args_and_kwargs["args"], - "kwargs": self._args_and_kwargs["kwargs"], - "state": self.get_state(), - } + def _save_module_metadata( + self, + checkpoint_dir: Union[str, pathlib.Path], + module_state_path: Union[str, pathlib.Path], + ): + """Saves the metadata of the module to checkpoint_dir. + + Includes: + - module class path + - module state path + - the module config + - the ray version used + - the ray commit hash used + - the date and time of the checkpoint was created + + """ + if isinstance(checkpoint_dir, str): + checkpoint_dir = pathlib.Path(checkpoint_dir) + if isinstance(module_state_path, str): + module_state_path = pathlib.Path(module_state_path) + gmt_time = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S GMT") + metadata = {} + # TODO (Avnishn): Find a way to incorporate the tune registry here. + metadata["module_class"] = serialize_type(self.__class__) + metadata["module_config"] = self.config.to_dict() + metadata["ray_version"] = ray.__version__ + metadata["ray_commit_hash"] = ray.__commit__ + metadata["checkpoint_date_time"] = gmt_time + metadata["module_state_path"] = str(module_state_path) + metadata_path = checkpoint_dir / METADATA_FILE_NAME + with open(metadata_path, "w") as f: + json.dump(metadata, f) @classmethod - def deserialize(cls, state: Mapping[str, Any]) -> "RLModule": - """Construct a module from a serialized state. + def _from_metadata_file(cls, metadata_path: Union[str, pathlib.Path]) -> "RLModule": + """Constructs a module from the metadata. Args: - state: The serialized state of the module. + metadata_path: The path to the metadata json file for a module. - NOTE: this state is typically obtained from `serialize()`. + Returns: + The module. + """ + if isinstance(metadata_path, str): + metadata_path = pathlib.Path(metadata_path) + if not metadata_path.exists(): + raise ValueError("The metadata path was not found.") + if not metadata_path.exists(): + raise ValueError( + "While constructing the module from the metadata, the " + f"metadata file was not found at {str(metadata_path)}" + ) + with open(metadata_path, "r") as f: + metadata = json.load(f) + module_class = deserialize_type(metadata["module_class"]) + module_config = RLModuleConfig.from_dict(metadata["module_config"]) + module = module_class(module_config) + return module - NOTE: This method needs to be implemented in order to support - checkpointing and fault tolerance. + def save_state_to_file(self, path: Union[str, pathlib.Path]) -> str: + """Saves the weights of this RLmodule to path. + + Args: + path: The directory to save the checkpoint to. Returns: - A deserialized RLModule. + The path to the saved checkpoint. """ - for key in ["class", "args", "kwargs", "state"]: - if key not in state: - raise ValueError( - "By default, the serialized state must contain the following " - f"keys: 'class', 'args', 'args', and 'kwargs'. Got: {state.keys()}" - ) - constructor = state["class"] - module = constructor(*state["args"], **state["kwargs"]) - module.set_state(state["state"]) + raise NotImplementedError + + def load_state_from_file(self, path: Union[str, pathlib.Path]) -> None: + """Loads the weights of an RLmodule from path. + + Args: + path: The directory to load the checkpoint from. + """ + raise NotImplementedError + + def save_to_checkpoint(self, checkpoint_dir_path: str) -> None: + """Saves the module to a checkpoint directory. + + Args: + dir_path: The directory to save the checkpoint to. + + Raises: + ValueError: If dir_path is not an absolute path. + """ + path = pathlib.Path(checkpoint_dir_path) + if not path.is_absolute(): + raise ValueError("dir_path must be an absolute path.") + path.mkdir(parents=True, exist_ok=True) + module_state_path = self.save_state_to_file(path) + self._save_module_metadata(path, module_state_path) + + @classmethod + def from_checkpoint(cls, checkpoint_dir_path: str) -> None: + """Loads the module from a checkpoint directory. + + Args: + dir_path: The directory to load the checkpoint from. + """ + path = pathlib.Path(checkpoint_dir_path) + if not path.exists(): + raise ValueError( + "While loading from checkpoint there was no directory" + " found at {}".format(checkpoint_dir_path) + ) + if not path.is_absolute(): + raise ValueError("dir_path must be an absolute path.") + if not path.is_dir(): + raise ValueError( + "While loading from checkpoint the checkpoint_dir_path " + "provided was not a directory." + ) + metadata_path = path / METADATA_FILE_NAME + with open(metadata_path, "r") as f: + metadata = json.load(f) + state_path = metadata["module_state_path"] + module = cls._from_metadata_file(metadata_path) + module.load_state_from_file(state_path) return module @abc.abstractmethod @@ -408,4 +550,6 @@ def as_multi_agent(self) -> "MultiAgentRLModule": """Returns a multi-agent wrapper around this module.""" from ray.rllib.core.rl_module.marl_module import MultiAgentRLModule - return MultiAgentRLModule({DEFAULT_POLICY_ID: self}) + marl_module = MultiAgentRLModule() + marl_module.add_module(DEFAULT_POLICY_ID, self) + return marl_module diff --git a/rllib/core/rl_module/tests/test_marl_module.py b/rllib/core/rl_module/tests/test_marl_module.py index 17aeccea41d1b..93e9880879ce9 100644 --- a/rllib/core/rl_module/tests/test_marl_module.py +++ b/rllib/core/rl_module/tests/test_marl_module.py @@ -1,8 +1,11 @@ import unittest -from ray.rllib.core.rl_module.rl_module import SingleAgentRLModuleSpec -from ray.rllib.core.rl_module.marl_module import MultiAgentRLModule, _get_module_configs +from ray.rllib.core.rl_module.rl_module import SingleAgentRLModuleSpec, RLModuleConfig +from ray.rllib.core.rl_module.marl_module import ( + MultiAgentRLModule, + MultiAgentRLModuleConfig, +) from ray.rllib.core.testing.torch.bc_module import DiscreteBCTorchModule from ray.rllib.env.multi_agent_env import make_multi_agent from ray.rllib.utils.test_utils import check @@ -16,45 +19,24 @@ def test_from_config(self): env_class = make_multi_agent("CartPole-v0") env = env_class({"num_agents": 2}) - module1 = DiscreteBCTorchModule.from_model_config( - env.observation_space, - env.action_space, + module1 = SingleAgentRLModuleSpec( + module_class=DiscreteBCTorchModule, + observation_space=env.observation_space, + action_space=env.action_space, model_config_dict={"fcnet_hiddens": [32]}, ) - module2 = DiscreteBCTorchModule.from_model_config( - env.observation_space, - env.action_space, + + module2 = SingleAgentRLModuleSpec( + module_class=DiscreteBCTorchModule, + observation_space=env.observation_space, + action_space=env.action_space, model_config_dict={"fcnet_hiddens": [32]}, ) - multi_agent_dict = {"module1": module1, "module2": module2} - marl_module = MultiAgentRLModule(multi_agent_dict) - - self.assertEqual(set(marl_module.keys()), {"module1", "module2"}) - self.assertIsInstance(marl_module["module1"], DiscreteBCTorchModule) - self.assertIsInstance(marl_module["module2"], DiscreteBCTorchModule) - - def test_from_multi_agent_config(self): - - env_class = make_multi_agent("CartPole-v0") - env = env_class({"num_agents": 2}) - - multi_agent_dict = { - "modules": { - "module1": SingleAgentRLModuleSpec( - module_class=DiscreteBCTorchModule, - model_config={"fcnet_hiddens": [64]}, - ), - "module2": SingleAgentRLModuleSpec( - module_class=DiscreteBCTorchModule, - model_config={"fcnet_hiddens": [32]}, - ), - }, - "observation_space": env.observation_space, # this is common - "action_space": env.action_space, # this is common - } - - marl_module = MultiAgentRLModule.from_multi_agent_config(multi_agent_dict) + config = MultiAgentRLModuleConfig( + modules={"module1": module1, "module2": module2} + ) + marl_module = MultiAgentRLModule(config) self.assertEqual(set(marl_module.keys()), {"module1", "module2"}) self.assertIsInstance(marl_module["module1"], DiscreteBCTorchModule) @@ -65,10 +47,12 @@ def test_as_multi_agent(self): env_class = make_multi_agent("CartPole-v0") env = env_class({"num_agents": 2}) - marl_module = DiscreteBCTorchModule.from_model_config( - env.observation_space, - env.action_space, - model_config_dict={"fcnet_hiddens": [32]}, + marl_module = DiscreteBCTorchModule( + config=RLModuleConfig( + env.observation_space, + env.action_space, + model_config_dict={"fcnet_hiddens": [32]}, + ) ).as_multi_agent() self.assertNotIsInstance(marl_module, DiscreteBCTorchModule) @@ -84,10 +68,12 @@ def test_get_set_state(self): env_class = make_multi_agent("CartPole-v0") env = env_class({"num_agents": 2}) - module = DiscreteBCTorchModule.from_model_config( - env.observation_space, - env.action_space, - model_config_dict={"fcnet_hiddens": [32]}, + module = DiscreteBCTorchModule( + config=RLModuleConfig( + env.observation_space, + env.action_space, + model_config_dict={"fcnet_hiddens": [32]}, + ) ).as_multi_agent() state = module.get_state() @@ -98,10 +84,12 @@ def test_get_set_state(self): set(module[DEFAULT_POLICY_ID].get_state().keys()), ) - module2 = DiscreteBCTorchModule.from_model_config( - env.observation_space, - env.action_space, - model_config_dict={"fcnet_hiddens": [32]}, + module2 = DiscreteBCTorchModule( + config=RLModuleConfig( + env.observation_space, + env.action_space, + model_config_dict={"fcnet_hiddens": [32]}, + ) ).as_multi_agent() state2 = module2.get_state() check(state, state2, false=True) @@ -116,18 +104,22 @@ def test_add_remove_modules(self): env_class = make_multi_agent("CartPole-v0") env = env_class({"num_agents": 2}) - module = DiscreteBCTorchModule.from_model_config( - env.observation_space, - env.action_space, - model_config_dict={"fcnet_hiddens": [32]}, + module = DiscreteBCTorchModule( + config=RLModuleConfig( + env.observation_space, + env.action_space, + model_config_dict={"fcnet_hiddens": [32]}, + ) ).as_multi_agent() module.add_module( "test", - DiscreteBCTorchModule.from_model_config( - env.observation_space, - env.action_space, - model_config_dict={"fcnet_hiddens": [32]}, + DiscreteBCTorchModule( + config=RLModuleConfig( + env.observation_space, + env.action_space, + model_config_dict={"fcnet_hiddens": [32]}, + ) ), ) self.assertEqual(set(module.keys()), {DEFAULT_POLICY_ID, "test"}) @@ -139,124 +131,28 @@ def test_add_remove_modules(self): ValueError, lambda: module.add_module( DEFAULT_POLICY_ID, - DiscreteBCTorchModule.from_model_config( - env.observation_space, - env.action_space, - model_config_dict={"fcnet_hiddens": [32]}, + DiscreteBCTorchModule( + config=RLModuleConfig( + env.observation_space, + env.action_space, + model_config_dict={"fcnet_hiddens": [32]}, + ) ), ), ) module.add_module( DEFAULT_POLICY_ID, - DiscreteBCTorchModule.from_model_config( - env.observation_space, - env.action_space, - model_config_dict={"fcnet_hiddens": [32]}, + DiscreteBCTorchModule( + config=RLModuleConfig( + env.observation_space, + env.action_space, + model_config_dict={"fcnet_hiddens": [32]}, + ) ), override=True, ) - def test_get_module_configs(self): - """Tests the method for getting the module configs from multi-agent config.""" - - config = { - "modules": { - "1": SingleAgentRLModuleSpec( - **{"module_class": "foo", "model_config": "bar"} - ), - "2": SingleAgentRLModuleSpec( - **{"module_class": "foo2", "model_config": "bar2"} - ), - }, - "observation_space": "obs_space", - "action_space": "action_space", - } - - expected_config = { - "1": SingleAgentRLModuleSpec( - **{ - "module_class": "foo", - "model_config": "bar", - "observation_space": "obs_space", - "action_space": "action_space", - } - ), - "2": SingleAgentRLModuleSpec( - **{ - "module_class": "foo2", - "model_config": "bar2", - "observation_space": "obs_space", - "action_space": "action_space", - } - ), - } - - self.assertDictEqual(_get_module_configs(config), expected_config) - - config = { - "modules": { - "1": SingleAgentRLModuleSpec( - **{ - "module_class": "foo", - "model_config": "bar", - "observation_space": "obs_space1", # won't get overwritten - "action_space": "action_space1", # won't get overwritten - } - ), - "2": SingleAgentRLModuleSpec( - **{"module_class": "foo2", "model_config": "bar2"} - ), - }, - "observation_space": "obs_space", - "action_space": "action_space", - } - - expected_config = { - "1": SingleAgentRLModuleSpec( - **{ - "module_class": "foo", - "model_config": "bar", - "observation_space": "obs_space1", - "action_space": "action_space1", - } - ), - "2": SingleAgentRLModuleSpec( - **{ - "module_class": "foo2", - "model_config": "bar2", - "observation_space": "obs_space", - "action_space": "action_space", - } - ), - } - - self.assertDictEqual(_get_module_configs(config), expected_config) - - def test_serialize_deserialize(self): - env_class = make_multi_agent("CartPole-v0") - env = env_class({"num_agents": 2}) - module1 = DiscreteBCTorchModule.from_model_config( - env.observation_space, - env.action_space, - model_config_dict={"fcnet_hiddens": [32]}, - ) - module2 = DiscreteBCTorchModule.from_model_config( - env.observation_space, - env.action_space, - model_config_dict={"fcnet_hiddens": [32]}, - ) - - multi_agent_dict = {"module1": module1, "module2": module2} - marl_module = MultiAgentRLModule(multi_agent_dict) - new_marl_module = marl_module.deserialize(marl_module.serialize()) - - self.assertNotEqual(id(marl_module), id(new_marl_module)) - self.assertEqual(set(marl_module.keys()), set(new_marl_module.keys())) - for key in marl_module.keys(): - self.assertNotEqual(id(marl_module[key]), id(new_marl_module[key])) - check(marl_module[key].get_state(), new_marl_module[key].get_state()) - if __name__ == "__main__": import pytest diff --git a/rllib/core/rl_module/tests/test_rl_module_specs.py b/rllib/core/rl_module/tests/test_rl_module_specs.py index 3f4230e0f5ad0..11f94aada99b0 100644 --- a/rllib/core/rl_module/tests/test_rl_module_specs.py +++ b/rllib/core/rl_module/tests/test_rl_module_specs.py @@ -10,12 +10,12 @@ from ray.rllib.core.testing.torch.bc_module import ( DiscreteBCTorchModule, BCTorchRLModuleWithSharedGlobalEncoder, - BCTorchMultiAgentSpec, + BCTorchMultiAgentModuleWithSharedEncoder, ) from ray.rllib.core.testing.tf.bc_module import ( DiscreteBCTFModule, BCTfRLModuleWithSharedGlobalEncoder, - BCTfMultiAgentSpec, + BCTfMultiAgentModuleWithSharedEncoder, ) MODULES = [DiscreteBCTorchModule, DiscreteBCTFModule] @@ -23,20 +23,10 @@ "torch": BCTorchRLModuleWithSharedGlobalEncoder, "tf": BCTfRLModuleWithSharedGlobalEncoder, } -CUSTOM_MARL_SPECS = {"torch": BCTorchMultiAgentSpec, "tf": BCTfMultiAgentSpec} - - -class BCRLModuleSpecCustom(SingleAgentRLModuleSpec): - """A customized SingleAgentRLModuleSpec.""" - - def build(self): - # this handles all implementation details - config = { - "input_dim": self.observation_space.shape[0], - "hidden_dim": self.model_config["fcnet_hiddens"][0], - "output_dim": self.action_space.n, - } - return self.module_class(**config) +CUSTOM_MARL_MODULES = { + "torch": BCTorchMultiAgentModuleWithSharedEncoder, + "tf": BCTfMultiAgentModuleWithSharedEncoder, +} class TestRLModuleSpecs(unittest.TestCase): @@ -48,47 +38,30 @@ def test_single_agent_spec(self): module_class=module_class, observation_space=env.observation_space, action_space=env.action_space, - model_config={"fcnet_hiddens": [64]}, + model_config_dict={"fcnet_hiddens": [64]}, ) module = spec.build() self.assertIsInstance(module, module_class) - def test_customized_single_agent_spec(self): - """Tests the a customized SingleAgentRLModuleSpec.""" - env = gym.make("CartPole-v1") - for module_class in MODULES: - - spec = BCRLModuleSpecCustom( - module_class=module_class, - observation_space=env.observation_space, - action_space=env.action_space, - model_config={"fcnet_hiddens": [64]}, - ) - module = spec.build() - self.assertIsInstance(module, module_class) - def test_multi_agent_spec(self): env = gym.make("CartPole-v1") num_agents = 2 - # make sure I use both default and cutomized single agent specs - single_agent_spec_classes = [SingleAgentRLModuleSpec, BCRLModuleSpecCustom] for module_class in MODULES: module_specs = {} for i in range(num_agents): - module_spec_cls = single_agent_spec_classes[i % num_agents] - module_specs[f"module_{i}"] = module_spec_cls( + module_specs[f"module_{i}"] = SingleAgentRLModuleSpec( module_class=module_class, observation_space=env.observation_space, action_space=env.action_space, - model_config={"fcnet_hiddens": [32 * (i + 1)]}, + model_config_dict={"fcnet_hiddens": [32 * (i + 1)]}, ) spec = MultiAgentRLModuleSpec(module_specs=module_specs) module = spec.build() self.assertIsInstance(module, MultiAgentRLModule) - def test_customized_multi_agent_spec(self): + def test_customized_multi_agent_module(self): """Tests creating a customized MARL BC module that owns a shared encoder.""" global_dim = 10 @@ -97,14 +70,14 @@ def test_customized_multi_agent_spec(self): # TODO (Kourosh): add tf support for fw in ["torch"]: - spec_cls = CUSTOM_MARL_SPECS[fw] - module_cls = CUSTOM_MODULES[fw] + marl_module_cls = CUSTOM_MARL_MODULES[fw] + rl_module_cls = CUSTOM_MODULES[fw] - spec = spec_cls( - marl_module_class=MultiAgentRLModule, + spec = MultiAgentRLModuleSpec( + marl_module_class=marl_module_cls, module_specs={ "agent_1": SingleAgentRLModuleSpec( - module_class=module_cls, + module_class=rl_module_cls, observation_space=gym.spaces.Dict( { "global": gym.spaces.Box( @@ -116,10 +89,10 @@ def test_customized_multi_agent_spec(self): } ), action_space=gym.spaces.Discrete(action_dims[0]), - model_config={"fcnet_hiddens": [128]}, + model_config_dict={"fcnet_hiddens": [128]}, ), "agent_2": SingleAgentRLModuleSpec( - module_class=module_cls, + module_class=rl_module_cls, observation_space=gym.spaces.Dict( { "global": gym.spaces.Box( @@ -131,7 +104,7 @@ def test_customized_multi_agent_spec(self): } ), action_space=gym.spaces.Discrete(action_dims[1]), - model_config={"fcnet_hiddens": [128]}, + model_config_dict={"fcnet_hiddens": [128]}, ), }, ) @@ -145,6 +118,156 @@ def test_customized_multi_agent_spec(self): foo.data = torch.ones_like(foo.data) self.assertTrue(torch.allclose(model["agent_2"].encoder[0].bias, foo)) + def test_get_spec_from_module_multi_agent(self): + """Tests wether MultiAgentRLModuleSpec.from_module() works.""" + env = gym.make("CartPole-v1") + num_agents = 2 + for module_class in MODULES: + module_specs = {} + for i in range(num_agents): + module_specs[f"module_{i}"] = SingleAgentRLModuleSpec( + module_class=module_class, + observation_space=env.observation_space, + action_space=env.action_space, + model_config_dict={"fcnet_hiddens": [32 * (i + 1)]}, + ) + + spec = MultiAgentRLModuleSpec(module_specs=module_specs) + module = spec.build() + + spec_from_module = MultiAgentRLModuleSpec.from_module(module) + self.assertEqual(spec, spec_from_module) + + def test_get_spec_from_module_single_agent(self): + """Tests wether SingleAgentRLModuleSpec.from_module() works.""" + env = gym.make("CartPole-v1") + for module_class in MODULES: + spec = SingleAgentRLModuleSpec( + module_class=module_class, + observation_space=env.observation_space, + action_space=env.action_space, + model_config_dict={"fcnet_hiddens": [32]}, + ) + + module = spec.build() + spec_from_module = SingleAgentRLModuleSpec.from_module(module) + self.assertEqual(spec, spec_from_module) + + def test_update_specs(self): + """Tests wether SingleAgentRLModuleSpec.update() works.""" + env = gym.make("CartPole-v0") + + # Test if SingleAgentRLModuleSpec.update() works. + module_spec_1 = SingleAgentRLModuleSpec( + module_class=DiscreteBCTorchModule, + observation_space=env.observation_space, + action_space=env.action_space, + model_config_dict="Update me!", + ) + module_spec_2 = SingleAgentRLModuleSpec( + model_config_dict={"fcnet_hiddens": [32]} + ) + self.assertEqual(module_spec_1.model_config_dict, "Update me!") + module_spec_1.update(module_spec_2) + self.assertEqual(module_spec_1.model_config_dict, {"fcnet_hiddens": [32]}) + + def test_update_specs_multi_agent(self): + """Test if updating a SingleAgentRLModuleSpec in MultiAgentRLModuleSpec works. + + This tests if we can update a `model_config_dict` field through different + kinds of updates: + - Create a SingleAgentRLModuleSpec and update its model_config_dict. + - Create two MultiAgentRLModuleSpecs and update the first one with the + second one without overwriting it. + - Check if the updated MultiAgentRLModuleSpec does not(!) have the + updated model_config_dict. + - Create two MultiAgentRLModuleSpecs and update the first one with the + second one with overwriting it. + - Check if the updated MultiAgentRLModuleSpec has(!) the updated + model_config_dict. + + """ + env = gym.make("CartPole-v0") + + # Test if SingleAgentRLModuleSpec.update() works. + module_spec_1 = SingleAgentRLModuleSpec( + module_class=DiscreteBCTorchModule, + observation_space="Do not update me!", + action_space=env.action_space, + model_config_dict="Update me!", + ) + module_spec_2 = SingleAgentRLModuleSpec( + model_config_dict={"fcnet_hiddens": [32]}, + ) + + self.assertEqual(module_spec_1.model_config_dict, "Update me!") + module_spec_1.update(module_spec_2) + self.assertEqual(module_spec_1.module_class, DiscreteBCTorchModule) + self.assertEqual(module_spec_1.observation_space, "Do not update me!") + self.assertEqual(module_spec_1.action_space, env.action_space) + self.assertEqual( + module_spec_1.model_config_dict, module_spec_2.model_config_dict + ) + + # Redefine module_spec_1 for following tests. + module_spec_1 = SingleAgentRLModuleSpec( + module_class=DiscreteBCTorchModule, + observation_space="Do not update me!", + action_space=env.action_space, + model_config_dict="Update me!", + ) + + marl_spec_1 = MultiAgentRLModuleSpec( + marl_module_class=BCTorchMultiAgentModuleWithSharedEncoder, + module_specs={"agent_1": module_spec_1}, + ) + marl_spec_2 = MultiAgentRLModuleSpec( + marl_module_class=BCTorchMultiAgentModuleWithSharedEncoder, + module_specs={"agent_1": module_spec_2}, + ) + + # Test if updating MultiAgentRLModuleSpec with overwriting works. This means + # that the single agent specs should be overwritten + self.assertEqual( + marl_spec_1.module_specs["agent_1"].model_config_dict, "Update me!" + ) + marl_spec_1.update(marl_spec_2, overwrite=True) + self.assertEqual(marl_spec_1.module_specs["agent_1"], module_spec_2) + + # Test if updating MultiAgentRLModuleSpec without overwriting works. This + # means that the single agent specs should not be overwritten + marl_spec_3 = MultiAgentRLModuleSpec( + marl_module_class=BCTorchMultiAgentModuleWithSharedEncoder, + module_specs={"agent_1": module_spec_1}, + ) + + self.assertEqual( + marl_spec_3.module_specs["agent_1"].observation_space, "Do not update me!" + ) + marl_spec_3.update(marl_spec_2, overwrite=False) + # If we would overwrite, we would replace the observation space even though + # it was None. This is not the case here. + self.assertEqual( + marl_spec_3.module_specs["agent_1"].observation_space, "Do not update me!" + ) + + # Test if updating with an additional SingleAgentRLModuleSpec works. + module_spec_3 = SingleAgentRLModuleSpec( + module_class=DiscreteBCTorchModule, + observation_space=env.observation_space, + action_space=env.action_space, + model_config_dict="I'm new!", + ) + marl_spec_3 = MultiAgentRLModuleSpec( + marl_module_class=BCTorchMultiAgentModuleWithSharedEncoder, + module_specs={"agent_2": module_spec_3}, + ) + self.assertEqual(marl_spec_1.module_specs.get("agent_2"), None) + marl_spec_1.update(marl_spec_3) + self.assertEqual( + marl_spec_1.module_specs["agent_2"].model_config_dict, "I'm new!" + ) + if __name__ == "__main__": import pytest diff --git a/rllib/core/rl_module/tf/tests/test_tf_rl_module.py b/rllib/core/rl_module/tf/tests/test_tf_rl_module.py index 78164d37cd06d..98fe5e71765fb 100644 --- a/rllib/core/rl_module/tf/tests/test_tf_rl_module.py +++ b/rllib/core/rl_module/tf/tests/test_tf_rl_module.py @@ -1,13 +1,13 @@ import gymnasium as gym import tensorflow as tf import tensorflow_probability as tfp -import threading +import tempfile from typing import Mapping import unittest +from ray.rllib.core.rl_module.rl_module import RLModuleConfig from ray.rllib.core.rl_module.tf.tf_rl_module import TfRLModule from ray.rllib.core.testing.tf.bc_module import DiscreteBCTFModule -from ray.rllib.utils.error import NotSerializable from ray.rllib.utils.test_utils import check @@ -15,10 +15,12 @@ class TestRLModule(unittest.TestCase): def test_compilation(self): env = gym.make("CartPole-v1") - module = DiscreteBCTFModule.from_model_config( - env.observation_space, - env.action_space, - model_config_dict={"fcnet_hiddens": [32]}, + module = DiscreteBCTFModule( + config=RLModuleConfig( + env.observation_space, + env.action_space, + model_config_dict={"fcnet_hiddens": [32]}, + ) ) self.assertIsInstance(module, TfRLModule) @@ -27,12 +29,13 @@ def test_forward_train(self): bsize = 1024 env = gym.make("CartPole-v1") - module = DiscreteBCTFModule.from_model_config( - env.observation_space, - env.action_space, - model_config_dict={"fcnet_hiddens": [32]}, + module = DiscreteBCTFModule( + config=RLModuleConfig( + env.observation_space, + env.action_space, + model_config_dict={"fcnet_hiddens": [32]}, + ) ) - obs_shape = env.observation_space.shape obs = tf.random.uniform((bsize,) + obs_shape) actions = tf.stack( @@ -59,10 +62,12 @@ def test_forward(self): """Test forward inference and exploration of""" env = gym.make("CartPole-v1") - module = DiscreteBCTFModule.from_model_config( - env.observation_space, - env.action_space, - model_config_dict={"fcnet_hiddens": [32]}, + module = DiscreteBCTFModule( + config=RLModuleConfig( + env.observation_space, + env.action_space, + model_config_dict={"fcnet_hiddens": [32]}, + ) ) obs_shape = env.observation_space.shape @@ -75,19 +80,23 @@ def test_forward(self): def test_get_set_state(self): env = gym.make("CartPole-v1") - module = DiscreteBCTFModule.from_model_config( - env.observation_space, - env.action_space, - model_config_dict={"fcnet_hiddens": [32]}, + module = DiscreteBCTFModule( + config=RLModuleConfig( + env.observation_space, + env.action_space, + model_config_dict={"fcnet_hiddens": [32]}, + ) ) state = module.get_state() self.assertIsInstance(state, dict) - module2 = DiscreteBCTFModule.from_model_config( - env.observation_space, - env.action_space, - model_config_dict={"fcnet_hiddens": [32]}, + module2 = DiscreteBCTFModule( + config=RLModuleConfig( + env.observation_space, + env.action_space, + model_config_dict={"fcnet_hiddens": [32]}, + ) ) state2 = module2.get_state() check(state["policy"][0], state2["policy"][0], false=True) @@ -96,42 +105,22 @@ def test_get_set_state(self): state2_after = module2.get_state() check(state, state2_after) - def test_serialize_deserialize(self): + def test_checkpointing(self): env = gym.make("CartPole-v1") - module = DiscreteBCTFModule.from_model_config( - env.observation_space, - env.action_space, - model_config_dict={"fcnet_hiddens": [32]}, + module = DiscreteBCTFModule( + config=RLModuleConfig( + env.observation_space, + env.action_space, + model_config_dict={"fcnet_hiddens": [32]}, + ) ) + with tempfile.TemporaryDirectory() as tmpdir: + module.save_to_checkpoint(tmpdir) + new_module = DiscreteBCTFModule.from_checkpoint(tmpdir) - # create a new module from the old module - new_module = module.deserialize(module.serialize()) - - # check that the new module is the same type - self.assertIsInstance(new_module, type(module)) - - # check that a parameter of their's is the same - self.assertEqual(new_module._input_dim, module._input_dim) - - # check that their states are the same check(module.get_state(), new_module.get_state()) - - # check that these 2 objects are not the same object self.assertNotEqual(id(module), id(new_module)) - # check that unpickleable parameters are not allowed by the RL Module - # constructor - unpickleable_param = threading.Thread() - - def bad_constructor(): - return DiscreteBCTFModule( - input_dim=unpickleable_param, - hidden_dim=unpickleable_param, - output_dim=unpickleable_param, - ) - - self.assertRaises(NotSerializable, bad_constructor) - if __name__ == "__main__": import pytest diff --git a/rllib/core/rl_module/tf/tf_rl_module.py b/rllib/core/rl_module/tf/tf_rl_module.py index b746242c4ae37..3322b2fb7f1f2 100644 --- a/rllib/core/rl_module/tf/tf_rl_module.py +++ b/rllib/core/rl_module/tf/tf_rl_module.py @@ -1,4 +1,5 @@ -from typing import Any, Mapping +import pathlib +from typing import Any, Mapping, Union from ray.rllib.core.rl_module import RLModule from ray.rllib.utils.annotations import override @@ -42,6 +43,20 @@ def get_state(self) -> Mapping[str, Any]: def set_state(self, state_dict: Mapping[str, Any]) -> None: self.set_weights(state_dict) + @override(RLModule) + def save_state_to_file(self, path: Union[str, pathlib.Path]) -> str: + if isinstance(path, str): + path = pathlib.Path(path) + module_state_dir = path / "module_state" + module_state_dir.mkdir(parents=True, exist_ok=True) + module_state_path = module_state_dir / "module_state" + self.save_weights(str(module_state_path), save_format="tf") + return str(module_state_path) + + @override(RLModule) + def load_state_from_file(self, path: Union[str, pathlib.Path]) -> None: + self.load_weights(path) + @override(RLModule) def make_distributed(self, dist_config: Mapping[str, Any] = None) -> None: """Makes the module distributed.""" diff --git a/rllib/core/rl_module/torch/tests/test_torch_rl_module.py b/rllib/core/rl_module/torch/tests/test_torch_rl_module.py index 7f2232f16c1d8..61747b906627a 100644 --- a/rllib/core/rl_module/torch/tests/test_torch_rl_module.py +++ b/rllib/core/rl_module/torch/tests/test_torch_rl_module.py @@ -1,12 +1,12 @@ -import threading import gymnasium as gym +import tempfile import torch from typing import Mapping import unittest +from ray.rllib.core.rl_module.rl_module import RLModuleConfig from ray.rllib.core.rl_module.torch import TorchRLModule from ray.rllib.core.testing.torch.bc_module import DiscreteBCTorchModule -from ray.rllib.utils.error import NotSerializable from ray.rllib.utils.test_utils import check @@ -14,10 +14,12 @@ class TestRLModule(unittest.TestCase): def test_compilation(self): env = gym.make("CartPole-v1") - module = DiscreteBCTorchModule.from_model_config( - env.observation_space, - env.action_space, - model_config_dict={"fcnet_hiddens": [32]}, + module = DiscreteBCTorchModule( + config=RLModuleConfig( + env.observation_space, + env.action_space, + model_config_dict={"fcnet_hiddens": [32]}, + ) ) self.assertIsInstance(module, TorchRLModule) @@ -26,10 +28,12 @@ def test_forward_train(self): bsize = 1024 env = gym.make("CartPole-v1") - module = DiscreteBCTorchModule.from_model_config( - env.observation_space, - env.action_space, - model_config_dict={"fcnet_hiddens": [32]}, + module = DiscreteBCTorchModule( + config=RLModuleConfig( + env.observation_space, + env.action_space, + model_config_dict={"fcnet_hiddens": [32]}, + ) ) obs_shape = env.observation_space.shape @@ -54,10 +58,12 @@ def test_forward(self): """Test forward inference and exploration of""" env = gym.make("CartPole-v1") - module = DiscreteBCTorchModule.from_model_config( - env.observation_space, - env.action_space, - model_config_dict={"fcnet_hiddens": [32]}, + module = DiscreteBCTorchModule( + config=RLModuleConfig( + env.observation_space, + env.action_space, + model_config_dict={"fcnet_hiddens": [32]}, + ) ) obs_shape = env.observation_space.shape @@ -70,19 +76,23 @@ def test_forward(self): def test_get_set_state(self): env = gym.make("CartPole-v1") - module = DiscreteBCTorchModule.from_model_config( - env.observation_space, - env.action_space, - model_config_dict={"fcnet_hiddens": [32]}, + module = DiscreteBCTorchModule( + config=RLModuleConfig( + env.observation_space, + env.action_space, + model_config_dict={"fcnet_hiddens": [32]}, + ) ) state = module.get_state() self.assertIsInstance(state, dict) - module2 = DiscreteBCTorchModule.from_model_config( - env.observation_space, - env.action_space, - model_config_dict={"fcnet_hiddens": [32]}, + module2 = DiscreteBCTorchModule( + config=RLModuleConfig( + env.observation_space, + env.action_space, + model_config_dict={"fcnet_hiddens": [32]}, + ) ) state2 = module2.get_state() check(state, state2, false=True) @@ -91,42 +101,22 @@ def test_get_set_state(self): state2_after = module2.get_state() check(state, state2_after) - def test_serialize_deserialize(self): + def test_checkpointing(self): env = gym.make("CartPole-v1") - module = DiscreteBCTorchModule.from_model_config( - env.observation_space, - env.action_space, - model_config_dict={"fcnet_hiddens": [32]}, + module = DiscreteBCTorchModule( + config=RLModuleConfig( + env.observation_space, + env.action_space, + model_config_dict={"fcnet_hiddens": [32]}, + ) ) + with tempfile.TemporaryDirectory() as tmpdir: + module.save_to_checkpoint(tmpdir) + new_module = DiscreteBCTorchModule.from_checkpoint(tmpdir) - # create a new module from the old module - new_module = module.deserialize(module.serialize()) - - # check that the new module is the same type - self.assertIsInstance(new_module, type(module)) - - # check that a parameter of their's is the same - self.assertEqual(new_module.input_dim, module.input_dim) - - # check that their states are the same check(module.get_state(), new_module.get_state()) - - # check that these 2 objects are not the same object self.assertNotEqual(id(module), id(new_module)) - # check that unpickleable parameters are not allowed by the RL Module - # constructor - unpickleable_param = threading.Thread() - - def bad_constructor(): - return DiscreteBCTorchModule( - input_dim=unpickleable_param, - hidden_dim=unpickleable_param, - output_dim=unpickleable_param, - ) - - self.assertRaises(NotSerializable, bad_constructor) - if __name__ == "__main__": import pytest diff --git a/rllib/core/rl_module/torch/torch_rl_module.py b/rllib/core/rl_module/torch/torch_rl_module.py index dcf4e5759678e..03741f8a015f3 100644 --- a/rllib/core/rl_module/torch/torch_rl_module.py +++ b/rllib/core/rl_module/torch/torch_rl_module.py @@ -1,7 +1,9 @@ -from typing import Any, Mapping +import pathlib +from typing import Any, Mapping, Union + +from ray.rllib.core.rl_module import RLModule from ray.rllib.utils.annotations import override from ray.rllib.utils.framework import try_import_torch -from ray.rllib.core.rl_module import RLModule torch, nn = try_import_torch() @@ -27,6 +29,24 @@ def get_state(self) -> Mapping[str, Any]: def set_state(self, state_dict: Mapping[str, Any]) -> None: self.load_state_dict(state_dict) + @override(RLModule) + def save_state_to_file(self, path: Union[str, pathlib.Path]) -> str: + if isinstance(path, str): + path = pathlib.Path(path) + module_state_path = path / "module_state.pt" + torch.save(self.state_dict(), str(module_state_path)) + return str(module_state_path) + + @override(RLModule) + def load_state_from_file(self, path: Union[str, pathlib.Path]) -> None: + if isinstance(path, str): + path = pathlib.Path(path) + if not path.exists(): + raise ValueError( + f"While loading state from path, the path does not exist: {path}" + ) + self.set_state(torch.load(str(path))) + @override(RLModule) def make_distributed(self, dist_config: Mapping[str, Any] = None) -> None: """Makes the module distributed.""" @@ -40,10 +60,10 @@ def is_distributed(self) -> bool: return False -class TorchDDPRLModule(nn.parallel.DistributedDataParallel, RLModule): +class TorchDDPRLModule(RLModule, nn.parallel.DistributedDataParallel): def __init__(self, *args, **kwargs) -> None: nn.parallel.DistributedDataParallel.__init__(self, *args, **kwargs) - # we do not want to call RLModule.__init__ here because it will all we need is + # we do not want to call RLModule.__init__ here because all we need is # the interface of that base-class not the actual implementation. @override(RLModule) diff --git a/rllib/core/testing/tests/test_bc_algorithm.py b/rllib/core/testing/tests/test_bc_algorithm.py index 591889699dba1..b3eeb21b61cb0 100644 --- a/rllib/core/testing/tests/test_bc_algorithm.py +++ b/rllib/core/testing/tests/test_bc_algorithm.py @@ -4,15 +4,16 @@ import ray from ray.rllib.core.testing.torch.bc_module import ( DiscreteBCTorchModule, - BCTorchMultiAgentSpec, BCTorchRLModuleWithSharedGlobalEncoder, + BCTorchMultiAgentModuleWithSharedEncoder, ) from ray.rllib.core.testing.tf.bc_module import ( DiscreteBCTFModule, - BCTfMultiAgentSpec, BCTfRLModuleWithSharedGlobalEncoder, + BCTfMultiAgentModuleWithSharedEncoder, ) from ray.rllib.core.rl_module.rl_module import SingleAgentRLModuleSpec +from ray.rllib.core.rl_module.marl_module import MultiAgentRLModuleSpec from ray.rllib.core.testing.bc_algorithm import BCConfigTest from ray.rllib.utils.test_utils import framework_iterator from ray.rllib.examples.env.multi_agent import MultiAgentCartPole @@ -79,16 +80,18 @@ def test_bc_algorithm_w_custom_marl_module(self): for fw in ["torch"]: if fw == "torch": - spec = BCTorchMultiAgentSpec( + spec = MultiAgentRLModuleSpec( + marl_module_class=BCTorchMultiAgentModuleWithSharedEncoder, module_specs=SingleAgentRLModuleSpec( module_class=BCTorchRLModuleWithSharedGlobalEncoder - ) + ), ) else: - spec = BCTfMultiAgentSpec( + spec = MultiAgentRLModuleSpec( + marl_module_class=BCTfMultiAgentModuleWithSharedEncoder, module_specs=SingleAgentRLModuleSpec( module_class=BCTfRLModuleWithSharedGlobalEncoder - ) + ), ) policies = {"policy_1", "policy_2"} diff --git a/rllib/core/testing/tf/bc_module.py b/rllib/core/testing/tf/bc_module.py index 6147642cb9253..e776ab469072e 100644 --- a/rllib/core/testing/tf/bc_module.py +++ b/rllib/core/testing/tf/bc_module.py @@ -1,10 +1,12 @@ -import gymnasium as gym import tensorflow as tf import tensorflow_probability as tfp -from typing import Any, Mapping, Optional +from typing import Any, Mapping -from ray.rllib.core.rl_module.rl_module import RLModule -from ray.rllib.core.rl_module.marl_module import MultiAgentRLModuleSpec, ModuleID +from ray.rllib.core.rl_module.rl_module import RLModule, RLModuleConfig +from ray.rllib.core.rl_module.marl_module import ( + MultiAgentRLModule, + MultiAgentRLModuleConfig, +) from ray.rllib.core.rl_module.tf.tf_rl_module import TfRLModule from ray.rllib.models.specs.typing import SpecType from ray.rllib.policy.sample_batch import SampleBatch @@ -13,15 +15,12 @@ class DiscreteBCTFModule(TfRLModule): - def __init__( - self, - input_dim: int, - hidden_dim: int, - output_dim: int, - ) -> None: - super().__init__( - input_dim=input_dim, output_dim=output_dim, hidden_dim=hidden_dim - ) + def __init__(self, config: RLModuleConfig) -> None: + super().__init__(config) + + input_dim = self.config.observation_space.shape[0] + hidden_dim = self.config.model_config_dict["fcnet_hiddens"][0] + output_dim = self.config.action_space.n layers = [] layers.append(tf.keras.Input(shape=(input_dim,))) @@ -72,24 +71,6 @@ def get_state(self) -> Mapping[str, Any]: def set_state(self, state: Mapping[str, Any]) -> None: self.policy.set_weights(state["policy"]) - @classmethod - @override(RLModule) - def from_model_config( - cls, - observation_space: "gym.Space", - action_space: "gym.Space", - *, - model_config_dict: Mapping[str, Any], - ) -> "DiscreteBCTFModule": - - config = { - "input_dim": observation_space.shape[0], - "hidden_dim": model_config_dict["fcnet_hiddens"][0], - "output_dim": action_space.n, - } - - return cls(**config) - class BCTfRLModuleWithSharedGlobalEncoder(TfRLModule): def __init__(self, encoder, local_dim, hidden_dim, action_dim): @@ -133,15 +114,16 @@ def _common_forward(self, batch): return {"action_dist": tf.distributions.Categorical(logits=action_logits)} -class BCTfMultiAgentSpec(MultiAgentRLModuleSpec): - def build(self, module_id: Optional[ModuleID] = None): +class BCTfMultiAgentModuleWithSharedEncoder(MultiAgentRLModule): + def __init__(self, config: MultiAgentRLModuleConfig) -> None: + super().__init__(config) - self._check_before_build() # constructing the global encoder based on the observation_space of the first # module - module_spec = next(iter(self.module_specs.values())) + module_specs = self.config.modules + module_spec = next(iter(module_specs.values())) global_dim = module_spec.observation_space["global"].shape[0] - hidden_dim = module_spec.model_config["fcnet_hiddens"][0] + hidden_dim = module_spec.model_config_dict["fcnet_hiddens"][0] shared_encoder = tf.keras.Sequential( [ tf.keras.Input(shape=(global_dim,)), @@ -150,21 +132,18 @@ def build(self, module_id: Optional[ModuleID] = None): ] ) - if module_id: - return module_spec.module_class( + for module_id, module_spec in module_specs.items(): + self._rl_modules[module_id] = module_spec.module_class( encoder=shared_encoder, local_dim=module_spec.observation_space["local"].shape[0], hidden_dim=hidden_dim, action_dim=module_spec.action_space.n, ) - rl_modules = {} - for module_id, module_spec in self.module_specs.items(): - rl_modules[module_id] = module_spec.module_class( - encoder=shared_encoder, - local_dim=module_spec.observation_space["local"].shape[0], - hidden_dim=hidden_dim, - action_dim=module_spec.action_space.n, - ) + def serialize(self): + # TODO (Kourosh): Implement when needed. + raise NotImplementedError - return self.marl_module_class(rl_modules) + def deserialize(self, data): + # TODO (Kourosh): Implement when needed. + raise NotImplementedError diff --git a/rllib/core/testing/torch/bc_module.py b/rllib/core/testing/torch/bc_module.py index 06291d999e107..ff4f5abc281b0 100644 --- a/rllib/core/testing/torch/bc_module.py +++ b/rllib/core/testing/torch/bc_module.py @@ -1,8 +1,10 @@ -import gymnasium as gym -from typing import Any, Mapping, Optional +from typing import Any, Mapping -from ray.rllib.core.rl_module import RLModule -from ray.rllib.core.rl_module.marl_module import MultiAgentRLModuleSpec, ModuleID +from ray.rllib.core.rl_module.rl_module import RLModule, RLModuleConfig +from ray.rllib.core.rl_module.marl_module import ( + MultiAgentRLModuleConfig, + MultiAgentRLModule, +) from ray.rllib.core.rl_module.torch.torch_rl_module import TorchRLModule from ray.rllib.models.specs.typing import SpecType from ray.rllib.utils.annotations import override @@ -13,15 +15,13 @@ class DiscreteBCTorchModule(TorchRLModule): - def __init__( - self, - input_dim: int, - hidden_dim: int, - output_dim: int, - ) -> None: - super().__init__( - input_dim=input_dim, hidden_dim=hidden_dim, output_dim=output_dim - ) + def __init__(self, config: RLModuleConfig) -> None: + super().__init__(config) + + input_dim = self.config.observation_space.shape[0] + hidden_dim = self.config.model_config_dict["fcnet_hiddens"][0] + output_dim = self.config.action_space.n + self.policy = nn.Sequential( nn.Linear(input_dim, hidden_dim), nn.ReLU(), @@ -57,24 +57,6 @@ def _forward_train(self, batch: NestedDict) -> Mapping[str, Any]: action_logits = self.policy(batch["obs"]) return {"action_dist": torch.distributions.Categorical(logits=action_logits)} - @classmethod - @override(RLModule) - def from_model_config( - cls, - observation_space: "gym.Space", - action_space: "gym.Space", - *, - model_config_dict: Mapping[str, Any], - ) -> "DiscreteBCTorchModule": - - config = { - "input_dim": observation_space.shape[0], - "hidden_dim": model_config_dict["fcnet_hiddens"][0], - "output_dim": action_space.n, - } - - return cls(**config) - class BCTorchRLModuleWithSharedGlobalEncoder(TorchRLModule): """An example of an RLModule that uses an encoder shared with other things. @@ -92,7 +74,7 @@ class BCTorchRLModuleWithSharedGlobalEncoder(TorchRLModule): def __init__( self, encoder: nn.Module, local_dim: int, hidden_dim: int, action_dim: int ) -> None: - super().__init__() + super().__init__(config=None) self.encoder = encoder self.policy_head = nn.Sequential( @@ -128,34 +110,24 @@ def _common_forward(self, batch): return {"action_dist": torch.distributions.Categorical(logits=action_logits)} -class BCTorchMultiAgentSpec(MultiAgentRLModuleSpec): +class BCTorchMultiAgentModuleWithSharedEncoder(MultiAgentRLModule): + def __init__(self, config: MultiAgentRLModuleConfig) -> None: + super().__init__(config) - # TODO: make sure the default class is MultiAgentRLModule + def build(self): - def build(self, module_id: Optional[ModuleID] = None): - - self._check_before_build() - # constructing the global encoder based on the observation_space of the first - # module - module_spec = next(iter(self.module_specs.values())) + module_specs = self.config.modules + module_spec = next(iter(module_specs.values())) global_dim = module_spec.observation_space["global"].shape[0] - hidden_dim = module_spec.model_config["fcnet_hiddens"][0] + hidden_dim = module_spec.model_config_dict["fcnet_hiddens"][0] shared_encoder = nn.Sequential( nn.Linear(global_dim, hidden_dim), nn.ReLU(), nn.Linear(hidden_dim, hidden_dim), ) - if module_id: - return module_spec.module_class( - encoder=shared_encoder, - local_dim=module_spec.observation_space["local"].shape[0], - hidden_dim=hidden_dim, - action_dim=module_spec.action_space.n, - ) - rl_modules = {} - for module_id, module_spec in self.module_specs.items(): + for module_id, module_spec in module_specs.items(): rl_modules[module_id] = module_spec.module_class( encoder=shared_encoder, local_dim=module_spec.observation_space["local"].shape[0], @@ -163,4 +135,12 @@ def build(self, module_id: Optional[ModuleID] = None): action_dim=module_spec.action_space.n, ) - return self.marl_module_class(rl_modules) + self._rl_modules = rl_modules + + def serialize(self): + # TODO (Kourosh): Implement when needed. + raise NotImplementedError + + def deserialize(self, data): + # TODO (Kourosh): Implement when needed. + raise NotImplementedError diff --git a/rllib/core/testing/utils.py b/rllib/core/testing/utils.py index cc6b4258b0a09..c572466f79cdf 100644 --- a/rllib/core/testing/utils.py +++ b/rllib/core/testing/utils.py @@ -61,7 +61,7 @@ def get_module_spec(framework: str, env: "gym.Env", is_multi_agent: bool = False module_class=get_module_class(framework), observation_space=env.observation_space, action_space=env.action_space, - model_config={"fcnet_hiddens": [32]}, + model_config_dict={"fcnet_hiddens": [32]}, ) if is_multi_agent: diff --git a/rllib/env/multi_agent_env.py b/rllib/env/multi_agent_env.py index a65b8c3160736..081a5eae76060 100644 --- a/rllib/env/multi_agent_env.py +++ b/rllib/env/multi_agent_env.py @@ -662,11 +662,14 @@ def send_actions(self, action_dict: MultiEnvDict) -> None: assert isinstance(terminateds, dict), "Not a multi-agent terminateds dict!" assert isinstance(truncateds, dict), "Not a multi-agent truncateds dict!" assert isinstance(infos, dict), "Not a multi-agent info dict!" - if isinstance(obs, dict) and set(infos).difference(set(obs)): - raise ValueError( - "Key set for infos must be a subset of obs: " - "{} vs {}".format(infos.keys(), obs.keys()) - ) + if isinstance(obs, dict): + info_diff = set(infos).difference(set(obs)) + if info_diff and info_diff != {"__common__"}: + raise ValueError( + "Key set for infos must be a subset of obs (plus optionally " + "the '__common__' key for infos concerning all/no agents): " + "{} vs {}".format(infos.keys(), obs.keys()) + ) if "__all__" not in terminateds: raise ValueError( "In multi-agent environments, '__all__': True|False must " diff --git a/rllib/env/wrappers/atari_wrappers.py b/rllib/env/wrappers/atari_wrappers.py index a4bdb554bd87f..b836961669d92 100644 --- a/rllib/env/wrappers/atari_wrappers.py +++ b/rllib/env/wrappers/atari_wrappers.py @@ -212,7 +212,9 @@ def __init__(self, env, skip=4): """Return only every `skip`-th frame""" gym.Wrapper.__init__(self, env) # most recent raw observations (for max pooling across time steps) - self._obs_buffer = np.zeros((2,) + env.observation_space.shape, dtype=np.uint8) + self._obs_buffer = np.zeros( + (2,) + env.observation_space.shape, dtype=env.observation_space.dtype + ) self._skip = skip def step(self, action): diff --git a/rllib/models/tf/tf_action_dist.py b/rllib/models/tf/tf_action_dist.py index c33729618b945..a28ac4cce4225 100644 --- a/rllib/models/tf/tf_action_dist.py +++ b/rllib/models/tf/tf_action_dist.py @@ -197,7 +197,7 @@ def required_model_output_shape( high_ = np.max(action_space.high) assert np.all(action_space.low == low_) assert np.all(action_space.high == high_) - np.prod(action_space.shape, dtype=np.int32) * (high_ - low_ + 1) + return np.prod(action_space.shape, dtype=np.int32) * (high_ - low_ + 1) # MultiDiscrete space. else: # nvec is already integer, so no casting needed. diff --git a/rllib/models/tf/tf_distributions.py b/rllib/models/tf/tf_distributions.py index 26f8e2b55c834..91cc995722e06 100644 --- a/rllib/models/tf/tf_distributions.py +++ b/rllib/models/tf/tf_distributions.py @@ -31,7 +31,7 @@ def __init__(self, *args, **kwargs): self._dist = self._get_tf_distribution(*args, **kwargs) @abc.abstractmethod - def _get_tf_distribution(self, *args, **kwargs) -> tfp.distributions.Distribution: + def _get_tf_distribution(self, *args, **kwargs) -> "tfp.distributions.Distribution": """Returns the tfp.distributions.Distribution object to use.""" @override(Distribution) @@ -118,7 +118,7 @@ def _get_tf_distribution( probs: tf.Tensor = None, logits: tf.Tensor = None, temperature: float = 1.0, - ) -> tfp.distributions.Distribution: + ) -> "tfp.distributions.Distribution": if logits is not None: assert temperature > 0.0, "Categorical `temperature` must be > 0.0!" logits /= temperature @@ -129,7 +129,7 @@ def _get_tf_distribution( def required_model_output_shape( space: gym.Space, model_config: ModelConfigDict ) -> Tuple[int, ...]: - return (space.n,) + return (int(space.n),) @override(TfDistribution) def _rsample(self, sample_shape=()): @@ -179,7 +179,7 @@ def __init__( super().__init__(loc=loc, scale=scale) @override(TfDistribution) - def _get_tf_distribution(self, loc, scale=None) -> tfp.distributions.Distribution: + def _get_tf_distribution(self, loc, scale=None) -> "tfp.distributions.Distribution": if scale is None: loc, log_scale = tf.split(loc, num_or_size_splits=2, axis=-1) scale = tf.exp(log_scale) @@ -202,7 +202,7 @@ def kl(self, other: "TfDistribution") -> TensorType: def required_model_output_shape( space: gym.Space, model_config: ModelConfigDict ) -> Tuple[int, ...]: - return tuple(np.prod(space.shape, dtype=np.int32) * 2) + return (int(np.prod(space.shape, dtype=np.int32) * 2),) @override(TfDistribution) def _rsample(self, sample_shape=()): @@ -283,7 +283,7 @@ def required_model_output_shape( space: gym.Space, model_config: ModelConfigDict ) -> Tuple[int, ...]: # TODO: This was copied from previous code. Is this correct? add unit test. - return tuple(np.prod(space.shape, dtype=np.int32)) + return (int(np.prod(space.shape, dtype=np.int32)),) @classmethod @override(Distribution) diff --git a/rllib/models/torch/torch_action_dist.py b/rllib/models/torch/torch_action_dist.py index dadbec72f2f1c..12f725b9e10d6 100644 --- a/rllib/models/torch/torch_action_dist.py +++ b/rllib/models/torch/torch_action_dist.py @@ -182,7 +182,7 @@ def required_model_output_shape( high_ = np.max(action_space.high) assert np.all(action_space.low == low_) assert np.all(action_space.high == high_) - np.prod(action_space.shape, dtype=np.int32) * (high_ - low_ + 1) + return np.prod(action_space.shape, dtype=np.int32) * (high_ - low_ + 1) # MultiDiscrete space. else: # `nvec` is already integer. No need to cast. diff --git a/rllib/models/torch/torch_distributions.py b/rllib/models/torch/torch_distributions.py index 0cf9c5258081e..f70bde3c68cc6 100644 --- a/rllib/models/torch/torch_distributions.py +++ b/rllib/models/torch/torch_distributions.py @@ -28,7 +28,7 @@ def __init__(self, *args, **kwargs): @abc.abstractmethod def _get_torch_distribution( self, *args, **kwargs - ) -> torch.distributions.Distribution: + ) -> "torch.distributions.Distribution": """Returns the torch.distributions.Distribution object to use.""" @override(Distribution) @@ -108,7 +108,7 @@ def _get_torch_distribution( probs: torch.Tensor = None, logits: torch.Tensor = None, temperature: float = 1.0, - ) -> torch.distributions.Distribution: + ) -> "torch.distributions.Distribution": if logits is not None: assert temperature > 0.0, "Categorical `temperature` must be > 0.0!" logits /= temperature @@ -119,7 +119,7 @@ def _get_torch_distribution( def required_model_output_shape( space: gym.Space, model_config: ModelConfigDict ) -> Tuple[int, ...]: - return (space.n,) + return (int(space.n),) @classmethod @override(Distribution) @@ -165,7 +165,7 @@ def __init__( def _get_torch_distribution( self, loc, scale=None - ) -> torch.distributions.Distribution: + ) -> "torch.distributions.Distribution": if scale is None: loc, log_std = torch.chunk(self.inputs, 2, dim=1) scale = torch.exp(log_std) @@ -188,7 +188,7 @@ def kl(self, other: "TorchDistribution") -> TensorType: def required_model_output_shape( space: gym.Space, model_config: ModelConfigDict ) -> Tuple[int, ...]: - return tuple(np.prod(space.shape, dtype=np.int32) * 2) + return (int(np.prod(space.shape, dtype=np.int32) * 2),) @classmethod @override(Distribution) @@ -268,7 +268,7 @@ def required_model_output_shape( space: gym.Space, model_config: ModelConfigDict ) -> Tuple[int, ...]: # TODO: This was copied from previous code. Is this correct? add unit test. - return tuple(np.prod(space.shape, dtype=np.int32)) + return (int(np.prod(space.shape, dtype=np.int32)),) @classmethod @override(Distribution) diff --git a/rllib/policy/policy.py b/rllib/policy/policy.py index 0c6ab850f6b8e..567b134891a48 100644 --- a/rllib/policy/policy.py +++ b/rllib/policy/policy.py @@ -392,11 +392,18 @@ def make_rl_module(self) -> "RLModule": "bug, please file a github issue." ) - module_spec = self.config["__marl_module_spec"] - if isinstance(module_spec, SingleAgentRLModuleSpec): - module = module_spec.build() + spec = self.config["__marl_module_spec"] + if isinstance(spec, SingleAgentRLModuleSpec): + module = spec.build() else: - module = module_spec.build(module_id=self.__policy_id) + # filter the module_spec to only contain the policy_id of this policy + marl_spec = type(spec)( + marl_module_class=spec.marl_module_class, + module_specs={self.__policy_id: spec.module_specs[self.__policy_id]}, + ) + marl_module = marl_spec.build() + module = marl_module[self.__policy_id] + return module @DeveloperAPI @@ -1341,7 +1348,10 @@ def _initialize_loss_from_dummy_batch( # Save for later so that loss init does not change global timestep global_ts_before_init = int(convert_to_numpy(self.global_timestep)) - sample_batch_size = max(self.batch_divisibility_req * 4, 32) + sample_batch_size = min( + max(self.batch_divisibility_req * 4, 32), + self.config["train_batch_size"], # Don't go over the asked batch size. + ) self._dummy_batch = self._get_dummy_batch_from_view_requirements( sample_batch_size ) diff --git a/rllib/utils/framework.py b/rllib/utils/framework.py index 226b2a441a3d8..d5cc2ce9ca270 100644 --- a/rllib/utils/framework.py +++ b/rllib/utils/framework.py @@ -26,7 +26,7 @@ def try_import_jax(error: bool = False): """ if "RLLIB_TEST_NO_JAX_IMPORT" in os.environ: logger.warning("Not importing JAX for test purposes.") - return None + return None, None try: import jax diff --git a/rllib/utils/serialization.py b/rllib/utils/serialization.py index 90560c1b1b225..5ac2fd5918694 100644 --- a/rllib/utils/serialization.py +++ b/rllib/utils/serialization.py @@ -1,8 +1,9 @@ import base64 import numpy as np import io +import importlib import zlib -from typing import Dict, Any, Sequence +from typing import Dict, Any, Optional, Sequence, Type, Union import ray from ray.rllib.utils.annotations import DeveloperAPI @@ -73,17 +74,17 @@ def _box(sp: gym.spaces.Box) -> Dict: def _discrete(sp: gym.spaces.Discrete) -> Dict: d = { "space": "discrete", - "n": sp.n, + "n": int(sp.n), } # Offset is a relatively new Discrete space feature. if hasattr(sp, "start"): - d["start"] = sp.start + d["start"] = int(sp.start) return d def _multi_binary(sp: gym.spaces.MultiBinary) -> Dict: return { "space": "multi-binary", - "n": sp.n, + "n": int(sp.n), "dtype": sp.dtype.str, } @@ -325,3 +326,59 @@ def check_if_args_kwargs_serializable(args: Sequence[Any], kwargs: Dict[str, Any f"Found non-serializable keyword argument: {k} = {v}.\n" f"Original serialization error: {e}" ) + + +@DeveloperAPI +def serialize_type(type_: Union[Type, str]) -> str: + """Converts a type into its full classpath ([module file] + "." + [class name]). + Args: + type_: The type to convert. + Returns: + The full classpath of the given type, e.g. "ray.rllib.algorithms.ppo.PPOConfig". + """ + # TODO (avnishn): find a way to incorporate the tune registry here. + # Already serialized. + if isinstance(type_, str): + return type_ + + return type_.__module__ + "." + type_.__qualname__ + + +@DeveloperAPI +def deserialize_type( + module: Union[str, Type], error: bool = False +) -> Optional[Union[str, Type]]: + """Resolves a class path to a class. + If the given module is already a class, it is returned as is. + If the given module is a string, it is imported and the class is returned. + Args: + module: The classpath (str) or type to resolve. + error: Whether to throw a ValueError if `module` could not be resolved into + a class. If False and `module` is not resolvable, returns None. + Returns: + The resolved class or `module` (if `error` is False and no resolution possible). + Raises: + ValueError: If `error` is True and `module` cannot be resolved. + """ + if isinstance(module, type): + return module + + elif isinstance(module, str): + # Try interpreting (as classpath) and importing the given module. + try: + module_path, class_name = module.rsplit(".", 1) + module = importlib.import_module(module_path) + return getattr(module, class_name) + # Module not found. + except (ModuleNotFoundError, ImportError, AttributeError, ValueError) as e: + if error: + raise ValueError( + f"Could not deserialize the given classpath `module={module}` into " + "a valid python class! Make sure you have all necessary pip " + "packages installed and all custom modules are in your " + "`PYTHONPATH` env variable." + ) from e + else: + raise ValueError(f"`module` ({module} must be type or string (classpath)!") + + return module diff --git a/src/mock/ray/core_worker/task_manager.h b/src/mock/ray/core_worker/task_manager.h index 7795744b37ee8..2b166fdf4a37e 100644 --- a/src/mock/ray/core_worker/task_manager.h +++ b/src/mock/ray/core_worker/task_manager.h @@ -54,7 +54,7 @@ class MockTaskFinisherInterface : public TaskFinisherInterface { (const, override)); MOCK_METHOD(bool, RetryTaskIfPossible, - (const TaskID &task_id, bool task_failed_due_to_oom), + (const TaskID &task_id, const rpc::RayErrorInfo &error_info), (override)); MOCK_METHOD(void, MarkDependenciesResolved, (const TaskID &task_id), (override)); MOCK_METHOD(void, diff --git a/src/ray/core_worker/task_event_buffer.cc b/src/ray/core_worker/task_event_buffer.cc index 15731ad46628a..9c2bb74b01912 100644 --- a/src/ray/core_worker/task_event_buffer.cc +++ b/src/ray/core_worker/task_event_buffer.cc @@ -31,14 +31,12 @@ TaskStatusEvent::TaskStatusEvent( const rpc::TaskStatus &task_status, int64_t timestamp, const std::shared_ptr &task_spec, - absl::optional node_id, - absl::optional worker_id) + absl::optional state_update) : TaskEvent(task_id, job_id, attempt_number), task_status_(task_status), timestamp_(timestamp), task_spec_(task_spec), - node_id_(node_id), - worker_id_(worker_id) {} + state_update_(state_update) {} TaskProfileEvent::TaskProfileEvent(TaskID task_id, JobID job_id, @@ -67,22 +65,30 @@ void TaskStatusEvent::ToRpcTaskEvents(rpc::TaskEvents *rpc_task_events) { } // Task status update. - auto state_updates = rpc_task_events->mutable_state_updates(); + auto dst_state_update = rpc_task_events->mutable_state_updates(); + gcs::FillTaskStatusUpdateTime(task_status_, timestamp_, dst_state_update); - if (node_id_.has_value()) { + if (!state_update_.has_value()) { + return; + } + + if (state_update_->node_id_.has_value()) { RAY_CHECK(task_status_ == rpc::TaskStatus::SUBMITTED_TO_WORKER) << "Node ID should be included when task status changes to " "SUBMITTED_TO_WORKER."; - state_updates->set_node_id(node_id_->Binary()); + dst_state_update->set_node_id(state_update_->node_id_->Binary()); } - if (worker_id_.has_value()) { + if (state_update_->worker_id_.has_value()) { RAY_CHECK(task_status_ == rpc::TaskStatus::SUBMITTED_TO_WORKER) << "Worker ID should be included when task status changes to " "SUBMITTED_TO_WORKER."; - state_updates->set_worker_id(worker_id_->Binary()); + dst_state_update->set_worker_id(state_update_->worker_id_->Binary()); + } + + if (state_update_->error_info_.has_value()) { + dst_state_update->set_error_type(state_update_->error_info_->error_type()); } - gcs::FillTaskStatusUpdateTime(task_status_, timestamp_, state_updates); } void TaskProfileEvent::ToRpcTaskEvents(rpc::TaskEvents *rpc_task_events) { diff --git a/src/ray/core_worker/task_event_buffer.h b/src/ray/core_worker/task_event_buffer.h index 2ee359846ffc3..c00aa8d145149 100644 --- a/src/ray/core_worker/task_event_buffer.h +++ b/src/ray/core_worker/task_event_buffer.h @@ -69,6 +69,25 @@ class TaskEvent { /// TaskStatusEvent is generated when a task changes its status. class TaskStatusEvent : public TaskEvent { public: + /// A class that contain data that will be converted to rpc::TaskStateUpdate + struct TaskStateUpdate { + TaskStateUpdate(const absl::optional &error_info) + : error_info_(error_info) {} + + TaskStateUpdate(const NodeID &node_id, const WorkerID &worker_id) + : node_id_(node_id), worker_id_(worker_id) {} + + private: + friend class TaskStatusEvent; + + /// Node id if it's a SUBMITTED_TO_WORKER status change. + const absl::optional node_id_ = absl::nullopt; + /// Worker id if it's a SUBMITTED_TO_WORKER status change. + const absl::optional worker_id_ = absl::nullopt; + /// Task error info. + const absl::optional error_info_ = absl::nullopt; + }; + explicit TaskStatusEvent( TaskID task_id, JobID job_id, @@ -76,8 +95,7 @@ class TaskStatusEvent : public TaskEvent { const rpc::TaskStatus &task_status, int64_t timestamp, const std::shared_ptr &task_spec = nullptr, - absl::optional node_id = absl::nullopt, - absl::optional worker_id = absl::nullopt); + absl::optional state_update = absl::nullopt); void ToRpcTaskEvents(rpc::TaskEvents *rpc_task_events) override; @@ -90,10 +108,8 @@ class TaskStatusEvent : public TaskEvent { const int64_t timestamp_ = -1; /// Pointer to the task spec. const std::shared_ptr task_spec_ = nullptr; - /// Node id if it's a SUBMITTED_TO_WORKER status change. - const absl::optional node_id_ = absl::nullopt; - /// Worker id if it's a SUBMITTED_TO_WORKER status change. - const absl::optional worker_id_ = absl::nullopt; + /// Pointer to the task state update + absl::optional state_update_ = absl::nullopt; }; /// TaskProfileEvent is generated when `RAY_enable_timeline` is on. diff --git a/src/ray/core_worker/task_manager.cc b/src/ray/core_worker/task_manager.cc index f7126fcff0fe7..794dbc3a64e90 100644 --- a/src/ray/core_worker/task_manager.cc +++ b/src/ray/core_worker/task_manager.cc @@ -381,7 +381,9 @@ void TaskManager::CompletePendingTask(const TaskID &task_id, it->second.num_successful_executions++; if (is_application_error) { - SetTaskStatus(it->second, rpc::TaskStatus::FAILED); + SetTaskStatus(it->second, + rpc::TaskStatus::FAILED, + gcs::GetRayErrorInfo(rpc::ErrorType::TASK_EXECUTION_EXCEPTION)); } else { SetTaskStatus(it->second, rpc::TaskStatus::FINISHED); } @@ -420,11 +422,12 @@ void TaskManager::CompletePendingTask(const TaskID &task_id, } bool TaskManager::RetryTaskIfPossible(const TaskID &task_id, - bool task_failed_due_to_oom) { + const rpc::RayErrorInfo &error_info) { TaskSpecification spec; bool will_retry = false; int32_t num_retries_left = 0; int32_t num_oom_retries_left = 0; + bool task_failed_due_to_oom = error_info.error_type() == rpc::ErrorType::OUT_OF_MEMORY; { absl::MutexLock lock(&mu_); auto it = submissible_tasks_.find(task_id); @@ -455,7 +458,7 @@ bool TaskManager::RetryTaskIfPossible(const TaskID &task_id, } } if (will_retry) { - MarkTaskRetryOnFailed(it->second); + MarkTaskRetryOnFailed(it->second, error_info); } } @@ -493,7 +496,9 @@ void TaskManager::FailPendingTask(const TaskID &task_id, // Note that this might be the __ray_terminate__ task, so we don't log // loudly with ERROR here. RAY_LOG(DEBUG) << "Task " << task_id << " failed with error " - << rpc::ErrorType_Name(error_type); + << rpc::ErrorType_Name(error_type) << ", ray_error_info: " + << ((ray_error_info == nullptr) ? "nullptr" + : ray_error_info->DebugString()); TaskSpecification spec; // Check whether the error should be stored in plasma or not. @@ -506,7 +511,10 @@ void TaskManager::FailPendingTask(const TaskID &task_id, RAY_CHECK(it->second.IsPending()) << "Tried to fail task that was not pending " << task_id; spec = it->second.spec; - SetTaskStatus(it->second, rpc::TaskStatus::FAILED); + SetTaskStatus( + it->second, + rpc::TaskStatus::FAILED, + ray_error_info == nullptr ? gcs::GetRayErrorInfo(error_type) : *ray_error_info); submissible_tasks_.erase(it); num_pending_tasks_--; @@ -556,7 +564,8 @@ bool TaskManager::FailOrRetryPendingTask(const TaskID &task_id, bool will_retry = false; if (!fail_immediately) { will_retry = RetryTaskIfPossible( - task_id, /*task_failed_due_to_oom*/ error_type == rpc::ErrorType::OUT_OF_MEMORY); + task_id, + ray_error_info == nullptr ? gcs::GetRayErrorInfo(error_type) : *ray_error_info); } if (!will_retry && mark_task_object_failed) { @@ -801,8 +810,7 @@ void TaskManager::MarkTaskWaitingForExecution(const TaskID &task_id, it->second.spec, rpc::TaskStatus::SUBMITTED_TO_WORKER, /* include_task_info */ false, - node_id, - worker_id); + worker::TaskStatusEvent::TaskStateUpdate(node_id, worker_id)); } void TaskManager::MarkTaskRetryOnResubmit(TaskEntry &task_entry) { @@ -822,10 +830,14 @@ void TaskManager::MarkTaskRetryOnResubmit(TaskEntry &task_entry) { /* include_task_info */ true); } -void TaskManager::MarkTaskRetryOnFailed(TaskEntry &task_entry) { +void TaskManager::MarkTaskRetryOnFailed(TaskEntry &task_entry, + const rpc::RayErrorInfo &error_info) { // Record the old attempt status as FAILED. - RecordTaskStatusEvent( - task_entry.spec.AttemptNumber(), task_entry.spec, rpc::TaskStatus::FAILED); + RecordTaskStatusEvent(task_entry.spec.AttemptNumber(), + task_entry.spec, + rpc::TaskStatus::FAILED, + /* include_task_info */ false, + worker::TaskStatusEvent::TaskStateUpdate(error_info)); task_entry.MarkRetryOnFailed(); // Mark the new status and also include task spec info for the new attempt. @@ -836,9 +848,16 @@ void TaskManager::MarkTaskRetryOnFailed(TaskEntry &task_entry) { /* include_task_info */ true); } -void TaskManager::SetTaskStatus(TaskEntry &task_entry, rpc::TaskStatus status) { +void TaskManager::SetTaskStatus( + TaskEntry &task_entry, + rpc::TaskStatus status, + const absl::optional &error_info) { task_entry.SetStatus(status); - RecordTaskStatusEvent(task_entry.spec.AttemptNumber(), task_entry.spec, status); + RecordTaskStatusEvent(task_entry.spec.AttemptNumber(), + task_entry.spec, + status, + /* include_task_info */ false, + worker::TaskStatusEvent::TaskStateUpdate(error_info)); } void TaskManager::FillTaskInfo(rpc::GetCoreWorkerStatsReply *reply, @@ -892,12 +911,12 @@ void TaskManager::RecordMetrics() { task_counter_.FlushOnChangeCallbacks(); } -void TaskManager::RecordTaskStatusEvent(int32_t attempt_number, - const TaskSpecification &spec, - rpc::TaskStatus status, - bool include_task_info, - absl::optional node_id, - absl::optional worker_id) { +void TaskManager::RecordTaskStatusEvent( + int32_t attempt_number, + const TaskSpecification &spec, + rpc::TaskStatus status, + bool include_task_info, + absl::optional state_update) { if (!task_event_buffer_.Enabled()) { return; } @@ -908,8 +927,7 @@ void TaskManager::RecordTaskStatusEvent(int32_t attempt_number, status, /* timestamp */ absl::GetCurrentTimeNanos(), include_task_info ? std::make_shared(spec) : nullptr, - node_id, - worker_id); + std::move(state_update)); task_event_buffer_.AddTaskEvent(std::move(task_event)); } diff --git a/src/ray/core_worker/task_manager.h b/src/ray/core_worker/task_manager.h index 6b17729700e93..dca839dc8a50a 100644 --- a/src/ray/core_worker/task_manager.h +++ b/src/ray/core_worker/task_manager.h @@ -38,7 +38,7 @@ class TaskFinisherInterface { bool is_application_error) = 0; virtual bool RetryTaskIfPossible(const TaskID &task_id, - bool task_failed_due_to_oom) = 0; + const rpc::RayErrorInfo &error_info) = 0; virtual void FailPendingTask(const TaskID &task_id, rpc::ErrorType error_type, @@ -170,10 +170,9 @@ class TaskManager : public TaskFinisherInterface, public TaskResubmissionInterfa /// Returns true if task can be retried. /// /// \param[in] task_id ID of the task to be retried. - /// \param[in] task_failed_due_to_oom last task attempt failed due to node running out - /// of memory. /// \return true if task is scheduled to be retried. - bool RetryTaskIfPossible(const TaskID &task_id, bool task_failed_due_to_oom) override; + bool RetryTaskIfPossible(const TaskID &task_id, + const rpc::RayErrorInfo &error_info) override; /// A pending task failed. This will either retry the task or mark the task /// as failed if there are no retries left. @@ -317,16 +316,13 @@ class TaskManager : public TaskFinisherInterface, public TaskResubmissionInterfa /// \param spec corresponding TaskSpecification of the task /// \param status the changed status. /// \param include_task_info True if TaskInfoEntry will be added to the Task events. - /// \param node_id Node ID of the worker for which the task's submitted. Only applicable - /// for SUBMITTED_TO_WORKER status change. - /// \param worker_id Worker ID of the worker for which the task's submitted. Only - /// applicable for SUBMITTED_TO_WORKER status change. - void RecordTaskStatusEvent(int32_t attempt_number, - const TaskSpecification &spec, - rpc::TaskStatus status, - bool include_task_info = false, - absl::optional node_id = absl::nullopt, - absl::optional worker_id = absl::nullopt); + void RecordTaskStatusEvent( + int32_t attempt_number, + const TaskSpecification &spec, + rpc::TaskStatus status, + bool include_task_info = false, + absl::optional state_update = + absl::nullopt); private: struct TaskEntry { @@ -487,7 +483,11 @@ class TaskManager : public TaskFinisherInterface, public TaskResubmissionInterfa /// /// \param task_entry corresponding TaskEntry of a task to record the event. /// \param status new status. - void SetTaskStatus(TaskEntry &task_entry, rpc::TaskStatus status); + /// \param error_info Optional error info for task execution. + void SetTaskStatus( + TaskEntry &task_entry, + rpc::TaskStatus status, + const absl::optional &error_info = absl::nullopt); /// Update the task entry for the task attempt to reflect retry on resubmit. /// @@ -503,7 +503,7 @@ class TaskManager : public TaskFinisherInterface, public TaskResubmissionInterfa /// the retry counter. /// /// \param task_entry Task entry for the corresponding task attempt - void MarkTaskRetryOnFailed(TaskEntry &task_entry); + void MarkTaskRetryOnFailed(TaskEntry &task_entry, const rpc::RayErrorInfo &error_info); /// Used to store task results. std::shared_ptr in_memory_store_; diff --git a/src/ray/core_worker/test/dependency_resolver_test.cc b/src/ray/core_worker/test/dependency_resolver_test.cc index 241cf0e5e31dc..4d2406e006ec2 100644 --- a/src/ray/core_worker/test/dependency_resolver_test.cc +++ b/src/ray/core_worker/test/dependency_resolver_test.cc @@ -69,7 +69,8 @@ class MockTaskFinisher : public TaskFinisherInterface { num_tasks_complete++; } - bool RetryTaskIfPossible(const TaskID &task_id, bool task_failed_due_to_oom) override { + bool RetryTaskIfPossible(const TaskID &task_id, + const rpc::RayErrorInfo &error_info) override { num_task_retries_attempted++; return false; } diff --git a/src/ray/core_worker/test/direct_task_transport_test.cc b/src/ray/core_worker/test/direct_task_transport_test.cc index 401481fd58397..61eb4370c3f45 100644 --- a/src/ray/core_worker/test/direct_task_transport_test.cc +++ b/src/ray/core_worker/test/direct_task_transport_test.cc @@ -125,7 +125,8 @@ class MockTaskFinisher : public TaskFinisherInterface { num_tasks_complete++; } - bool RetryTaskIfPossible(const TaskID &task_id, bool task_failed_due_to_oom) override { + bool RetryTaskIfPossible(const TaskID &task_id, + const rpc::RayErrorInfo &error_info) override { num_task_retries_attempted++; return false; } diff --git a/src/ray/core_worker/transport/direct_actor_task_submitter.cc b/src/ray/core_worker/transport/direct_actor_task_submitter.cc index b380365ee6179..cab04a6cebe51 100644 --- a/src/ray/core_worker/transport/direct_actor_task_submitter.cc +++ b/src/ray/core_worker/transport/direct_actor_task_submitter.cc @@ -140,8 +140,8 @@ Status CoreWorkerDirectActorTaskSubmitter::SubmitTask(TaskSpecification task_spe absl::MutexLock lock(&mu_); const auto queue_it = client_queues_.find(task_spec.ActorId()); const auto &death_cause = queue_it->second.death_cause; - error_type = GenErrorTypeFromDeathCause(death_cause); error_info = GetErrorInfoFromActorDeathCause(death_cause); + error_type = error_info.error_type(); } auto status = Status::IOError("cancelling task of dead actor"); // No need to increment the number of completed tasks since the actor is @@ -304,8 +304,8 @@ void CoreWorkerDirectActorTaskSubmitter::DisconnectActor( // Failing tasks has to be done without mu_ hold because the callback // might require holding mu_ which will lead to a deadlock. auto status = Status::IOError("cancelling all pending tasks of dead actor"); - rpc::ErrorType error_type = GenErrorTypeFromDeathCause(death_cause); const auto error_info = GetErrorInfoFromActorDeathCause(death_cause); + const auto error_type = error_info.error_type(); for (auto &task_id : task_ids_to_fail) { // No need to increment the number of completed tasks since the actor is @@ -528,7 +528,7 @@ void CoreWorkerDirectActorTaskSubmitter::HandlePushTaskReply( is_actor_dead = queue.state == rpc::ActorTableData::DEAD; const auto &death_cause = queue.death_cause; error_info = GetErrorInfoFromActorDeathCause(death_cause); - error_type = GenErrorTypeFromDeathCause(death_cause); + error_type = error_info.error_type(); fail_immediatedly = error_info.has_actor_died_error() && error_info.actor_died_error().has_oom_context() && error_info.actor_died_error().oom_context().fail_immediately(); diff --git a/src/ray/core_worker/transport/direct_task_transport.cc b/src/ray/core_worker/transport/direct_task_transport.cc index 7de9c58aa7f01..39538d5ddf458 100644 --- a/src/ray/core_worker/transport/direct_task_transport.cc +++ b/src/ray/core_worker/transport/direct_task_transport.cc @@ -524,7 +524,7 @@ void CoreWorkerDirectTaskSubmitter::RequestNewWorkerIfNeeded( } } } - + error_info.set_error_type(error_type); while (!tasks_to_fail.empty()) { auto &task_spec = tasks_to_fail.front(); if (task_spec.IsActorCreationTask() && @@ -641,7 +641,8 @@ void CoreWorkerDirectTaskSubmitter::PushNormalTask( !reply.is_retryable_error() || !task_finisher_->RetryTaskIfPossible( task_id, - /*task_failed_due_to_oom*/ false)) { + gcs::GetRayErrorInfo( + rpc::ErrorType::TASK_EXECUTION_EXCEPTION))) { task_finisher_->CompletePendingTask( task_id, reply, addr.ToProto(), reply.is_application_error()); } diff --git a/src/ray/gcs/pb_util.h b/src/ray/gcs/pb_util.h index 3e74ea8042d27..cef44e37979c5 100644 --- a/src/ray/gcs/pb_util.h +++ b/src/ray/gcs/pb_util.h @@ -138,22 +138,6 @@ inline const rpc::RayException *GetCreationTaskExceptionFromDeathCause( return &(death_cause->creation_task_failure_context()); } -/// Generate object error type from ActorDeathCause. -inline rpc::ErrorType GenErrorTypeFromDeathCause( - const rpc::ActorDeathCause &death_cause) { - if (death_cause.context_case() == ContextCase::kCreationTaskFailureContext) { - return rpc::ErrorType::ACTOR_DIED; - } else if (death_cause.context_case() == ContextCase::kRuntimeEnvFailedContext) { - return rpc::ErrorType::RUNTIME_ENV_SETUP_FAILED; - } else if (death_cause.context_case() == ContextCase::kActorUnschedulableContext) { - return rpc::ErrorType::ACTOR_UNSCHEDULABLE_ERROR; - } else if (death_cause.context_case() == ContextCase::kOomContext) { - return rpc::ErrorType::OUT_OF_MEMORY; - } else { - return rpc::ErrorType::ACTOR_DIED; - } -} - inline const std::string &GetActorDeathCauseString( const rpc::ActorDeathCause &death_cause) { static absl::flat_hash_map death_cause_string{ @@ -179,17 +163,22 @@ inline rpc::RayErrorInfo GetErrorInfoFromActorDeathCause( if (death_cause.context_case() == ContextCase::kActorDiedErrorContext || death_cause.context_case() == ContextCase::kCreationTaskFailureContext) { error_info.mutable_actor_died_error()->CopyFrom(death_cause); + error_info.set_error_type(rpc::ErrorType::ACTOR_DIED); } else if (death_cause.context_case() == ContextCase::kRuntimeEnvFailedContext) { error_info.mutable_runtime_env_setup_failed_error()->CopyFrom( death_cause.runtime_env_failed_context()); + error_info.set_error_type(rpc::ErrorType::RUNTIME_ENV_SETUP_FAILED); } else if (death_cause.context_case() == ContextCase::kActorUnschedulableContext) { *(error_info.mutable_error_message()) = death_cause.actor_unschedulable_context().error_message(); + error_info.set_error_type(rpc::ErrorType::ACTOR_UNSCHEDULABLE_ERROR); } else if (death_cause.context_case() == ContextCase::kOomContext) { error_info.mutable_actor_died_error()->CopyFrom(death_cause); *(error_info.mutable_error_message()) = death_cause.oom_context().error_message(); + error_info.set_error_type(rpc::ErrorType::OUT_OF_MEMORY); } else { RAY_CHECK(death_cause.context_case() == ContextCase::CONTEXT_NOT_SET); + error_info.set_error_type(rpc::ErrorType::ACTOR_DIED); } return error_info; } @@ -270,6 +259,15 @@ inline void FillTaskInfo(rpc::TaskInfoEntry *task_info, } } +/// Generate a RayErrorInfo from ErrorType +inline rpc::RayErrorInfo GetRayErrorInfo(const rpc::ErrorType &error_type, + const std::string &error_msg = "") { + rpc::RayErrorInfo error_info; + error_info.set_error_type(error_type); + error_info.set_error_message(error_msg); + return error_info; +} + /// Get the timestamp of the task status if available. /// /// \param task_event Task event. diff --git a/src/ray/protobuf/common.proto b/src/ray/protobuf/common.proto index 6660b9277ddeb..55d6718323554 100644 --- a/src/ray/protobuf/common.proto +++ b/src/ray/protobuf/common.proto @@ -214,7 +214,7 @@ message RayErrorInfo { string error_message = 4; } // The type of error that caused the exception. - optional ErrorType error_type = 11; + ErrorType error_type = 11; } message OutOfMemoryErrorContext { diff --git a/src/ray/protobuf/gcs.proto b/src/ray/protobuf/gcs.proto index bd470ddfb82de..3e4c4cce84f68 100644 --- a/src/ray/protobuf/gcs.proto +++ b/src/ray/protobuf/gcs.proto @@ -215,6 +215,8 @@ message TaskStateUpdate { optional int64 failed_ts = 7; // Worker that runs the task. optional bytes worker_id = 8; + // Task faulure info. + optional ErrorType error_type = 9; } // Represents events and state changes from a single task run. diff --git a/src/ray/protobuf/usage.proto b/src/ray/protobuf/usage.proto index e30501aeb9971..e60553ea706dd 100644 --- a/src/ray/protobuf/usage.proto +++ b/src/ray/protobuf/usage.proto @@ -94,4 +94,9 @@ enum TagKey { NUM_ACTOR_TASKS = 306; NUM_NORMAL_TASKS = 307; NUM_DRIVERS = 308; + + // Data + // Logical operators, stored in JSON format with operator name and count. + // Example: {"MapBatches": 2, "Filter": 1} + DATA_LOGICAL_OPS = 400; } diff --git a/src/ray/raylet/worker_pool.cc b/src/ray/raylet/worker_pool.cc index aeadc4180cadb..093abb41841c6 100644 --- a/src/ray/raylet/worker_pool.cc +++ b/src/ray/raylet/worker_pool.cc @@ -1233,8 +1233,7 @@ void WorkerPool::PopWorker(const TaskSpecification &task_spec, const int runtime_env_hash = task_spec.GetRuntimeEnvHash(); for (auto it = idle_of_all_languages_.rbegin(); it != idle_of_all_languages_.rend(); it++) { - if (task_spec.GetLanguage() != it->first->GetLanguage() || - state.pending_disconnection_workers.count(it->first) > 0 || it->first->IsDead()) { + if (task_spec.GetLanguage() != it->first->GetLanguage() || it->first->IsDead()) { continue; } @@ -1421,8 +1420,6 @@ void WorkerPool::DisconnectWorker(const std::shared_ptr &worker return; } - RAY_UNUSED(RemoveWorker(state.pending_disconnection_workers, worker)); - for (auto it = idle_of_all_languages_.begin(); it != idle_of_all_languages_.end(); it++) { if (it->first == worker) { @@ -1432,23 +1429,6 @@ void WorkerPool::DisconnectWorker(const std::shared_ptr &worker } } RemoveWorker(state.idle, worker); - if (disconnect_type != rpc::WorkerExitType::INTENDED_USER_EXIT) { - // A Java worker process may have multiple workers. If one of them disconnects - // unintentionally (which means that the worker process has died), we remove the - // others from idle pool so that the failed actor will not be rescheduled on the same - // process. - auto pid = worker->GetProcess().GetId(); - for (auto worker2 : state.registered_workers) { - if (worker2->GetProcess().GetId() == pid) { - // NOTE(kfstorm): We have to use a new field to record these workers (instead of - // just removing them from idle sets) because they may haven't announced worker - // port yet. When they announce worker port, they'll be marked idle again. So - // removing them from idle sets here doesn't really prevent them from being popped - // later. - state.pending_disconnection_workers.insert(worker2); - } - } - } } void WorkerPool::DisconnectDriver(const std::shared_ptr &driver) { diff --git a/src/ray/raylet/worker_pool.h b/src/ray/raylet/worker_pool.h index d21ec5f758c41..6118a9f7d95a0 100644 --- a/src/ray/raylet/worker_pool.h +++ b/src/ray/raylet/worker_pool.h @@ -513,9 +513,6 @@ class WorkerPool : public WorkerPoolInterface, public IOWorkerPoolInterface { std::unordered_set> registered_workers; /// All drivers that have registered and are still connected. std::unordered_set> registered_drivers; - /// All workers that have registered but is about to disconnect. They shouldn't be - /// popped anymore. - std::unordered_set> pending_disconnection_workers; /// A map from the startup tokens of worker processes, assigned by the raylet, to /// the extra information of the process. Note that the shim process PID is the /// same with worker process PID, except worker process in container.