build(ingest/boto3): Update boto3-stubs to fix CI (datahub-project#8425)

metadata-ingestion/setup.py

@@ -413,7 +413,7 @@ def get_long_description():
  "types-cachetools",
  # versions 0.1.13 and 0.1.14 seem to have issues
  "types-click==0.1.12",
- "boto3-stubs[s3,glue,sagemaker,sts]",
+ "boto3-stubs[s3,glue,sagemaker,sts]>=1.28.3",
  "types-tabulate",
  # avrogen package requires this
  "types-pytz",

metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/feature_groups.py

@@ -23,9 +23,9 @@
 if TYPE_CHECKING:
  from mypy_boto3_sagemaker import SageMakerClient
  from mypy_boto3_sagemaker.type_defs import (
- DescribeFeatureGroupResponseTypeDef,
- FeatureDefinitionTypeDef,
- FeatureGroupSummaryTypeDef,
+ DescribeFeatureGroupResponseOutputTypeDef,
+ FeatureDefinitionOutputTypeDef,
+ FeatureGroupSummaryOutputTypeDef,
  )
 
 
@@ -35,7 +35,7 @@ class FeatureGroupProcessor:
  env: str
  report: SagemakerSourceReport
 
- def get_all_feature_groups(self) -> List["FeatureGroupSummaryTypeDef"]:
+ def get_all_feature_groups(self) -> List["FeatureGroupSummaryOutputTypeDef"]:
  """
  List all feature groups in SageMaker.
  """
@@ -51,7 +51,7 @@ def get_all_feature_groups(self) -> List["FeatureGroupSummaryTypeDef"]:
 
  def get_feature_group_details(
  self, feature_group_name: str
- ) -> "DescribeFeatureGroupResponseTypeDef":
+ ) -> "DescribeFeatureGroupResponseOutputTypeDef":
  """
  Get details of a feature group (including list of component features).
  """
@@ -75,7 +75,7 @@ def get_feature_group_details(
  return feature_group
 
  def get_feature_group_wu(
- self, feature_group_details: "DescribeFeatureGroupResponseTypeDef"
+ self, feature_group_details: "DescribeFeatureGroupResponseOutputTypeDef"
  ) -> MetadataWorkUnit:
  """
  Generate an MLFeatureTable workunit for a SageMaker feature group.
@@ -146,8 +146,8 @@ def get_feature_type(self, aws_type: str, feature_name: str) -> str:
 
  def get_feature_wu(
  self,
- feature_group_details: "DescribeFeatureGroupResponseTypeDef",
- feature: "FeatureDefinitionTypeDef",
+ feature_group_details: "DescribeFeatureGroupResponseOutputTypeDef",
+ feature: "FeatureDefinitionOutputTypeDef",
  ) -> MetadataWorkUnit:
  """
  Generate an MLFeature workunit for a SageMaker feature.

metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/lineage.py

@@ -9,10 +9,10 @@
 if TYPE_CHECKING:
  from mypy_boto3_sagemaker import SageMakerClient
  from mypy_boto3_sagemaker.type_defs import (
- ActionSummaryTypeDef,
- ArtifactSummaryTypeDef,
- AssociationSummaryTypeDef,
- ContextSummaryTypeDef,
+ ActionSummaryOutputTypeDef,
+ ArtifactSummaryOutputTypeDef,
+ AssociationSummaryOutputTypeDef,
+ ContextSummaryOutputTypeDef,
  )
 
 
@@ -49,7 +49,7 @@ class LineageProcessor:
  nodes: Dict[str, Dict[str, Any]] = field(default_factory=dict)
  lineage_info: LineageInfo = field(default_factory=LineageInfo)
 
- def get_all_actions(self) -> List["ActionSummaryTypeDef"]:
+ def get_all_actions(self) -> List["ActionSummaryOutputTypeDef"]:
  """
  List all actions in SageMaker.
  """
@@ -63,7 +63,7 @@ def get_all_actions(self) -> List["ActionSummaryTypeDef"]:
 
  return actions
 
- def get_all_artifacts(self) -> List["ArtifactSummaryTypeDef"]:
+ def get_all_artifacts(self) -> List["ArtifactSummaryOutputTypeDef"]:
  """
  List all artifacts in SageMaker.
  """
@@ -77,7 +77,7 @@ def get_all_artifacts(self) -> List["ArtifactSummaryTypeDef"]:
 
  return artifacts
 
- def get_all_contexts(self) -> List["ContextSummaryTypeDef"]:
+ def get_all_contexts(self) -> List["ContextSummaryOutputTypeDef"]:
  """
  List all contexts in SageMaker.
  """
@@ -91,7 +91,9 @@ def get_all_contexts(self) -> List["ContextSummaryTypeDef"]:
 
  return contexts
 
- def get_incoming_edges(self, node_arn: str) -> List["AssociationSummaryTypeDef"]:
+ def get_incoming_edges(
+ self, node_arn: str
+ ) -> List["AssociationSummaryOutputTypeDef"]:
  """
  Get all incoming edges for a node in the lineage graph.
  """
@@ -105,7 +107,9 @@ def get_incoming_edges(self, node_arn: str) -> List["AssociationSummaryTypeDef"]
 
  return edges
 
- def get_outgoing_edges(self, node_arn: str) -> List["AssociationSummaryTypeDef"]:
+ def get_outgoing_edges(
+ self, node_arn: str
+ ) -> List["AssociationSummaryOutputTypeDef"]:
  """
  Get all outgoing edges for a node in the lineage graph.
  """

metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/models.py

@@ -45,12 +45,12 @@
 if TYPE_CHECKING:
  from mypy_boto3_sagemaker import SageMakerClient
  from mypy_boto3_sagemaker.type_defs import (
- DescribeEndpointOutputTypeDef,
- DescribeModelOutputTypeDef,
- DescribeModelPackageGroupOutputTypeDef,
- EndpointSummaryTypeDef,
- ModelPackageGroupSummaryTypeDef,
- ModelSummaryTypeDef,
+ DescribeEndpointOutputOutputTypeDef,
+ DescribeModelOutputOutputTypeDef,
+ DescribeModelPackageGroupOutputOutputTypeDef,
+ EndpointSummaryOutputTypeDef,
+ ModelPackageGroupSummaryOutputTypeDef,
+ ModelSummaryOutputTypeDef,
  )
 
 ENDPOINT_STATUS_MAP: Dict[str, str] = {
@@ -91,7 +91,7 @@ class ModelProcessor:
 
  group_arn_to_name: Dict[str, str] = field(default_factory=dict)
 
- def get_all_models(self) -> List["ModelSummaryTypeDef"]:
+ def get_all_models(self) -> List["ModelSummaryOutputTypeDef"]:
  """
  List all models in SageMaker.
  """
@@ -105,15 +105,15 @@ def get_all_models(self) -> List["ModelSummaryTypeDef"]:
 
  return models
 
- def get_model_details(self, model_name: str) -> "DescribeModelOutputTypeDef":
+ def get_model_details(self, model_name: str) -> "DescribeModelOutputOutputTypeDef":
  """
  Get details of a model.
  """
 
  # see https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sagemaker.html#SageMaker.Client.describe_model
  return self.sagemaker_client.describe_model(ModelName=model_name)
 
- def get_all_groups(self) -> List["ModelPackageGroupSummaryTypeDef"]:
+ def get_all_groups(self) -> List["ModelPackageGroupSummaryOutputTypeDef"]:
  """
  List all model groups in SageMaker.
  """
@@ -128,7 +128,7 @@ def get_all_groups(self) -> List["ModelPackageGroupSummaryTypeDef"]:
 
  def get_group_details(
  self, group_name: str
- ) -> "DescribeModelPackageGroupOutputTypeDef":
+ ) -> "DescribeModelPackageGroupOutputOutputTypeDef":
  """
  Get details of a model group.
  """
@@ -138,7 +138,7 @@ def get_group_details(
  ModelPackageGroupName=group_name
  )
 
- def get_all_endpoints(self) -> List["EndpointSummaryTypeDef"]:
+ def get_all_endpoints(self) -> List["EndpointSummaryOutputTypeDef"]:
  endpoints = []
 
  # see https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sagemaker.html#SageMaker.Client.list_endpoints
@@ -151,7 +151,7 @@ def get_all_endpoints(self) -> List["EndpointSummaryTypeDef"]:
 
  def get_endpoint_details(
  self, endpoint_name: str
- ) -> "DescribeEndpointOutputTypeDef":
+ ) -> "DescribeEndpointOutputOutputTypeDef":
  # see https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sagemaker.html#SageMaker.Client.describe_endpoint
  return self.sagemaker_client.describe_endpoint(EndpointName=endpoint_name)
 
@@ -171,7 +171,7 @@ def get_endpoint_status(
  return endpoint_status
 
  def get_endpoint_wu(
- self, endpoint_details: "DescribeEndpointOutputTypeDef"
+ self, endpoint_details: "DescribeEndpointOutputOutputTypeDef"
  ) -> MetadataWorkUnit:
  """a
  Get a workunit for an endpoint.
@@ -215,7 +215,7 @@ def get_endpoint_wu(
 
  def get_model_endpoints(
  self,
- model_details: "DescribeModelOutputTypeDef",
+ model_details: "DescribeModelOutputOutputTypeDef",
  endpoint_arn_to_name: Dict[str, str],
  model_image: Optional[str],
  model_uri: Optional[str],
@@ -244,7 +244,7 @@ def get_model_endpoints(
  return model_endpoints_sorted
 
  def get_group_wu(
- self, group_details: "DescribeModelPackageGroupOutputTypeDef"
+ self, group_details: "DescribeModelPackageGroupOutputOutputTypeDef"
  ) -> MetadataWorkUnit:
  """
  Get a workunit for a model group.
@@ -294,7 +294,7 @@ def get_group_wu(
  return MetadataWorkUnit(id=group_name, mce=mce)
 
  def match_model_jobs(
- self, model_details: "DescribeModelOutputTypeDef"
+ self, model_details: "DescribeModelOutputOutputTypeDef"
  ) -> Tuple[Set[str], Set[str], List[MLHyperParamClass], List[MLMetricClass]]:
  model_training_jobs: Set[str] = set()
  model_downstream_jobs: Set[str] = set()
@@ -387,7 +387,7 @@ def strip_quotes(string: str) -> str:
 
  def get_model_wu(
  self,
- model_details: "DescribeModelOutputTypeDef",
+ model_details: "DescribeModelOutputOutputTypeDef",
  endpoint_arn_to_name: Dict[str, str],
  ) -> MetadataWorkUnit:
  """