remove(ingest/snowflake): Remove legacy snowflake lineage (datahub-pr…

…oject#8653) Co-authored-by: Tamas Nemeth <[email protected]> Co-authored-by: Aseem Bansal <[email protected]>
jinlintt · Aug 23, 2023 · 8141e2d · 8141e2d
1 parent 68abf9c
commit 8141e2d
Show file tree

Hide file tree

Showing 9 changed files with 6 additions and 1,226 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py
@@ -91,13 +91,8 @@ class SnowflakeV2Config(
  description="Whether `schema_pattern` is matched against fully qualified schema name `<catalog>.<schema>`.",
  )
 
- use_legacy_lineage_method: bool = Field(
- default=False,
- description=(
- "Whether to use the legacy lineage computation method. "
- "By default, uses new optimised lineage extraction method that requires less ingestion process memory. "
- "Table-to-view and view-to-view column-level lineage are not supported with the legacy method."
- ),
+ _use_legacy_lineage_method_removed = pydantic_removed_field(
+ "use_legacy_lineage_method"
  )
 
  validate_upstreams_against_patterns: bool = Field(
@@ -113,7 +108,7 @@ class SnowflakeV2Config(
  # This is required since access_history table does not capture whether the table was temporary table.
  temporary_tables_pattern: List[str] = Field(
  default=DEFAULT_TABLES_DENY_LIST,
- description="[Advanced] Regex patterns for temporary tables to filter in lineage ingestion. Specify regex to match the entire table name in database.schema.table format. Defaults are to set in such a way to ignore the temporary staging tables created by known ETL tools. Not used if `use_legacy_lineage_method=True`",
+ description="[Advanced] Regex patterns for temporary tables to filter in lineage ingestion. Specify regex to match the entire table name in database.schema.table format. Defaults are to set in such a way to ignore the temporary staging tables created by known ETL tools.",
  )
 
  rename_upstreams_deny_pattern_to_temporary_table_pattern = pydantic_renamed_field(