Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix Snowflake FILE_FORMAT options #221

Closed
dmaresma opened this issue Jan 5, 2024 · 1 comment
Closed

fix Snowflake FILE_FORMAT options #221

dmaresma opened this issue Jan 5, 2024 · 1 comment

Comments

@dmaresma
Copy link
Contributor

dmaresma commented Jan 5, 2024

Support the Stage and File format options of snowflake for external table ddl statement following the snowflake requirements
https://docs.snowflake.com/en/sql-reference/sql/create-file-format


def test_virtual_column_table():
    ddl = """
    create or replace table if not exists TABLE_DATA_SRC.EXT_PAYLOAD_MANIFEST_WEB (
       id bigint,
       derived bigint as (id * 10)
       )
    location = @ADL_Azure_Storage_Account_Container_Name/
    auto_refresh = false
    file_format = (TYPE=JSON NULL_IF=('field') DATE_FORMAT=AUTO TRIM_SPACE=TRUE)
    stage_file_format = (TYPE=JSON NULL_IF=())
    ;
    """
    result_ext_table = DDLParser(ddl, normalize_names=True, debug=True).run(
        output_mode="snowflake"
    )

    expected_ext_table = [
        {
            "alter": {},
            "checks": [],
            "clone": None,
            "columns": [
                {
                    "name": "id",
                    "type": "bigint",
                    "size": None,
                    "references": None,
                    "unique": False,
                    "nullable": True,
                    "default": None,
                    "check": None,
                },
                {
                    "name": "derived",
                    "type": "bigint",
                    "size": None,
                    "references": None,
                    "unique": False,
                    "nullable": True,
                    "default": None,
                    "check": None,
                    "generated" : {"as" : "id * 10" }
                }
            ],
            "index": [],
            "partitioned_by" : [],
            "primary_key": [],
            "primary_key_enforced": None,
            "auto_refresh" : False,
            "schema": "TABLE_DATA_SRC",
            "table_name": "EXT_PAYLOAD_MANIFEST_WEB",
            "tablespace": None,
            "replace" : True,
            "if_not_exists": True,
            "location" : "@ADL_Azure_Storage_Account_Container_Name/",
            "file_format": ['TYPE=JSON', "NULL_IF=('field')",'DATE_FORMAT=AUTO','TRIM_SPACE=TRUE'],
            'stage_file_format': ['TYPE=JSON','NULL_IF=()']
            }
    ]

    assert result_ext_table == expected_ext_table
@xnuinside
Copy link
Owner

merged & released @ 0.31.3

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
None yet
Projects
None yet
Development

No branches or pull requests

2 participants