More consistent recipes

metadata-ingestion/source_docs/bigquery.md

@@ -86,13 +86,15 @@ source:
     options:
       # See https://googleapis.dev/python/logging/latest/client.html for details.
       credentials: ~ # optional - see docs
-    env: PROD
 
+    # Common usage stats options
     bucket_duration: "DAY"
     start_time: ~ # defaults to the last full day in UTC (or hour)
     end_time: ~ # defaults to the last full day in UTC (or hour)
 
     top_n_queries: 10 # number of queries to save for each table
+
+    env: PROD
 ```
 
 :::note

metadata-ingestion/source_docs/dbt.md

@@ -23,11 +23,21 @@ This plugin pulls metadata from dbt's artifact files:
 source:
   type: "dbt"
   config:
+    # https://docs.getdbt.com/reference/artifacts/manifest-json
     manifest_path: "./path/dbt/manifest_file.json"
+    # https://docs.getdbt.com/reference/artifacts/catalog-json
     catalog_path: "./path/dbt/catalog_file.json"
+    # https://docs.getdbt.com/reference/artifacts/sources-json
     sources_path: "./path/dbt/sources_file.json" # (optional, used for freshness checks)
+
+    # the platform that dbt is loading onto
     target_platform: "postgres" # optional, eg "postgres", "snowflake", etc.
+
+    # whether to load schemas of datasets from dbt
+    # (otherwise, only includes a simple list of tables)
     load_schemas: True or False
+
+    # regex pattern to allow/deny nodes
     node_type_pattern: # optional
       deny:
         - ^test.*

metadata-ingestion/source_docs/glue.md

@@ -27,7 +27,20 @@ source:
 
     extract_transforms: True # whether to ingest Glue jobs, defaults to True
 
-    # Filtering patterns for databases and tables to scan
-    database_pattern: # Optional, to filter databases scanned, same as schema_pattern above.
+    # Regex filters for databases to scan
+    database_pattern:
+      deny:
+        # Note that the deny patterns take precedence over the allow patterns.
+        - "bad_database"
+        - "junk_database"
+        # Can also be a regular expression
+        - "(old|used|deprecated)_database"
+      allow:
+        - "good_database"
+        - "excellent_database"
     table_pattern: # Optional, to filter tables scanned, same as table_pattern above.
+      deny:
+        # ...
+      allow:
+        # ...
 ```

metadata-ingestion/source_docs/hive.md

@@ -60,7 +60,6 @@ source:
       allow:
         # ...
 
-    include_views: True # whether to include views, defaults to True
     include_tables: True # whether to include views, defaults to True
 ```
 

metadata-ingestion/source_docs/kafka.md

@@ -13,9 +13,24 @@ source:
   config:
     connection:
       bootstrap: "broker:9092"
-      consumer_config: {} # passed to https://docs.confluent.io/platform/current/clients/confluent-kafka-python/html/index.html#confluent_kafka.DeserializingConsumer
       schema_registry_url: https://localhost:8081
-      schema_registry_config: {} # passed to https://docs.confluent.io/platform/current/clients/confluent-kafka-python/html/index.html#confluent_kafka.schema_registry.SchemaRegistryClient
+
+      # Extra schema registry config.
+      # These options will be passed into Kafka's SchemaRegistryClient.
+      # See https://docs.confluent.io/platform/current/clients/confluent-kafka-python/html/index.html?#schemaregistryclient
+      schema_registry_config: {}
+
+      # Extra consumer config.
+      # These options will be passed into Kafka's DeserializingConsumer.
+      # See https://docs.confluent.io/platform/current/clients/confluent-kafka-python/html/index.html#deserializingconsumer
+      # and https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md.
+      consumer_config: {}
+
+      # Extra producer config.
+      # These options will be passed into Kafka's SerializingProducer.
+      # See https://docs.confluent.io/platform/current/clients/confluent-kafka-python/html/index.html#serializingproducer
+      # and https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md.
+      producer_config: {}
 ```
 
 The options in the consumer config and schema registry config are passed to the Kafka DeserializingConsumer and SchemaRegistryClient respectively.