lfw envs OK

ColumbiaDVMM · Jan 24, 2020 · d39f6e7 · d39f6e7
1 parent d5f6570
commit d39f6e7
Show file tree

Hide file tree

Showing 9 changed files with 40 additions and 61 deletions.
diff --git a/cufacesearch/cufacesearch/pusher/kinesis_pusher.py b/cufacesearch/cufacesearch/pusher/kinesis_pusher.py
@@ -3,6 +3,8 @@
 import time
 import json
 import boto3
+import urllib3
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 from datetime import datetime
 #from cufacesearch.common.conf_reader import ConfReader
 from ..common.conf_reader import ConfReader

diff --git a/cufacesearch/cufacesearch/pusher/local_images_pusher.py b/cufacesearch/cufacesearch/pusher/local_images_pusher.py
@@ -1,6 +1,8 @@
 from __future__ import print_function
 import json
 import time
+import urllib3
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 from argparse import ArgumentParser
 from cufacesearch.common.conf_reader import ConfReader
 # TODO: separate consumer/producer

diff --git a/cufacesearch/cufacesearch/requirements.txt b/cufacesearch/cufacesearch/requirements.txt
@@ -7,7 +7,7 @@ thriftpy
 flask
 flask_restful
 matplotlib
-requests
+requests>=2.16
 gevent
 kafka-python
 elasticsearch

diff --git a/setup/all-in-one/.env.catlech101local.kafka b/setup/all-in-one/.env.catlech101local.kafka
@@ -26,8 +26,7 @@ kafka_security=
 image_ingestion_type=kafka
 images_topic=test-local-caltech101
 updates_topic=test-local-caltech101-sbpycaffeimg-updates
-extr_check_consumer_group=test-local-caltech101-sbpycaffeimg-extrchecker
-extr_proc_consumer_group=test-local-caltech101-sbpycaffeimg-extrproc
+images_consumer_group=test-local-caltech101-sbpycaffeimg-extrchecker
 
 # Updates ingestion should be from HBase
 update_ingestion_type=hbase

diff --git a/setup/all-in-one/.env.catlech101local.kinesis b/setup/all-in-one/.env.catlech101local.kinesis
@@ -32,9 +32,6 @@ aws_profile=kinesis
 # Updates ingestion should be from HBase
 update_ingestion_type=hbase
 
-#update_ingestion_type=kinesis
-#update_stream=updates-test-local-kinesis-caltech101
-
 # HBase settings
 # (local)
 hbase_host=hbase

diff --git a/setup/all-in-one/.env.lfwlocal.kafka b/setup/all-in-one/.env.lfwlocal.kafka
@@ -31,8 +31,7 @@ images_topic=lfw-local-kafka
 
 # This is extraction specific
 updates_topic=lfw-local-dlibface-updates
-extr_check_consumer_group=lfw-local-dlibface-extrchecker
-extr_proc_consumer_group=lfw-local-dlibface-extrproc
+images_consumer_group=lfw-local-dlibface-extrchecker
 
 # Updates ingestion should be from HBase
 update_ingestion_type=hbase
@@ -43,6 +42,13 @@ hbase_host=hbase
 table_sha1infos=lfw_local_kafka_dlib_infos
 table_updateinfos=lfw_local_kafka_dlib_update_infos
 batch_update_size=1000
+# OK to use default values for these
+#column_list_sha1s=sha1s
+#extr_column_family=data
+#image_info_column_family=data
+#image_buffer_column_family=img
+#image_buffer_column_name=img
+#update_info_column_family=up
 
 # Searcher settings
 search_conf_name=lfw_local_kafka_dlib_lopqpca

diff --git a/setup/all-in-one/.env.lfwlocal.kinesis b/setup/all-in-one/.env.lfwlocal.kinesis
@@ -12,7 +12,6 @@ verbose=3
 # Example source for LFW (to be used with DLib face detection/featurizer)
 input_conf_name=lfw_local_kinesis
 input_type=local
-update_ingestion_type=hbase
 source_zip=http:https://vis-www.cs.umass.edu/lfw/lfw.tgz
 # Should we mount a volume to make sure path are consistent ?
 input_path=./data/input_images_lfw
@@ -31,27 +30,25 @@ region_name=us-east-1
 endpoint_url=https://kinesis:4567
 verify_certificates=0
 aws_profile=kinesis
-
 # This can be shared across extractions
 images_topic=lfw-local-kinesis
 
-# This is extraction specific
-#updates_topic=test-local-lfw-dlibface-updates # To test if fall back to HBase would work...
-extr_check_consumer_group=lfw-local-dlibface-extrchecker
-extr_proc_consumer_group=lfw-local-dlibface-extrproc
+# Updates ingestion should be from HBase
+update_ingestion_type=hbase
 
 # HBase settings
 # (local)
 hbase_host=hbase
 table_sha1infos=lfw_local_kinesis_dlib_infos
 table_updateinfos=lfw_local_kinesis_dlib_update_infos
 batch_update_size=1000
-column_list_sha1s=sha1s
-extr_column_family=data
-image_info_column_family=data
-image_buffer_column_family=img
-image_buffer_column_name=img
-update_info_column_family=up
+# OK to use default values for these
+#column_list_sha1s=sha1s
+#extr_column_family=data
+#image_info_column_family=data
+#image_buffer_column_family=img
+#image_buffer_column_name=img
+#update_info_column_family=up
 
 # Searcher settings
 search_conf_name=lfw_local_kinesis_dlib_lopqpca

diff --git a/setup/all-in-one/docker-compose_kinesis.yml b/setup/all-in-one/docker-compose_kinesis.yml
@@ -78,18 +78,10 @@ services:
  - input_type
  - "image_pushing_type=${image_ingestion_type}"
  - input_path
- - input_nb_threads
  - verbose
  - source_zip
- - input_nb_workers
- #- "nb_workers=${input_nb_workers}"
  - "conf_name=${input_conf_name}"
-# # Kafka related environment variables
-# - input_topic
-# - input_consumer_group
-# - images_topic
-# - kafka_servers
-# - kafka_security
+ # Kinesis related environment variables
  - images_stream
  - region_name
  - create_stream
@@ -98,7 +90,7 @@ services:
  - endpoint_url
  - producer_type
  - producer_prefix
- #- "PYTHONWARNINGS='ignore:Unverified HTTPS request'" # Does not seem to work
+ - "PYTHONWARNINGS='ignore:Unverified HTTPS request'" # Does not seem to work
  cap_add:
  - IPC_LOCK
  # nothing really needs to be persistent here. no other volumes needed
@@ -130,24 +122,18 @@ services:
  - image_ingestion_type
  - update_ingestion_type
  - producer_type
+ - verbose
+ - "conf_name=${extr_conf_name}"
+ - extr_type
+ - extr_nb_threads
+ - extr_check_max_delay
+ # Kinesis related environment variables
  - region_name
  - images_stream
  - "image_shard_infos_filename=${indocker_repo_path}/kinesis_${images_stream}_${extr_conf_name}.json"
  - aws_profile
  - verify_certificates
  - endpoint_url
- - verbose
- - "conf_name=${extr_conf_name}"
- - extr_type
- - extr_nb_threads
- - extr_check_max_delay
-# # Kafka related environment variables
-# - images_topic
-# - kafka_servers
-# - kafka_security
-# - extr_check_consumer_group
-# - extr_proc_consumer_group
-# - updates_topic
  # Hbase related environment variables
  - hbase_host
  - table_sha1infos
@@ -159,7 +145,7 @@ services:
  - image_buffer_column_family
  - image_buffer_column_name
  - update_info_column_family
- - "PYTHONWARNINGS='ignore:Unverified HTTPS request'"
+ - "PYTHONWARNINGS='ignore:Unverified HTTPS request'" # Does not seem to work
  # nothing really needs to be persistent here. no other volumes needed
  volumes:
  - $repo_path:$indocker_repo_path

diff --git a/setup/all-in-one/docker-compose_kinesis_monitor.yml b/setup/all-in-one/docker-compose_kinesis_monitor.yml
@@ -107,14 +107,7 @@ services:
  - input_path
  - verbose
  - source_zip
- - "nb_workers=${input_nb_workers}"
  - "conf_name=${input_conf_name}"
-# # Kafka related environment variables
-# - input_topic
-# - input_consumer_group
-# - images_topic
-# - kafka_servers
-# - kafka_security
  # Kinesis related environment variables
  - images_stream
  - region_name
@@ -124,6 +117,7 @@ services:
  - endpoint_url
  - producer_type
  - producer_prefix
+ - "PYTHONWARNINGS='ignore:Unverified HTTPS request'" # Does not seem to work
  cap_add:
  - IPC_LOCK
  # nothing really needs to be persistent here. no other volumes needed
@@ -152,26 +146,21 @@ services:
  environment:
  # General environment variables
  - input_type
+ - image_ingestion_type
+ - update_ingestion_type
+ - producer_type
  - verbose
  - "conf_name=${extr_conf_name}"
  - extr_type
  - extr_nb_threads
  - extr_check_max_delay
- - ingestion_input
- # Kinesis related variables
- - producer_type
+ # Kinesis related environment variables
  - region_name
  - images_stream
+ - "image_shard_infos_filename=${indocker_repo_path}/kinesis_${images_stream}_${extr_conf_name}.json"
  - aws_profile
  - verify_certificates
  - endpoint_url
-# # Kafka related environment variables
-# - images_topic
-# - kafka_servers
-# - kafka_security
-# - extr_check_consumer_group
-# - extr_proc_consumer_group
-# - updates_topic
  # Hbase related environment variables
  - hbase_host
  - table_sha1infos
@@ -183,6 +172,7 @@ services:
  - image_buffer_column_family
  - image_buffer_column_name
  - update_info_column_family
+ - "PYTHONWARNINGS='ignore:Unverified HTTPS request'" # Does not seem to work
  # nothing really needs to be persistent here. no other volumes needed
  volumes:
  - $repo_path:$indocker_repo_path
@@ -223,7 +213,7 @@ services:
  - table_updateinfos
  - batch_update_size
  - column_list_sha1s
- - extr_family_column
+ - extr_column_family
  - image_info_column_family
  - image_buffer_column_family
  - image_buffer_column_name