docker-compose_kinesis.yml

version: "3"

services:

  kinesis:
    build: ./kinesalite
    hostname: kinesis
    container_name: kinesis
    ports:
      - "4567:4567"
    networks:
      cu_imgsearch_net:
# Add a volume for persistence?
#    environment:
#      VIRTUAL_HOST: kinesis
#      VIRTUAL_PORT: 4567
  dynamodb:
    image: amazon/dynamodb-local
    hostname: dynamodb
    container_name: dynamodb
    ports:
      - "4568:8000"
    networks:
      cu_imgsearch_net:
  zookeeper:
    image: wurstmeister/zookeeper
    hostname: zookeeper
    container_name: zookeeper
    environment:
      KAFKA_HEAP_OPTS: "-Xmx256m -Xms256m"
    logging:
      driver: "json-file"
      options:
        max-size: "500k"
        max-file: "7"
    ports:
      - "2181:2181"
    networks:
      cu_imgsearch_net:
  hbase:
    image: kevinsvds/hbase
    hostname: hbase
    container_name: hbase
    ports:
      - "9090:9090"
      - "9095:9095"
      - "60000:60000"
      - "60010:60010"
      - "60020:60020"
      - "60030:60030"
    links:
      - zookeeper:zk
    depends_on:
      - zookeeper
    logging:
      driver: "json-file"
      options:
        max-size: "500k"
        max-file: "7"
    #In /etc/hbase/conf/hbase-site.xml:
    #<name>hbase.rootdir</name>
    #<value>hdfs:https://localhost/hbase</value>
    #In /etc/hadoop/conf/hdfs-site.xml
    #<name>hadoop.tmp.dir</name>
    #<value>/var/lib/hadoop-hdfs/cache/${user.name}</value>
    # So mount volume at /var/lib/hadoop-hdfs for persistence?
    volumes:
      - cuimgsearch-hbase-volume:/var/lib/hadoop-hdfs
    networks:
      cu_imgsearch_net:
  img_pusher:
    image: svebork/columbia_image_search:1.0
    hostname: ${input_conf_name}
    stdin_open: true
    tty: true
    environment:
      # General environment variables
      - input_type
      - "image_pushing_type=${image_ingestion_type}"
      - input_path
      - verbose
      - source_zip
      - "conf_name=${input_conf_name}"
      # Kinesis related environment variables
      - images_stream
      - region_name
      - create_stream
      - aws_profile
      - verify_certificates
      - endpoint_url
      - producer_type
      - producer_prefix
      - "PYTHONWARNINGS='ignore:Unverified HTTPS request'" # Does not seem to work
    cap_add:
      - IPC_LOCK
    # nothing really needs to be persistent here. no other volumes needed
    volumes:
      - $repo_path:$indocker_repo_path
      - $repo_path/conf/aws_credentials/:/home/ubuntu/.aws/
    command: ["bash", "-c", "mkdir ${indocker_repo_path}/conf/generated/ || true && python ${indocker_repo_path}/setup/ConfGenerator/create_conf_ingester.py -o ${indocker_repo_path}/conf/generated/ && ${indocker_repo_path}/scripts/run_images_pusher.sh -c ${input_conf_name} -r ${indocker_repo_path}"]
    logging:
      driver: "json-file"
      options:
        max-size: "500k"
        max-file: "7"
    links:
      - kinesis
    depends_on:
      - kinesis
    networks:
      cu_imgsearch_net:
  img_processor:
    image: svebork/columbia_image_search:1.0
    hostname: ${extr_conf_name}
    stdin_open: true
    tty: true
    cap_add:
      - IPC_LOCK
    environment:
      # General environment variables
      - input_type
      - image_ingestion_type
      - update_ingestion_type
      - producer_type
      - verbose
      - "conf_name=${extr_conf_name}"
      - extr_type
      - extr_nb_threads
      - extr_check_max_delay
      # Kinesis related environment variables
      - region_name
      - images_stream
      - "image_shard_infos_filename=${indocker_repo_path}/kinesis_${images_stream}_${extr_conf_name}.json"
      - aws_profile
      - verify_certificates
      - use_ssl
      - endpoint_url
      # Hbase related environment variables
      - hbase_host
      - table_sha1infos
      - table_updateinfos
      - batch_update_size
      - column_list_sha1s
      - extr_column_family
      - image_info_column_family
      - image_buffer_column_family
      - image_buffer_column_name
      - update_info_column_family
      - "PYTHONWARNINGS='ignore:Unverified HTTPS request'" # Does not seem to work
    # nothing really needs to be persistent here. no other volumes needed
    volumes:
      - $repo_path:$indocker_repo_path
      - $repo_path/conf/aws_credentials/:/home/ubuntu/.aws/
    # TODO: Should we pass all related env var and call conf generation in the docker?
    #command: "mkdir ${indocker_repo_path}/conf/generated/ && python ${indocker_repo_path}/setup/ConfGenerator/create_conf_extractor.py -o ${indocker_repo_path}/conf/generated/ && bash ${indocker_repo_path}/scripts/run_processing.sh -c ${extr_conf_name} -r ${indocker_repo_path}"
    command: ["bash", "-c", "mkdir ${indocker_repo_path}/conf/generated/ || true && python ${indocker_repo_path}/setup/ConfGenerator/create_conf_extractor.py -o ${indocker_repo_path}/conf/generated/ && bash ${indocker_repo_path}/scripts/run_processing.sh -c ${extr_conf_name} -r ${indocker_repo_path}"]
    logging:
      driver: "json-file"
      options:
        max-size: "500k"
        max-file: "7"
    links:
      - kinesis
      - hbase
    depends_on:
      - kinesis
      - hbase
    networks:
      cu_imgsearch_net:
  img_search:
    image: svebork/columbia_image_search:1.0
    hostname: ${search_conf_name}
    stdin_open: true
    tty: true
    cap_add:
      - IPC_LOCK
    environment:
      # General environment variables
      - input_type
      - verbose
      - "conf_name=${search_conf_name}"
      #- "extr_type=${extr_type}"
      - extr_type
      - storer
      # Hbase related environment variables
      - hbase_host
      - table_sha1infos
      - table_updateinfos
      - batch_update_size
      - column_list_sha1s
      - extr_column_family
      - image_info_column_family
      - image_buffer_column_family
      - image_buffer_column_name
      - update_info_column_family
      # Search related environment variables
      - "model_type=${model_type}"
      - "nb_train=${nb_train}"
      - "nb_min_train=${nb_min_train}"
      - "lopq_V=${lopq_V}"
      - "lopq_M=${lopq_M}"
      - "lopq_subq=${lopq_subq}"
      - "reranking=${reranking}"
      # If model_type is lopq_pca:
      - "nb_train_pca=${nb_train_pca}"
      - "nb_min_train_pca=${nb_min_train_pca}"
      - "lopq_pcadims=${lopq_pcadims}"
    # need to add a volume that store the search index data
    volumes:
      - $repo_path:$indocker_repo_path
      - cuimgsearch-volume:/data
      - $repo_path/conf/aws_credentials/:/home/ubuntu/.aws/
    ports:
      - $port_host:5000
    command: ["bash", "-c", "mkdir ${indocker_repo_path}/conf/generated/ || true && python ${indocker_repo_path}/setup/ConfGenerator/create_conf_searcher.py -o ${indocker_repo_path}/conf/generated/ && bash ${indocker_repo_path}/scripts/run_search.sh -c ${search_conf_name} -r ${indocker_repo_path} -e ${endpoint}"]
    logging:
      driver: "json-file"
      options:
        max-size: "500k"
        max-file: "7"
    links:
      - hbase
    depends_on:
      - hbase
    networks:
      cu_imgsearch_net:

networks:
## Something like that should be used when adding image search to an already existing docker-compose
#  external:
#    name: cu_imgsearch_net
  cu_imgsearch_net:
    driver: bridge
    ipam:
      driver: default
      config:
        - subnet: 172.66.0.0/16

volumes:
  cuimgsearch-volume:
  cuimgsearch-hbase-volume: