Merge pull request datopian#8 from datahq/feature/rds

[RDS][m]: create RDS instance - fixes datopian#5.

.env.template

@@ -25,6 +25,12 @@ AWS_SECRET_KEY=
 # Bucket locations (used by various services)
 PKGSTORE_BUCKET=pkgstore-${STAGE}.${DOMAIN_BASE}
 
+
+# RDS
+RDS_DATABASE_USERNAME=${PROJECT}
+RDS_DATABASE_PASSWORD=
+RDS_DATABASE_URI=
+
 # ============
 # auth service
 # ============
@@ -45,4 +51,3 @@ GITHUB_SECRET=
 
 # NOTE: storage credentials are above in Object Storage
 RAWSTORE_BUCKET=rawstore-${STAGE}.${DOMAIN_BASE}
-

docker-cloud.yml

@@ -24,32 +24,20 @@ auth:
  autoredeploy: true
  restart: always
  image: datopian/auth
- links:
- - postgres
  ports:
  - "8000"
  target_num_containers: 1
  environment:
  VIRTUAL_HOST: ${DOMAIN_API}/auth/*
  GUNICORN_PORT: 8000
- DATABASE_URL: "postgresql:https://datahub@postgres/datahub"
+ DATABASE_URL: ${RDS_DATABASE_URI}
  EXTERNAL_ADDRESS: ${DOMAIN_API}
  PRIVATE_KEY:
  PUBLIC_KEY:
  GOOGLE_KEY:
  GOOGLE_SECRET:
  GITHUB_KEY:
  GITHUB_SECRET:
-postgres:
- autoredeploy: true
- restart: always
- image: postgres
- target_num_containers: 1
- ports:
- - "5432"
- environment:
- POSTGRES_USER: "datahub"
- POSTGRES_DB: "datahub"
 rawstore:
  autoredeploy: true
  restart: always
@@ -71,7 +59,7 @@ assembler:
  restart: always
  environment:
  VIRTUAL_HOST: ${DOMAIN_API}/pipelines/*
- SOURCESPEC_REGISTRY_DB_ENGINE: postgresql:https://datahub@postgres/datahub
+ SOURCESPEC_REGISTRY_DB_ENGINE:  ${RDS_DATABASE_URI}
  DPP_BASE_PATH: /pipelines/
  PKGSTORE_BUCKET: ${PKGSTORE_BUCKET}
  AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY}
@@ -85,8 +73,7 @@ specstore:
  environment:
  VIRTUAL_HOST: ${DOMAIN_API}/source/*
  AUTH_SERVER: auth:8000
- DATABASE_URL: postgresql:https://datahub@postgres/datahub
+ DATABASE_URL:  ${RDS_DATABASE_URI}
  expose:
  - '8000'
  image: 'datopian/specstore:latest'
-

main.py

@@ -5,9 +5,10 @@
 import subprocess
 import optparse
 import sys
-import time
+from time import sleep
 
 import boto3
+from botocore.exceptions import ClientError
 import dotenv
 
 
@@ -27,6 +28,9 @@ def __init__(self, configfile='.env'):
  # we need stuff in the environment for docker
  os.environ.update(out)
  self.config = os.environ
+ rds_uri = self.config.get('RDS_DATABASE_URI')
+ if not rds_uri:
+ print('Warning: RDS_DATABASE_URI is not set. please set, or run `python main.py rds`')
 
  @property
  def stackname(self):
@@ -148,6 +152,95 @@ def _s3_enable_logs(self, client, bucket):
  }
  )
 
+ def rds(self):
+ """Create an RDS instance and enable public access"""
+ client = boto3.client(
+ 'rds',
+ aws_access_key_id=self.config['AWS_ACCESS_KEY'],
+ aws_secret_access_key=self.config['AWS_SECRET_KEY']
+ )
+ try:
+ self._rds_create(client)
+ return self._rds_enable_public_access()
+ except Exception as e:
+ if 'DBInstanceAlreadyExists' in e.message:
+ print('RDS instance already exists - reusing')
+ self._rds_exists(client)
+ return True
+ else:
+ print(e.message)
+ return False
+
+ def _rds_create(self, client):
+ """Boot an RDS instance"""
+
+ rds_instance = '%(PROJECT)s-%(STAGE)s' % self.config
+ client.create_db_instance(
+ DBName=rds_instance.replace('-','_'),
+ DBInstanceIdentifier=rds_instance,
+ AllocatedStorage=10,
+ Engine='postgres',
+ MasterUsername=self.config['RDS_DATABASE_USERNAME'],
+ MasterUserPassword=self.config['RDS_DATABASE_PASSWORD'],
+ BackupRetentionPeriod=30,
+ Port=5432,
+ MultiAZ=False,
+ PubliclyAccessible=True,
+ DBInstanceClass='db.t2.micro'
+ )
+ self._rds_exists(client, wait=1500)
+
+
+ def _rds_exists(self, client, wait=0):
+ """Will check rds instance is already exists or not"""
+ rds_instance = '%(PROJECT)s-%(STAGE)s' % self.config
+ seconds = 0
+ while True:
+ response = client.describe_db_instances(DBInstanceIdentifier=rds_instance)
+ instance = response['DBInstances'][0]
+ if instance['DBInstanceStatus'] == "available":
+ rds_uri = 'postgres:https://{user}:{password}@{endpoint}:{port}/{db}'\
+ .format(
+ user=self.config['RDS_DATABASE_USERNAME'],
+ password=self.config['RDS_DATABASE_PASSWORD'],
+ endpoint=instance['Endpoint']['Address'],
+ port=5432,
+ db=rds_instance.replace('-','_')
+ )
+ self.config['RDS_DATABASE_URI'] = rds_uri
+ print('Please set RDS_DATABASE_URI in your .env file:\n%s' % rds_uri)
+ break
+ print("%s: %d seconds elapsed" % (response['DBInstances'][0]['DBInstanceStatus'], seconds))
+ sleep(5)
+ seconds += 5
+ if seconds > wait:
+ return False
+ return True
+
+ def _rds_enable_public_access(self):
+ """Enable public access for RDS database """
+ client = boto3.client(
+ 'ec2',
+ aws_access_key_id=self.config['AWS_ACCESS_KEY'],
+ aws_secret_access_key=self.config['AWS_SECRET_KEY']
+ )
+ try:
+ client.authorize_security_group_ingress(
+ GroupId='sg-c20048bf',
+ IpProtocol="-1",
+ CidrIp="0.0.0.0/0",
+ FromPort=0,
+ ToPort=65535
+ )
+ return True
+ except Exception as e:
+ if 'InvalidPermission.Duplicate' in e.message:
+ print('The specified rule already exists')
+ return True
+ else:
+ print(e.message)
+ return False
+
 
 # ==============================================
 # CLI