Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

1040 - 1/3 Create ALB version of the API #2303

Merged
merged 5 commits into from
Jun 20, 2024
Merged

1040 - 1/3 Create ALB version of the API #2303

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
c72ffdf
fix(infra): 1/3 create ALB version of the API
leite08 Jun 19, 2024
5bd5ce6
fix(infra): 1/3 permissions to the ALB based fargate service
leite08 Jun 19, 2024
cfe3969
fix(infra): 1/3 add DB access to new ALB fargate service
leite08 Jun 19, 2024
3f316aa
fix(infra): 1/3 NLB pointing to ALB
leite08 Jun 19, 2024
e032d95
fix(infra): 1/3 use existing log on new ALB fargate
leite08 Jun 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions packages/infra/lib/api-stack.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -396,6 +396,7 @@ export class APIStack extends Stack {
service: apiService,
loadBalancerAddress: apiLoadBalancerAddress,
serverAddress: apiServerUrl,
apiServiceAdditional,
} = createAPIService({
stack: this,
props,
Expand Down Expand Up @@ -456,6 +457,8 @@ export class APIStack extends Stack {

// Access grant for Aurora DB
dbCluster.connections.allowDefaultPortFrom(apiService.service);
// TODO remove this on 3/3
dbCluster.connections.allowDefaultPortFrom(apiServiceAdditional);

// setup a private link so the API can talk to the NLB
const link = new apig.VpcLink(this, "link", {
Expand Down
201 changes: 201 additions & 0 deletions packages/infra/lib/api-stack/api-service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,15 @@ import * as dynamodb from "aws-cdk-lib/aws-dynamodb";
import * as ec2 from "aws-cdk-lib/aws-ec2";
import { Repository } from "aws-cdk-lib/aws-ecr";
import * as ecs from "aws-cdk-lib/aws-ecs";
import { FargateService } from "aws-cdk-lib/aws-ecs";
import * as ecs_patterns from "aws-cdk-lib/aws-ecs-patterns";
import {
ApplicationProtocol,
NetworkLoadBalancer,
NetworkTargetGroup,
Protocol,
} from "aws-cdk-lib/aws-elasticloadbalancingv2";
import { AlbTarget } from "aws-cdk-lib/aws-elasticloadbalancingv2-targets";
import * as iam from "aws-cdk-lib/aws-iam";
import { IFunction as ILambda } from "aws-cdk-lib/aws-lambda";
import * as rds from "aws-cdk-lib/aws-rds";
Expand Down Expand Up @@ -92,6 +100,7 @@ export function createAPIService({
service: ecs_patterns.NetworkLoadBalancedFargateService;
serverAddress: string;
loadBalancerAddress: string;
apiServiceAdditional: FargateService;
} {
// Create a new Amazon Elastic Container Service (ECS) cluster
const cluster = new ecs.Cluster(stack, "APICluster", { vpc, containerInsights: true });
Expand All @@ -117,6 +126,121 @@ export function createAPIService({
port: dbReadReplicaEndpoint.port,
});
// Run some servers on fargate containers
const listenerPort = 80;
const containerPort = 8080;
// TODO RENAME to remove "Alb"
const fargateServiceAlb = new ecs_patterns.ApplicationLoadBalancedFargateService(
stack,
"APIFargateServiceAlb",
{
cluster: cluster,
// Watch out for the combination of vCPUs and memory, more vCPU requires more memory
// https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definition_parameters.html#task_size
cpu: isProd(props.config) ? 2048 : 1024,
desiredCount: isProd(props.config) ? 2 : 1,
taskImageOptions: {
image: ecs.ContainerImage.fromEcrRepository(ecrRepo, "latest"),
containerPort,
containerName: "API-Server",
secrets: {
DB_CREDS: ecs.Secret.fromSecretsManager(dbCredsSecret),
SEARCH_PASSWORD: ecs.Secret.fromSecretsManager(searchAuth.secret),
...secretsToECS(secrets),
},
environment: {
NODE_ENV: "production", // Determines its being run in the cloud, the logical env is set on ENV_TYPE
ENV_TYPE: props.config.environmentType, // staging, production, sandbox
...(props.version ? { METRIPORT_VERSION: props.version } : undefined),
AWS_REGION: props.config.region,
DB_READ_REPLICA_ENDPOINT: dbReadReplicaEndpointAsString,
TOKEN_TABLE_NAME: dynamoDBTokenTable.tableName,
API_URL: `https://${props.config.subdomain}.${props.config.domain}`,
...(props.config.apiGatewayUsagePlanId
? { API_GW_USAGE_PLAN_ID: props.config.apiGatewayUsagePlanId }
: {}),
CONNECT_WIDGET_URL: connectWidgetUrlEnvVar,
SYSTEM_ROOT_OID: props.config.systemRootOID,
SYSTEM_ROOT_ORG_NAME: props.config.systemRootOrgName,
...props.config.commonwell.envVars,
...(props.config.slack ? props.config.slack : undefined),
...(props.config.sentryDSN ? { SENTRY_DSN: props.config.sentryDSN } : undefined),
...(props.config.usageReportUrl && {
USAGE_URL: props.config.usageReportUrl,
}),
...(props.config.medicalDocumentsBucketName && {
MEDICAL_DOCUMENTS_BUCKET_NAME: props.config.medicalDocumentsBucketName,
}),
...(props.config.medicalDocumentsUploadBucketName && {
MEDICAL_DOCUMENTS_UPLOADS_BUCKET_NAME: props.config.medicalDocumentsUploadBucketName,
}),
...(isSandbox(props.config) && {
SANDBOX_SEED_DATA_BUCKET_NAME: props.config.sandboxSeedDataBucketName,
}),
CONVERT_DOC_LAMBDA_NAME: cdaToVisualizationLambda.functionName,
DOCUMENT_DOWNLOADER_LAMBDA_NAME: documentDownloaderLambda.functionName,
...(iheGateway
? {
IHE_GW_URL: `http:https://${iheGateway.outboundSubdomain}.${props.config.domain}`,
IHE_GW_PORT_PD: iheGateway.outboundPorts.patientDiscovery.toString(),
IHE_GW_PORT_DQ: iheGateway.outboundPorts.documentQuery.toString(),
IHE_GW_PORT_DR: iheGateway.outboundPorts.documentRetrieval.toString(),
}
: undefined),
OUTBOUND_PATIENT_DISCOVERY_LAMBDA_NAME: outboundPatientDiscoveryLambda.functionName,
OUTBOUND_DOC_QUERY_LAMBDA_NAME: outboundDocumentQueryLambda.functionName,
OUTBOUND_DOC_RETRIEVAL_LAMBDA_NAME: outboundDocumentRetrievalLambda.functionName,
...(fhirToMedicalRecordLambda && {
FHIR_TO_MEDICAL_RECORD_LAMBDA_NAME: fhirToMedicalRecordLambda.functionName,
}),
...(fhirToCdaConverterLambda && {
FHIR_TO_CDA_CONVERTER_LAMBDA_NAME: fhirToCdaConverterLambda.functionName,
}),
FHIR_SERVER_URL: fhirServerUrl,
...(fhirServerQueueUrl && {
FHIR_SERVER_QUEUE_URL: fhirServerQueueUrl,
}),
...(fhirConverterQueueUrl && {
FHIR_CONVERTER_QUEUE_URL: fhirConverterQueueUrl,
}),
...(fhirConverterServiceUrl && {
FHIR_CONVERTER_SERVER_URL: fhirConverterServiceUrl,
}),
SEARCH_INGESTION_QUEUE_URL: searchIngestionQueue.queueUrl,
SEARCH_ENDPOINT: searchEndpoint,
SEARCH_USERNAME: searchAuth.userName,
SEARCH_INDEX: searchIndexName,
...(props.config.carequality?.envVars?.CQ_ORG_URLS && {
CQ_ORG_URLS: props.config.carequality.envVars.CQ_ORG_URLS,
}),
...(props.config.carequality?.envVars?.CQ_URLS_TO_EXCLUDE && {
CQ_URLS_TO_EXCLUDE: props.config.carequality.envVars.CQ_URLS_TO_EXCLUDE,
}),
...(props.config.locationService && {
PLACE_INDEX_NAME: props.config.locationService.placeIndexName,
PLACE_INDEX_REGION: props.config.locationService.placeIndexRegion,
}),
// app config
APPCONFIG_APPLICATION_ID: appConfigEnvVars.appId,
APPCONFIG_CONFIGURATION_ID: appConfigEnvVars.configId,
...(coverageEnhancementConfig && {
CW_MANAGEMENT_URL: coverageEnhancementConfig.managementUrl,
}),
...(cookieStore && {
CW_MANAGEMENT_COOKIE_SECRET_ARN: cookieStore.secretArn,
}),
...(props.config.iheGateway?.trustStoreBucketName && {
CQ_TRUST_BUNDLE_BUCKET_NAME: props.config.iheGateway.trustStoreBucketName,
}),
},
},
memoryLimitMiB: isProd(props.config) ? 4096 : 2048,
healthCheckGracePeriod: Duration.seconds(60),
protocol: ApplicationProtocol.HTTP,
listenerPort,
publicLoadBalancer: false,
}
);
// TODO DEPRECATED start
const fargateService = new ecs_patterns.NetworkLoadBalancedFargateService(
stack,
"APIFargateService",
Expand Down Expand Up @@ -226,6 +350,7 @@ export function createAPIService({
publicLoadBalancer: false,
}
);
// TODO DEPRECATED end
const serverAddress = fargateService.loadBalancer.loadBalancerDnsName;
const apiUrl = `${props.config.subdomain}.${props.config.domain}`;
new r53.ARecord(stack, "APIDomainPrivateRecord", {
Expand All @@ -236,8 +361,38 @@ export function createAPIService({
),
});

const alb = fargateServiceAlb.loadBalancer;
const nlb = new NetworkLoadBalancer(stack, `ApiNetworkLoadBalancer`, {
vpc,
internetFacing: false,
});
const nlbListener = nlb.addListener(`ApiNetworkLoadBalancerListener`, {
port: listenerPort,
protocol: Protocol.TCP,
});
const nlbTargetGroup = new NetworkTargetGroup(stack, `ApiNetworkTargetGroup`, {
port: listenerPort,
protocol: Protocol.TCP,
vpc,
targets: [new AlbTarget(alb, listenerPort)],
});
nlbListener.addTargetGroups("ApiNetworkLoadBalancerTargetGroup", nlbTargetGroup);

// Health checks
const healthcheck = {
healthyThresholdCount: 2,
unhealthyThresholdCount: 2,
interval: Duration.seconds(10),
};
fargateServiceAlb.targetGroup.configureHealthCheck(healthcheck);
nlbTargetGroup.configureHealthCheck({
...healthcheck,
interval: healthcheck.interval.plus(Duration.seconds(3)),
});

// Access grant for Aurora DB's secret
dbCredsSecret.grantRead(fargateService.taskDefinition.taskRole);
dbCredsSecret.grantRead(fargateServiceAlb.taskDefinition.taskRole);
// RW grant for Dynamo DB
dynamoDBTokenTable.grantReadWriteData(fargateService.taskDefinition.taskRole);
cdaToVisualizationLambda.grantInvoke(fargateService.taskDefinition.taskRole);
Expand All @@ -246,18 +401,29 @@ export function createAPIService({
outboundDocumentQueryLambda.grantInvoke(fargateService.taskDefinition.taskRole);
outboundDocumentRetrievalLambda.grantInvoke(fargateService.taskDefinition.taskRole);
fhirToCdaConverterLambda?.grantInvoke(fargateService.taskDefinition.taskRole);
dynamoDBTokenTable.grantReadWriteData(fargateServiceAlb.taskDefinition.taskRole);
cdaToVisualizationLambda.grantInvoke(fargateServiceAlb.taskDefinition.taskRole);
documentDownloaderLambda.grantInvoke(fargateServiceAlb.taskDefinition.taskRole);
outboundPatientDiscoveryLambda.grantInvoke(fargateServiceAlb.taskDefinition.taskRole);
outboundDocumentQueryLambda.grantInvoke(fargateServiceAlb.taskDefinition.taskRole);
outboundDocumentRetrievalLambda.grantInvoke(fargateServiceAlb.taskDefinition.taskRole);
fhirToCdaConverterLambda?.grantInvoke(fargateServiceAlb.taskDefinition.taskRole);

// Access grant for medical document buckets
medicalDocumentsUploadBucket.grantReadWrite(fargateService.taskDefinition.taskRole);
medicalDocumentsUploadBucket.grantReadWrite(fargateServiceAlb.taskDefinition.taskRole);

if (fhirToMedicalRecordLambda) {
fhirToMedicalRecordLambda.grantInvoke(fargateService.taskDefinition.taskRole);
fhirToMedicalRecordLambda.grantInvoke(fargateServiceAlb.taskDefinition.taskRole);
cdaToVisualizationLambda.grantInvoke(fhirToMedicalRecordLambda);
}

if (cookieStore) {
cookieStore.grantRead(fargateService.service.taskDefinition.taskRole);
cookieStore.grantWrite(fargateService.service.taskDefinition.taskRole);
cookieStore.grantRead(fargateServiceAlb.service.taskDefinition.taskRole);
cookieStore.grantWrite(fargateServiceAlb.service.taskDefinition.taskRole);
}

// Allow access to search services/infra
Expand All @@ -267,6 +433,12 @@ export function createAPIService({
resource: fargateService.taskDefinition.taskRole,
});
searchAuth.secret.grantRead(fargateService.taskDefinition.taskRole);
provideAccessToQueue({
accessType: "send",
queue: searchIngestionQueue,
resource: fargateServiceAlb.taskDefinition.taskRole,
});
searchAuth.secret.grantRead(fargateServiceAlb.taskDefinition.taskRole);

// Setting permissions for AppConfig
fargateService.taskDefinition.taskRole.attachInlinePolicy(
Expand All @@ -289,6 +461,26 @@ export function createAPIService({
],
})
);
fargateServiceAlb.taskDefinition.taskRole.attachInlinePolicy(
new iam.Policy(stack, "ApiPermissionsForAppConfig", {
statements: [
new iam.PolicyStatement({
actions: [
"appconfig:StartConfigurationSession",
"appconfig:GetLatestConfiguration",
"appconfig:GetConfiguration",
"apigateway:GET",
],
resources: ["*"],
}),
new iam.PolicyStatement({
actions: ["geo:SearchPlaceIndexForText"],
resources: [`arn:aws:geo:*`],
effect: iam.Effect.ALLOW,
}),
],
})
);

// CloudWatch Alarms and Notifications

Expand Down Expand Up @@ -323,10 +515,17 @@ export function createAPIService({
ec2.Port.allTraffic(),
"Allow traffic from within the VPC to the service secure port"
);
fargateServiceAlb.service.connections.allowFrom(
ec2.Peer.ipv4(vpc.vpcCidrBlock),
ec2.Port.allTraffic(),
"Allow traffic from within the VPC to the service secure port"
);
// TODO: #489 ain't the most secure, but the above code doesn't work as CDK complains we can't use the connections
// from the cluster created above, should be fine for now as it will only accept connections in the VPC
fargateService.service.connections.allowFromAnyIpv4(ec2.Port.allTcp());
fargateServiceAlb.service.connections.allowFromAnyIpv4(ec2.Port.allTcp());

// TODO DEPRECATED start
// This speeds up deployments so the tasks are swapped quicker.
// See for details: https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-target-groups.html#deregistration-delay
fargateService.targetGroup.setAttribute("deregistration_delay.timeout_seconds", "17");
Expand All @@ -337,6 +536,7 @@ export function createAPIService({
healthyThresholdCount: 2,
interval: Duration.seconds(10),
});
// TODO DEPRECATED end

// hookup autoscaling based on 90% thresholds
const scaling = fargateService.service.autoScaleTaskCount({
Expand All @@ -359,5 +559,6 @@ export function createAPIService({
service: fargateService,
serverAddress: apiUrl,
loadBalancerAddress: serverAddress,
apiServiceAdditional: fargateServiceAlb.service,
};
}
Loading