forked from pulumi/examples
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add a Python provisioning example (pulumi#675)
- Loading branch information
Showing
7 changed files
with
315 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
*.pyc | ||
venv/ | ||
rsa | ||
rsa.pub |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
name: aws-py-ec2-provisioners | ||
runtime: python | ||
description: An example of manually configuring an AWS EC2 virtual machine |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,54 @@ | ||
# AWS WebServer with Manual Provisioning (in Python) | ||
|
||
This demonstrates using Pulumi dynamic providers to accomplish post-provisioning configuration steps. | ||
|
||
Using these building blocks, one can accomplish much of the same as Terraform provisioners. | ||
|
||
https://github.com/pulumi/pulumi/issues/1691 tracks designing and developing a complete replacement for provisioners. | ||
|
||
## Running the Example | ||
|
||
First, create a stack, using `pulumi stack init`. | ||
|
||
Now, we need to ensure that our dependencies are installed: | ||
|
||
``` | ||
$ python3 -m venv venv | ||
$ source venv/bin/activate | ||
$ pip3 install -r requirements.txt | ||
``` | ||
|
||
Next, generate an OpenSSH keypair for use with your server - as per the AWS [Requirements][1] | ||
|
||
``` | ||
$ ssh-keygen -t rsa -f rsa -b 4096 -m PEM | ||
``` | ||
|
||
This will output two files, `rsa` and `rsa.pub`, in the current directory. Be sure not to commit these files! | ||
|
||
We then need to configure our stack so that the public key is used by our EC2 instance, and the private key used | ||
for subsequent SCP and SSH steps that will configure our server after it is stood up. | ||
|
||
``` | ||
$ cat rsa.pub | pulumi config set publicKey -- | ||
$ cat rsa | pulumi config set privateKey --secret -- | ||
``` | ||
|
||
If your key is protected by a passphrase, add that too: | ||
|
||
``` | ||
$ pulumi config set privateKeyPassphrase --secret [yourPassphraseHere] | ||
``` | ||
|
||
Notice that we've used `--secret` for both `privateKey` and `privateKeyPassphrase`. This ensures their are | ||
stored in encrypted form in the Pulumi secrets system. | ||
|
||
Also set your desired AWS region: | ||
|
||
``` | ||
$ pulumi config set aws:region us-west-2 | ||
``` | ||
|
||
From there, you can run `pulumi up` and all resources will be provisioned and configured. | ||
|
||
[1]: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html#how-to-generate-your-own-key-and-import-it-to-aws |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,79 @@ | ||
# Copyright 2020, Pulumi Corporation. All rights reserved. | ||
|
||
import pulumi | ||
import pulumi_aws as aws | ||
import provisioners | ||
|
||
# Get the config ready to go. | ||
config = pulumi.Config() | ||
|
||
# If keyName is provided, an existing KeyPair is used, else if publicKey is provided a new KeyPair | ||
# derived from the publicKey is created. | ||
key_name = config.get('keyName') | ||
public_key = config.get('publicKey') | ||
|
||
# The privateKey associated with the selected key must be provided (either directly or base64 encoded), | ||
# along with an optional passphrase if needed. | ||
def decode_key(key): | ||
if key.startswith('-----BEGIN RSA PRIVATE KEY-----'): | ||
return key | ||
return key.encode('ascii') | ||
private_key = config.require_secret('privateKey').apply(decode_key) | ||
private_key_passphrase = config.get_secret('privateKeyPassphrase') | ||
|
||
# Create a new security group that permits SSH and web access. | ||
secgrp = aws.ec2.SecurityGroup('secgrp', | ||
description='Foo', | ||
ingress=[ | ||
{ 'protocol': 'tcp', 'from_port': 22, 'to_port': 22, 'cidr_blocks': ['0.0.0.0/0'] }, | ||
{ 'protocol': 'tcp', 'from_port': 80, 'to_port': 80, 'cidr_blocks': ['0.0.0.0/0'] }, | ||
], | ||
) | ||
|
||
# Get the AMI | ||
ami = aws.get_ami( | ||
owners=['amazon'], | ||
most_recent=True, | ||
filters=[{ | ||
'name': 'name', | ||
'values': ['amzn2-ami-hvm-2.0.????????-x86_64-gp2'], | ||
}], | ||
) | ||
|
||
# Create an EC2 server that we'll then provision stuff onto. | ||
size = 't2.micro' | ||
if key_name is None: | ||
key = aws.ec2.KeyPair('key', public_key=public_key) | ||
key_name = key.key_name | ||
server = aws.ec2.Instance('server', | ||
instance_type=size, | ||
ami=ami.id, | ||
key_name=key_name, | ||
vpc_security_group_ids=[ secgrp.id ], | ||
) | ||
conn = provisioners.ConnectionArgs( | ||
host=server.public_ip, | ||
username='ec2-user', | ||
private_key=private_key, | ||
private_key_passphrase=private_key_passphrase, | ||
) | ||
|
||
# Copy a config file to our server. | ||
cp_config = provisioners.CopyFile('config', | ||
conn=conn, | ||
src='myapp.conf', | ||
dest='myapp.conf', | ||
opts=pulumi.ResourceOptions(depends_on=[server]), | ||
) | ||
|
||
# Execute a basic command on our server. | ||
cat_config = provisioners.RemoteExec('cat-config', | ||
conn=conn, | ||
commands=['cat myapp.conf'], | ||
opts=pulumi.ResourceOptions(depends_on=[cp_config]), | ||
) | ||
|
||
# Export the server's IP/host and stdout from the command. | ||
pulumi.export('publicIp', server.public_ip) | ||
pulumi.export('publicHostName', server.public_dns) | ||
pulumi.export('catConfigStdout', cat_config.results[0]['stdout']) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
[test] | ||
x = 42 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,169 @@ | ||
# Copyright 2020, Pulumi Corporation. All rights reserved. | ||
|
||
import abc | ||
import json | ||
import io | ||
import paramiko | ||
import pulumi | ||
from pulumi import dynamic | ||
import sys | ||
import time | ||
from typing import Any, Optional | ||
from typing_extensions import TypedDict | ||
from uuid import uuid4 | ||
|
||
# ConnectionArgs tells a provisioner how to access a remote resource. It includes the hostname | ||
# and optional port (default is 22), username, password, and private key information. | ||
class ConnectionArgs(TypedDict): | ||
host: pulumi.Input[str] | ||
"""The host to SSH into.""" | ||
port: Optional[pulumi.Input[int]] = None | ||
"""The port to SSH into (default 22).""" | ||
username: Optional[pulumi.Input[str]] = None | ||
"""The username for the SSH login.""" | ||
password: Optional[pulumi.Input[str]] = None | ||
"""The optional password for the SSH login (private key is recommended instead).""" | ||
private_key: Optional[pulumi.Input[str]] = None | ||
"""The private key, as an ASCII string, to use for the SSH connection.""" | ||
private_key_passphrase: Optional[pulumi.Input[str]] = None | ||
"""The private key passphrase, if any, to use for the SSH private key.""" | ||
|
||
def connect(conn: ConnectionArgs) -> paramiko.SSHClient: | ||
ssh = paramiko.SSHClient() | ||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) | ||
skey = io.StringIO(conn['private_key']) | ||
pkey = paramiko.RSAKey.from_private_key(skey, password=conn.get('private_key_passphrase')) | ||
|
||
# Retry the connection until the endpoint is available (up to 2 minutes). | ||
retries = 0 | ||
while True: | ||
try: | ||
ssh.connect( | ||
hostname=conn['host'], | ||
port=conn.get('port') or 22, | ||
username=conn.get('username'), | ||
password=conn.get('password'), | ||
pkey=pkey, | ||
) | ||
return ssh | ||
except paramiko.ssh_exception.NoValidConnectionsError: | ||
if retries == 24: | ||
raise | ||
time.sleep(5) | ||
retries = retries + 1 | ||
pass | ||
|
||
class ProvisionerProvider(dynamic.ResourceProvider): | ||
__metaclass__ = abc.ABCMeta | ||
|
||
@abc.abstractmethod | ||
def on_create(self, inputs: Any) -> Any: | ||
return | ||
|
||
def create(self, inputs): | ||
outputs = self.on_create(inputs) | ||
return dynamic.CreateResult(id_=uuid4().hex, outs=outputs) | ||
|
||
def diff(self, _id, olds, news): | ||
# If anything changed in the inputs, replace the resource. | ||
diffs = [] | ||
for key in olds: | ||
if key not in news: | ||
diffs.append(key) | ||
else: | ||
olds_value = json.dumps(olds[key], sort_keys=True, indent=2) | ||
news_value = json.dumps(news[key], sort_keys=True, indent=2) | ||
if olds_value != news_value: | ||
diffs.append(key) | ||
for key in news: | ||
if key not in olds: | ||
diffs.append(key) | ||
|
||
return dynamic.DiffResult(changes=len(diffs) > 0, replaces=diffs, delete_before_replace=True) | ||
|
||
# CopyFileProvider implements the resource lifecycle for the CopyFile resource type below. | ||
class CopyFileProvider(ProvisionerProvider): | ||
def on_create(self, inputs: Any) -> Any: | ||
ssh = connect(inputs['conn']) | ||
scp = ssh.open_sftp() | ||
try: | ||
scp.put(inputs['src'], inputs['dest']) | ||
finally: | ||
scp.close() | ||
ssh.close() | ||
return inputs | ||
|
||
# CopyFile is a provisioner step that can copy a file over an SSH connection. | ||
class CopyFile(dynamic.Resource): | ||
def __init__(self, name: str, conn: pulumi.Input[ConnectionArgs], | ||
src: str, dest: str, opts: Optional[pulumi.ResourceOptions] = None): | ||
self.conn = conn | ||
"""conn contains information on how to connect to the destination, in addition to dependency information.""" | ||
self.src = src | ||
""" | ||
src is the source of the file or directory to copy. It can be specified as relative to the current | ||
working directory or as an absolute path. This cannot be specified if content is set. | ||
""" | ||
self.dest = dest | ||
"""dest is required and specifies the absolute path on the target where the file will be copied to.""" | ||
|
||
super().__init__( | ||
CopyFileProvider(), | ||
name, | ||
{ | ||
'dep': conn, | ||
'conn': conn, | ||
'src': src, | ||
'dest': dest, | ||
}, | ||
opts, | ||
) | ||
|
||
# RunCommandResult is the result of running a command. | ||
class RunCommandResult(TypedDict): | ||
stdout: str | ||
"""The stdout of the command that was executed.""" | ||
stderr: str | ||
"""The stderr of the command that was executed.""" | ||
|
||
# RemoteExecProvider implements the resource lifecycle for the RemoteExec resource type below. | ||
class RemoteExecProvider(ProvisionerProvider): | ||
def on_create(self, inputs: Any) -> Any: | ||
ssh = connect(inputs['conn']) | ||
try: | ||
results = [] | ||
for command in inputs['commands']: | ||
stdin, stdout, stderr = ssh.exec_command(command) | ||
results.append({ | ||
'stdout': ''.join(stdout.readlines()), | ||
'stderr': ''.join(stderr.readlines()), | ||
}) | ||
inputs['results'] = results | ||
print(f'results: {results}') | ||
finally: | ||
ssh.close() | ||
return inputs | ||
|
||
# RemoteExec runs remote one or more commands over an SSH connection. It returns the resulting | ||
# stdout and stderr from the commands in the results property. | ||
class RemoteExec(dynamic.Resource): | ||
results: pulumi.Output[list] | ||
|
||
def __init__(self, name: str, conn: ConnectionArgs, commands: list, opts: Optional[pulumi.ResourceOptions] = None): | ||
self.conn = conn | ||
"""conn contains information on how to connect to the destination, in addition to dependency information.""" | ||
self.commands = commands | ||
"""The commands to execute. Exactly one of 'command' and 'commands' is required.""" | ||
self.results = [] | ||
"""The resulting command outputs.""" | ||
|
||
super().__init__( | ||
RemoteExecProvider(), | ||
name, | ||
{ | ||
'conn': conn, | ||
'commands': commands, | ||
'results': None, | ||
}, | ||
opts, | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
pulumi>=2.0.0,<3.0.0 | ||
pulumi-aws>=2.0.0,<3.0.0 | ||
paramiko>=2.7.1 | ||
typing_extensions>=3.7.4 |