Add a Python provisioning example (pulumi#675)

aws-py-ec2-provisioners/.gitignore

@@ -0,0 +1,4 @@
+*.pyc
+venv/
+rsa
+rsa.pub

aws-py-ec2-provisioners/Pulumi.yaml

@@ -0,0 +1,3 @@
+name: aws-py-ec2-provisioners
+runtime: python
+description: An example of manually configuring an AWS EC2 virtual machine

aws-py-ec2-provisioners/README.md

@@ -0,0 +1,54 @@
+# AWS WebServer with Manual Provisioning (in Python)
+
+This demonstrates using Pulumi dynamic providers to accomplish post-provisioning configuration steps.
+
+Using these building blocks, one can accomplish much of the same as Terraform provisioners.
+
+https://github.com/pulumi/pulumi/issues/1691 tracks designing and developing a complete replacement for provisioners.
+
+## Running the Example
+
+First, create a stack, using `pulumi stack init`.
+
+Now, we need to ensure that our dependencies are installed:
+
+```
+$ python3 -m venv venv
+$ source venv/bin/activate
+$ pip3 install -r requirements.txt
+```
+
+Next, generate an OpenSSH keypair for use with your server - as per the AWS [Requirements][1]
+
+```
+$ ssh-keygen -t rsa -f rsa -b 4096 -m PEM
+```
+
+This will output two files, `rsa` and `rsa.pub`, in the current directory. Be sure not to commit these files!
+
+We then need to configure our stack so that the public key is used by our EC2 instance, and the private key used
+for subsequent SCP and SSH steps that will configure our server after it is stood up.
+
+```
+$ cat rsa.pub | pulumi config set publicKey --
+$ cat rsa | pulumi config set privateKey --secret --
+```
+
+If your key is protected by a passphrase, add that too:
+
+```
+$ pulumi config set privateKeyPassphrase --secret [yourPassphraseHere]
+```
+
+Notice that we've used `--secret` for both `privateKey` and `privateKeyPassphrase`. This ensures their are
+stored in encrypted form in the Pulumi secrets system.
+
+Also set your desired AWS region:
+
+```
+$ pulumi config set aws:region us-west-2
+```
+
+From there, you can run `pulumi up` and all resources will be provisioned and configured.
+
+[1]: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html#how-to-generate-your-own-key-and-import-it-to-aws

aws-py-ec2-provisioners/__main__.py

@@ -0,0 +1,79 @@
+# Copyright 2020, Pulumi Corporation. All rights reserved.
+
+import pulumi
+import pulumi_aws as aws
+import provisioners
+
+# Get the config ready to go.
+config = pulumi.Config()
+
+# If keyName is provided, an existing KeyPair is used, else if publicKey is provided a new KeyPair
+# derived from the publicKey is created.
+key_name = config.get('keyName')
+public_key = config.get('publicKey')
+
+# The privateKey associated with the selected key must be provided (either directly or base64 encoded),
+# along with an optional passphrase if needed.
+def decode_key(key):
+ if key.startswith('-----BEGIN RSA PRIVATE KEY-----'):
+ return key
+ return key.encode('ascii')
+private_key = config.require_secret('privateKey').apply(decode_key)
+private_key_passphrase = config.get_secret('privateKeyPassphrase')
+
+# Create a new security group that permits SSH and web access.
+secgrp = aws.ec2.SecurityGroup('secgrp',
+ description='Foo',
+ ingress=[
+ { 'protocol': 'tcp', 'from_port': 22, 'to_port': 22, 'cidr_blocks': ['0.0.0.0/0'] },
+ { 'protocol': 'tcp', 'from_port': 80, 'to_port': 80, 'cidr_blocks': ['0.0.0.0/0'] },
+ ],
+)
+
+# Get the AMI
+ami = aws.get_ami(
+ owners=['amazon'],
+ most_recent=True,
+ filters=[{
+ 'name': 'name',
+ 'values': ['amzn2-ami-hvm-2.0.????????-x86_64-gp2'],
+ }],
+)
+
+# Create an EC2 server that we'll then provision stuff onto.
+size = 't2.micro'
+if key_name is None:
+ key = aws.ec2.KeyPair('key', public_key=public_key)
+ key_name = key.key_name
+server = aws.ec2.Instance('server',
+ instance_type=size,
+ ami=ami.id,
+ key_name=key_name,
+ vpc_security_group_ids=[ secgrp.id ],
+)
+conn = provisioners.ConnectionArgs(
+ host=server.public_ip,
+ username='ec2-user',
+ private_key=private_key,
+ private_key_passphrase=private_key_passphrase,
+)
+
+# Copy a config file to our server.
+cp_config = provisioners.CopyFile('config',
+ conn=conn,
+ src='myapp.conf',
+ dest='myapp.conf',
+ opts=pulumi.ResourceOptions(depends_on=[server]),
+)
+
+# Execute a basic command on our server.
+cat_config = provisioners.RemoteExec('cat-config',
+ conn=conn,
+ commands=['cat myapp.conf'],
+ opts=pulumi.ResourceOptions(depends_on=[cp_config]),
+)
+
+# Export the server's IP/host and stdout from the command.
+pulumi.export('publicIp', server.public_ip)
+pulumi.export('publicHostName', server.public_dns)
+pulumi.export('catConfigStdout', cat_config.results[0]['stdout'])

aws-py-ec2-provisioners/myapp.conf

@@ -0,0 +1,2 @@
+[test]
+x = 42

aws-py-ec2-provisioners/provisioners.py

@@ -0,0 +1,169 @@
+# Copyright 2020, Pulumi Corporation. All rights reserved.
+
+import abc
+import json
+import io
+import paramiko
+import pulumi
+from pulumi import dynamic
+import sys
+import time
+from typing import Any, Optional
+from typing_extensions import TypedDict
+from uuid import uuid4
+
+# ConnectionArgs tells a provisioner how to access a remote resource. It includes the hostname
+# and optional port (default is 22), username, password, and private key information.
+class ConnectionArgs(TypedDict):
+ host: pulumi.Input[str]
+ """The host to SSH into."""
+ port: Optional[pulumi.Input[int]] = None
+ """The port to SSH into (default 22)."""
+ username: Optional[pulumi.Input[str]] = None
+ """The username for the SSH login."""
+ password: Optional[pulumi.Input[str]] = None
+ """The optional password for the SSH login (private key is recommended instead)."""
+ private_key: Optional[pulumi.Input[str]] = None
+ """The private key, as an ASCII string, to use for the SSH connection."""
+ private_key_passphrase: Optional[pulumi.Input[str]] = None
+ """The private key passphrase, if any, to use for the SSH private key."""
+
+def connect(conn: ConnectionArgs) -> paramiko.SSHClient:
+ ssh = paramiko.SSHClient()
+ ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ skey = io.StringIO(conn['private_key'])
+ pkey = paramiko.RSAKey.from_private_key(skey, password=conn.get('private_key_passphrase'))
+
+ # Retry the connection until the endpoint is available (up to 2 minutes).
+ retries = 0
+ while True:
+ try:
+ ssh.connect(
+ hostname=conn['host'],
+ port=conn.get('port') or 22,
+ username=conn.get('username'),
+ password=conn.get('password'),
+ pkey=pkey,
+ )
+ return ssh
+ except paramiko.ssh_exception.NoValidConnectionsError:
+ if retries == 24:
+ raise
+ time.sleep(5)
+ retries = retries + 1
+ pass
+
+class ProvisionerProvider(dynamic.ResourceProvider):
+ __metaclass__ = abc.ABCMeta
+
+ @abc.abstractmethod
+ def on_create(self, inputs: Any) -> Any:
+ return
+
+ def create(self, inputs):
+ outputs = self.on_create(inputs)
+ return dynamic.CreateResult(id_=uuid4().hex, outs=outputs)
+
+ def diff(self, _id, olds, news):
+ # If anything changed in the inputs, replace the resource.
+ diffs = []
+ for key in olds:
+ if key not in news:
+ diffs.append(key)
+ else:
+ olds_value = json.dumps(olds[key], sort_keys=True, indent=2)
+ news_value = json.dumps(news[key], sort_keys=True, indent=2)
+ if olds_value != news_value:
+ diffs.append(key)
+ for key in news:
+ if key not in olds:
+ diffs.append(key)
+
+ return dynamic.DiffResult(changes=len(diffs) > 0, replaces=diffs, delete_before_replace=True)
+
+# CopyFileProvider implements the resource lifecycle for the CopyFile resource type below.
+class CopyFileProvider(ProvisionerProvider):
+ def on_create(self, inputs: Any) -> Any:
+ ssh = connect(inputs['conn'])
+ scp = ssh.open_sftp()
+ try:
+ scp.put(inputs['src'], inputs['dest'])
+ finally:
+ scp.close()
+ ssh.close()
+ return inputs
+
+# CopyFile is a provisioner step that can copy a file over an SSH connection.
+class CopyFile(dynamic.Resource):
+ def __init__(self, name: str, conn: pulumi.Input[ConnectionArgs],
+ src: str, dest: str, opts: Optional[pulumi.ResourceOptions] = None):
+ self.conn = conn
+ """conn contains information on how to connect to the destination, in addition to dependency information."""
+ self.src = src
+ """
+ src is the source of the file or directory to copy. It can be specified as relative to the current
+ working directory or as an absolute path. This cannot be specified if content is set.
+ """
+ self.dest = dest
+ """dest is required and specifies the absolute path on the target where the file will be copied to."""
+
+ super().__init__(
+ CopyFileProvider(),
+ name,
+ {
+ 'dep': conn,
+ 'conn': conn,
+ 'src': src,
+ 'dest': dest,
+ },
+ opts,
+ )
+
+# RunCommandResult is the result of running a command.
+class RunCommandResult(TypedDict):
+ stdout: str
+ """The stdout of the command that was executed."""
+ stderr: str
+ """The stderr of the command that was executed."""
+
+# RemoteExecProvider implements the resource lifecycle for the RemoteExec resource type below.
+class RemoteExecProvider(ProvisionerProvider):
+ def on_create(self, inputs: Any) -> Any:
+ ssh = connect(inputs['conn'])
+ try:
+ results = []
+ for command in inputs['commands']:
+ stdin, stdout, stderr = ssh.exec_command(command)
+ results.append({
+ 'stdout': ''.join(stdout.readlines()),
+ 'stderr': ''.join(stderr.readlines()),
+ })
+ inputs['results'] = results
+ print(f'results: {results}')
+ finally:
+ ssh.close()
+ return inputs
+
+# RemoteExec runs remote one or more commands over an SSH connection. It returns the resulting
+# stdout and stderr from the commands in the results property.
+class RemoteExec(dynamic.Resource):
+ results: pulumi.Output[list]
+
+ def __init__(self, name: str, conn: ConnectionArgs, commands: list, opts: Optional[pulumi.ResourceOptions] = None):
+ self.conn = conn
+ """conn contains information on how to connect to the destination, in addition to dependency information."""
+ self.commands = commands
+ """The commands to execute. Exactly one of 'command' and 'commands' is required."""
+ self.results = []
+ """The resulting command outputs."""
+
+ super().__init__(
+ RemoteExecProvider(),
+ name,
+ {
+ 'conn': conn,
+ 'commands': commands,
+ 'results': None,
+ },
+ opts,
+ )

aws-py-ec2-provisioners/requirements.txt

@@ -0,0 +1,4 @@
+pulumi>=2.0.0,<3.0.0
+pulumi-aws>=2.0.0,<3.0.0
+paramiko>=2.7.1
+typing_extensions>=3.7.4