Spark in Azure HDInsight sample (pulumi#303)

azure-ts-hdinsight-spark/Pulumi.yaml

@@ -0,0 +1,14 @@
+name: azure-ts-hdinsight-spark
+runtime: nodejs
+description: Spark on Azure HDInsight example
+template:
+ config:
+ azure:environment:
+ description: The Azure environment to use (`public`, `usgovernment`, `german`, `china`)
+ default: public
+ username:
+ description: Spark username
+ secret: true
+ password:
+ description: "Spark password (complex enough to satisfy Azure policy: 8+ chars of lowercase, uppercase, digits, and special symbols)"
+ secret: true

azure-ts-hdinsight-spark/README.md

@@ -0,0 +1,49 @@
+[![Deploy](https://get.pulumi.com/new/button.svg)](https://app.pulumi.com/new)
+
+# Spark on Azure HDInsight example
+
+An example Pulumi component that deploys a Spark cluster on Azure HDInsight.
+
+## Running the App
+
+1. Create a new stack:
+
+ ```
+ $ pulumi stack init dev
+ ```
+
+1. Login to Azure CLI (you will be prompted to do this during deployment if you forget this step):
+
+ ```
+ $ az login
+ ```
+
+1. Restore NPM dependencies:
+
+ ```
+ $ npm install
+ ```
+
+1. Run `pulumi up` to preview and deploy changes:
+
+ ``` 
+ $ pulumi up
+ Previewing changes:
+ ...
+
+ Performing changes:
+ ...
+ info: 5 changes performed:
+ + 5 resources created
+ Update duration: 15m6s
+ ```
+
+1. Check the deployed Spark endpoint:
+
+ ```
+ $ pulumi stack output endpoint
+ https://myspark1234abcd.azurehdinsight.net/
+ 
+ # For instance, Jupyter notebooks are available at https://myspark1234abcd.azurehdinsight.net/jupyter/
+ # Follow https://docs.microsoft.com/en-us/azure/hdinsight/spark/apache-spark-load-data-run-query to test it out
+ ```

azure-ts-hdinsight-spark/index.ts

@@ -0,0 +1,65 @@
+import * as pulumi from "@pulumi/pulumi";
+import * as azure from "@pulumi/azure";
+
+const config = new pulumi.Config();
+const username = config.require("username");
+const password = config.require("password");
+
+// Create an Azure Resource Group
+const resourceGroup = new azure.core.ResourceGroup("spark-rg", {
+ location: "West US",
+});
+
+// Create a storage account and a container for Spark
+const storageAccount = new azure.storage.Account("sparksa", {
+ resourceGroupName: resourceGroup.name,
+ accountReplicationType: "LRS",
+ accountTier: "Standard",
+});
+
+const storageContainer = new azure.storage.Container("spark", {
+ resourceGroupName: resourceGroup.name,
+ storageAccountName: storageAccount.name,
+ containerAccessType: "private",
+});
+
+// Create a Spark cluster in HDInsight
+const sparkCluster = new azure.hdinsight.SparkCluster("myspark", {
+ resourceGroupName: resourceGroup.name,
+ clusterVersion: "3.6",
+ componentVersion: {
+ spark: "2.3",
+ },
+ tier: "Standard",
+ storageAccounts: [{
+ isDefault: true,
+ storageAccountKey: storageAccount.primaryAccessKey,
+ storageContainerId: storageContainer.id,
+ }],
+ gateway: {
+ enabled: true,
+ username,
+ password,
+ },
+ roles: {
+ headNode: {
+ vmSize: "Standard_A3",
+ username,
+ password,
+ },
+ workerNode: {
+ targetInstanceCount: 3,
+ vmSize: "Standard_A3",
+ username,
+ password,
+ },
+ zookeeperNode: {
+ vmSize: "Standard_A3",
+ username,
+ password,
+ },
+ },
+});
+
+// Export the endpoint
+export const endpoint = pulumi.interpolate`https://${sparkCluster.httpsEndpoint}/`;

azure-ts-hdinsight-spark/package.json

@@ -0,0 +1,11 @@
+{
+ "name": "azure-ts-hdinsight-spark",
+ "version": "1.0.0",
+ "devDependencies": {
+ "@types/node": "^10.3.3"
+ },
+ "dependencies": {
+ "@pulumi/azure": "latest",
+ "@pulumi/pulumi": "latest"
+ }
+}

azure-ts-hdinsight-spark/tsconfig.json

@@ -0,0 +1,22 @@
+{
+ "compilerOptions": {
+ "outDir": "bin",
+ "target": "es6",
+ "lib": [
+ "es6"
+ ],
+ "module": "commonjs",
+ "moduleResolution": "node",
+ "sourceMap": true,
+ "experimentalDecorators": true,
+ "pretty": true,
+ "noFallthroughCasesInSwitch": true,
+ "noImplicitAny": true,
+ "noImplicitReturns": true,
+ "forceConsistentCasingInFileNames": true,
+ "strictNullChecks": true
+ },
+ "files": [
+ "index.ts",
+ ]
+}