feat: Support for targetting arbitrary IC network on the cli (#302)

* WIP

* More updates

* Finished refactoring

* clippy

* Some fixes after merging

* Add tests and fix some encountered issues

* bazel repin

* Remove unused struct

* Fix bazel build of slack-notifications

* Allow network access from bazel tests

* Cleanup confusingly duplicate names

* Ignore failures of the staging IC test runs, due to firewall restrictions

* fix ic-admin args

* Prefer private pem key if provided, for trustworthy metrics

.bazelrc

@@ -26,7 +26,7 @@ build:lint --config=clippy
 
 build:ci --progress_report_interval=30
 
-build --nosandbox_default_allow_network
+build --sandbox_default_allow_network
 build --incompatible_strict_action_env # use an environment with a static value for PATH and do not inherit LD_LIBRARY_PATH
 
 # default to optimized and unstripped binaries.

.vscode/settings.json

@@ -1,5 +1,6 @@
 {
  "yaml.schemas": {
  "release-index-shema.json": "release-index.yaml"
- }
+ },
+ "rust-analyzer.showUnlinkedFileNotification": false
 }

Cargo.Bazel.lock

@@ -1,5 +1,5 @@
 {
- "checksum": "31058f9f66529f2b114e4c4508e3cc6dbf7243335ca17851bca859ae720bcccd",
+ "checksum": "b6737f3f6bdb970486578a3c4c615afe6fd16e0bf32c3930b649d568c6be06c5",
  "crates": {
  "actix-codec 0.5.2": {
  "name": "actix-codec",
@@ -22116,6 +22116,10 @@
  "id": "chrono 0.4.37",
  "target": "chrono"
  },
+ {
+ "id": "futures 0.3.30",
+ "target": "futures"
+ },
  {
  "id": "ic-base-types 0.9.0",
  "target": "ic_base_types"
@@ -22152,13 +22156,26 @@
  "id": "strum 0.26.2",
  "target": "strum"
  },
+ {
+ "id": "tokio 1.36.0",
+ "target": "tokio"
+ },
  {
  "id": "url 2.5.0",
  "target": "url"
  }
  ],
  "selects": {}
  },
+ "deps_dev": {
+ "common": [
+ {
+ "id": "wiremock 0.6.0",
+ "target": "wiremock"
+ }
+ ],
+ "selects": {}
+ },
  "edition": "2021",
  "proc_macro_deps": {
  "common": [

Cargo.toml

@@ -175,6 +175,7 @@ tokio = { version = "1.2.0", features = ["full"] }
 url = "2.5.0"
 urlencoding = "2.1.0"
 warp = "0.3"
+wiremock = "0.6.0"
 
 
 [profile.release]

dashboard/package.json

@@ -7,9 +7,9 @@
  },
  "scripts": {
  "dev-staging": "concurrently \"yarn start\" \"yarn start-backend\" \"yarn start-rust-backend-staging\"",
- "start-rust-backend-staging": "cd .. && BACKEND_PORT=8081 RUST_BACKTRACE=1 NETWORK=staging NNS_URL=http:https://[2600:3000:6100:200:5000:b0ff:fe8e:6b7b]:8080 cargo watch -C rs/ic-management-backend -x 'run --release --color=always'",
+ "start-rust-backend-staging": "cd .. && BACKEND_PORT=8081 RUST_BACKTRACE=1 cargo watch -C rs/ic-management-backend -x 'run --release --color=always -- --network staging'",
  "dev-mainnet": "concurrently \"yarn start\" \"yarn start-backend\" \"yarn start-rust-backend-mainnet\"",
- "start-rust-backend-mainnet": "cd .. && RUST_BACKTRACE=1 NETWORK=mainnet NNS_URL=https://ic0.app cargo watch -C rs/ic-management-backend -x 'run --release --color=always'",
+ "start-rust-backend-mainnet": "cd .. && RUST_BACKTRACE=1 cargo watch -C rs/ic-management-backend -x 'run --release --color=always -- --network mainnet'",
  "dev": "concurrently \"yarn start\" \"yarn start-backend\" \"yarn start-rust-backend-staging\" \"yarn start-rust-backend-mainnet\"",
  "start": "yarn workspace app start",
  "start-backend": "yarn workspace backend start",
@@ -71,4 +71,4 @@
  ]
  },
  "dependencies": {}
-}
+}

rs/cli/Cargo.toml

@@ -60,7 +60,7 @@ url = { workspace = true }
 tempfile = "3.10.0"
 
 [dev-dependencies]
-wiremock = "0.6.0"
+wiremock = { workspace = true }
 
 [[bin]]
 name = "dre"

rs/cli/src/cli.rs

@@ -4,9 +4,9 @@ use clap::{Parser, Subcommand};
 use clap_num::maybe_hex;
 use ic_base_types::PrincipalId;
 use ic_management_types::{Artifact, Network};
-use log::error;
+use url::Url;
 
-use crate::detect_neuron::{detect_hsm_auth, detect_neuron, Auth, Neuron};
+use crate::detect_neuron::Neuron;
 
 // For more info about the version setup, look at https://docs.rs/clap/latest/clap/struct.Command.html#method.version
 #[derive(Parser, Clone, Default)]
@@ -38,9 +38,14 @@ pub struct Opts {
  #[clap(long, env = "VERBOSE", global = true)]
  pub verbose: bool,
 
- // Specify the target network: "mainnet" (default), "staging", or NNS URL
+ // Specify the target network: "mainnet" (default), "staging", or a testnet name
  #[clap(long, env = "NETWORK", default_value = "mainnet")]
- pub network: Network,
+ pub network: String,
+
+ // NNS_URLs for the target network, comma separated.
+ // The argument is mandatory for testnets, and is optional for mainnet and staging
+ #[clap(long, env = "NNS_URLS", aliases = &["registry-url", "nns-url"], value_delimiter = ',')]
+ pub nns_urls: Vec<Url>,
 
  #[clap(subcommand)]
  pub subcommand: Commands,
@@ -409,11 +414,11 @@ pub mod nodes {
 }
 
 #[derive(Clone)]
-pub struct Cli {
- pub ic_admin: Option<String>,
- pub nns_url: url::Url,
+pub struct ParsedCli {
+ pub network: Network,
+ pub ic_admin_bin_path: Option<String>,
  pub yes: bool,
- pub neuron: Option<Neuron>,
+ pub neuron: Neuron,
 }
 
 #[derive(Clone)]
@@ -427,15 +432,11 @@ pub struct UpdateVersion {
  pub versions_to_retire: Option<Vec<String>>,
 }
 
-impl Cli {
- pub fn get_neuron(&self) -> &Option<Neuron> {
+impl ParsedCli {
+ pub fn get_neuron(&self) -> &Neuron {
  &self.neuron
  }
 
- pub fn get_nns_url(&self) -> &url::Url {
- &self.nns_url
- }
-
  pub fn get_update_cmd_args(update_version: &UpdateVersion) -> Vec<String> {
  [
  [
@@ -462,39 +463,29 @@ impl Cli {
  }
 
  pub async fn from_opts(opts: &Opts, require_authentication: bool) -> anyhow::Result<Self> {
- let nns_url = opts.network.get_url();
- let neuron = if let Some(id) = opts.neuron_id {
- Some(Neuron {
- id,
- auth: if let Some(path) = opts.private_key_pem.clone() {
- Auth::Keyfile { path }
- } else if let (Some(slot), Some(pin), Some(key_id)) =
- (opts.hsm_slot, opts.hsm_pin.clone(), opts.hsm_key_id.clone())
- {
- Auth::Hsm { pin, slot, key_id }
- } else {
- detect_hsm_auth()?
- .ok_or_else(|| anyhow::anyhow!("No valid authentication method found for neuron: {id}"))?
- },
- })
- } else if require_authentication {
- // Early warn if there will be a problem because a neuron was not detected.
- match detect_neuron(nns_url.clone()).await {
- Ok(Some(n)) => Some(n),
- Ok(None) => {
- error!("No neuron detected. Your HSM device is not detectable (or override variables HSM_PIN, HSM_SLOT, HSM_KEY_ID are incorrectly set); your variables NEURON_ID, PRIVATE_KEY_PEM might not be defined either.");
- None
- },
- Err(e) => return Err(anyhow::anyhow!("Failed to detect neuron: {}. Your HSM device is not detectable (or override variables HSM_PIN, HSM_SLOT, HSM_KEY_ID are incorrectly set); your variables NEURON_ID, PRIVATE_KEY_PEM might not be defined either.", e)),
- }
- } else {
- None
- };
- Ok(Cli {
+ let network = Network::new(&opts.network, &opts.nns_urls).await.map_err(|e| {
+ anyhow::anyhow!(
+ "Failed to parse network from name {} and NNS urls {:?}. Error: {}",
+ opts.network,
+ opts.nns_urls,
+ e
+ )
+ })?;
+ let neuron = Neuron::new(
+ &network,
+ require_authentication,
+ opts.neuron_id,
+ opts.private_key_pem.clone(),
+ opts.hsm_slot.clone(),
+ opts.hsm_pin.clone(),
+ opts.hsm_key_id.clone(),
+ )
+ .await?;
+ Ok(ParsedCli {
+ network: network,
  yes: opts.yes,
  neuron,
- ic_admin: opts.ic_admin.clone(),
- nns_url,
+ ic_admin_bin_path: opts.ic_admin.clone(),
  })
  }
 }

rs/cli/src/clients.rs

@@ -18,7 +18,7 @@ pub struct DashboardBackendClient {
 impl DashboardBackendClient {
  // Only used in tests, which should be cleaned up together with this code.
  #[allow(dead_code)]
- pub fn new(network: Network, dev: bool) -> DashboardBackendClient {
+ pub fn new(network: &Network, dev: bool) -> DashboardBackendClient {
  Self {
  url: reqwest::Url::parse(if !dev {
  "https://dashboard.internal.dfinity.network/"
@@ -28,16 +28,12 @@ impl DashboardBackendClient {
  .expect("invalid base url")
  .join("api/proxy/registry/")
  .expect("failed to join url")
- .join(match network {
- Network::Mainnet => "mainnet/",
- Network::Staging => "staging/",
- Network::Url(_) => "/",
- })
+ .join(&network.name)
  .expect("failed to join url"),
  }
  }
 
- pub fn new_with_network_url(url: String) -> Self {
+ pub fn new_with_backend_url(url: String) -> Self {
  Self {
  url: reqwest::Url::parse(&url).unwrap(),
  }
@@ -154,22 +150,28 @@ impl RESTRequestBuilder for reqwest::RequestBuilder {
 mod tests {
  use super::*;
 
- #[test]
- fn dashboard_backend_client_url() {
+ #[tokio::test]
+ async fn dashboard_backend_client_url() {
+ let mainnet = Network::new("mainnet", &vec![])
+ .await
+ .expect("failed to create mainnet network");
+ let staging = Network::new("staging", &vec![])
+ .await
+ .expect("failed to create staging network");
  assert_eq!(
- DashboardBackendClient::new(Network::Mainnet, false).url.to_string(),
+ DashboardBackendClient::new(&mainnet, false).url.to_string(),
  "https://dashboard.internal.dfinity.network/api/proxy/registry/mainnet/"
  );
  assert_eq!(
- DashboardBackendClient::new(Network::Staging, false).url.to_string(),
+ DashboardBackendClient::new(&staging, false).url.to_string(),
  "https://dashboard.internal.dfinity.network/api/proxy/registry/staging/"
  );
  assert_eq!(
- DashboardBackendClient::new(Network::Mainnet, true).url.to_string(),
+ DashboardBackendClient::new(&mainnet, true).url.to_string(),
  "http:https://localhost:17000/api/proxy/registry/mainnet/"
  );
  assert_eq!(
- DashboardBackendClient::new(Network::Staging, true).url.to_string(),
+ DashboardBackendClient::new(&staging, true).url.to_string(),
  "http:https://localhost:17000/api/proxy/registry/staging/"
  );
  }